From 739c4089dbb14d9a7e7c857dbcd24dfe4ab7e71c Mon Sep 17 00:00:00 2001 From: PeterDKC Date: Tue, 4 Oct 2022 10:24:16 -0500 Subject: [PATCH 1/3] [WL5] Import State Data --- app/Console/Commands/LegiScanImport.php | 108 ++++++++++++------ app/Models/LegiScan/Bill.php | 75 ------------ app/Models/LegiScan/State.php | 10 +- database/factories/LegiScan/StateFactory.php | 13 +-- .../2022_10_03_200810_create_states_table.php | 22 ++++ tests/Unit/Models/LegiScan/StateTest.php | 2 +- 6 files changed, 105 insertions(+), 125 deletions(-) create mode 100644 database/migrations/2022_10_03_200810_create_states_table.php diff --git a/app/Console/Commands/LegiScanImport.php b/app/Console/Commands/LegiScanImport.php index 8a00360..8f8693b 100644 --- a/app/Console/Commands/LegiScanImport.php +++ b/app/Console/Commands/LegiScanImport.php @@ -2,58 +2,96 @@ namespace App\Console\Commands; +use App\Models\LegiScan\State; use Illuminate\Console\Command; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; +/** + * @codeCoverageIgnore + */ class LegiScanImport extends Command { - /** - * The name and signature of the console command. - * - * @var string - */ - protected $signature = 'legiscan:import'; - - /** - * The console command description. - * - * @var string - */ + protected $signature = 'legiscan:import {--d|debug}'; + protected $description = 'Regularly scheduled LegiScan import command'; - /** - * Execute the console command. - * - * @return int - */ + protected string $separator = '--------------------------------'; + public function handle() { - // RUN THIS COMMAND USING "SAIL" (WHEN IN LOCAL MODE) !!!!!!!!!!!! + $start = now(); + + $this->importLegiscanData() + ->translateStates(); + + $this->info(''); + $this->info( + 'Finished in ' . now()->diff($start)->format('%I:%S') + ); + $this->info(''); + } + + protected function translateStates(): static + { + $this->info('Translating State Data'); + + $states = DB::select('SELECT * FROM ls_state'); - $script_filepath = base_path('lib/legiscan/legiscan-bulk.php'); + $progress = $this->output->createProgressBar(count($states)); - $command = ''; + $progress->start(); - $command .= ' HOST='.config('database.connections.mysql.host'); - $command .= ' PORT='.config('database.connections.mysql.port'); - $command .= ' NAME='.config('database.connections.mysql.database'); - $command .= ' USER='.config('database.connections.mysql.username'); - $command .= ' PASS='.config('database.connections.mysql.password'); + foreach ($states as $state) { + State::updateOrCreate( + ['id' => $state->state_id], + [ + 'id' => $state->state_id, + 'name' => $state->state_name, + 'abbreviation' => $state->state_abbr, + ] + ); - $command .= ' LEGISCAN_API_KEY='.config('legiscan.api_key'); - $command .= ' MAIL_FROM_ADDRESS='.config('mail.from.address'); + $progress->advance(); + } - $command .= ' php'; - // $command .= ' -d display_errors 0'; - // $command .= ' -d error_reporting 5'; // Same as (E_ERROR | E_PARSE); see https://www.php.net/manual/en/errorfunc.constants.php - $command .= ' ' . $script_filepath; - $command .= ' --bulk'; - $command .= ' --import'; - $command .= ' --yes'; + $progress->finish(); - // echo $command; + $this->info("\n".$this->separator); + + return $this; + } + + protected function importLegiscanData(): static + { + $mysqlConfig = config('database.connections.mysql'); + $apiKey = config('legiscan.api_key'); + $mailFrom = config('mail.from.address'); + $debug = $this->option('debug') ? '-d display_errors 0 -d error_reporting 5 ' : ''; + $scriptFilepath = base_path('lib/legiscan/legiscan-bulk.php'); + + $command = sprintf( + 'HOST=%s PORT=%s NAME=%s USER=%s PASS=%s LEGISCAN_API_KEY=%s MAIL_FROM_ADDRESS=%s php %s%s --bulk --import --yes', + $mysqlConfig['host'], + $mysqlConfig['port'], + $mysqlConfig['database'], + $mysqlConfig['username'], + $mysqlConfig['password'], + $apiKey, + $mailFrom, + $debug, + $scriptFilepath, + ); + + $this->info('Importing LegiScan data. This may take a minute.'); + exec($command); Log::info('LegiScan import completed'); + + $this->info('LegiScan import completed.'); + $this->info($this->separator); + + return $this; } } diff --git a/app/Models/LegiScan/Bill.php b/app/Models/LegiScan/Bill.php index 4a2a464..6d7429d 100644 --- a/app/Models/LegiScan/Bill.php +++ b/app/Models/LegiScan/Bill.php @@ -128,79 +128,4 @@ public function votes() { return $this->hasMany(BillVote::class, 'bill_id'); } - - #endregion - - #region Foreign model assignment attributes - - public function setStateAttribute(State|int $state) - { - $state = is_int($state) ? State::find($state) : $state; - - $this->state_id = is_int($state) ? $state : $state->getKey(); - $this->setRelation('state', $state); - } - - public function setSessionAttribute(Session|int $session) - { - $session = is_int($session) ? Session::find($session) : $session; - - $this->session_id = $session->getKey(); - $this->setRelation('session', $session); - } - - public function setBodyAttribute(Body|int $body) - { - $body = is_int($body) ? Body::find($body) : $body; - - $this->body_id = $body->getKey(); - $this->setRelation('body', $body); - } - - public function setCurrentBodyAttribute(Body|int $current_body) - { - $current_body = is_int($current_body) ? Body::find($current_body) : $current_body; - - $this->body_id = $current_body->getKey(); - $this->setRelation('current_body', $current_body); - } - - public function setBillTypeAttribute(Type|int $bill_type) - { - $bill_type = is_int($bill_type) ? Type::find($bill_type) : $bill_type; - - $this->body_id = $bill_type->getKey(); - $this->setRelation('bill_type', $bill_type); - } - - public function setStatusAttribute(Progress|int $status) - { - $status = is_int($status) ? Progress::find($status) : $status; - - $this->body_id = $status->getKey(); - $this->setRelation('progress', $status); - } - - public function setPendingCommitteeAttribute(Committee|int $pending_committee) - { - $pending_committee = is_int($pending_committee) ? Committee::find($pending_committee) : $pending_committee; - - $this->body_id = $pending_committee->getKey(); - $this->setRelation('pending_committee', $pending_committee); - } - - #endregion - - #region Simplify naming attributes - - public function getNumberAttribute() - { - return $this->bill_number; - } - public function setNumberAttribute($number) - { - $this->bill_number = $number; - } - - #endregion } diff --git a/app/Models/LegiScan/State.php b/app/Models/LegiScan/State.php index 3290a0c..bd8748a 100644 --- a/app/Models/LegiScan/State.php +++ b/app/Models/LegiScan/State.php @@ -9,12 +9,14 @@ class State extends Model { use HasFactory; - protected $table = 'ls_state'; - - protected $primaryKey = 'state_id'; - public $timestamps = false; + protected $fillable = [ + 'id', + 'name', + 'abbreviation', + ]; + public function bills() { return $this->hasMany(Bill::class, 'state_id'); diff --git a/database/factories/LegiScan/StateFactory.php b/database/factories/LegiScan/StateFactory.php index c1ea280..6ebd439 100644 --- a/database/factories/LegiScan/StateFactory.php +++ b/database/factories/LegiScan/StateFactory.php @@ -15,17 +15,10 @@ class StateFactory extends Factory public function definition() { - static $increment = 1; - return [ - 'state_id' => $increment++, - 'state_abbr' => $this->faker->stateAbbr, - 'state_name' => $this->faker->state, - 'biennium' => 0, - 'carry_over' => Arr::random(['YES', 'NO']), - 'capitol' => $this->faker->city, - 'latitude' => $this->faker->randomFloat(), - 'longitude' => $this->faker->randomFloat(), + 'id' => (State::all()->max('id') ?? 0) + 1, + 'name' => $this->faker->state, + 'abbreviation' => $this->faker->stateAbbr, ]; } } diff --git a/database/migrations/2022_10_03_200810_create_states_table.php b/database/migrations/2022_10_03_200810_create_states_table.php new file mode 100644 index 0000000..a5ea945 --- /dev/null +++ b/database/migrations/2022_10_03_200810_create_states_table.php @@ -0,0 +1,22 @@ +unsignedInteger('id'); + $table->string('name'); + $table->string('abbreviation'); + }); + } + + public function down() + { + Schema::dropIfExists('states'); + } +}; diff --git a/tests/Unit/Models/LegiScan/StateTest.php b/tests/Unit/Models/LegiScan/StateTest.php index 18f65c6..309843e 100644 --- a/tests/Unit/Models/LegiScan/StateTest.php +++ b/tests/Unit/Models/LegiScan/StateTest.php @@ -13,7 +13,7 @@ public function statesHaveManyBills() { $state = State::factory()->create(); Bill::factory()->create([ - 'state' => $state, + 'state_id' => $state->id, 'description' => 'A saucy little piece of legislation.', ]); From 41c0b995dd5a0a1c3291bfd51262ebb6f857e5d4 Mon Sep 17 00:00:00 2001 From: PeterDKC Date: Tue, 4 Oct 2022 11:46:51 -0500 Subject: [PATCH 2/3] Add in capitol, lat & long --- app/Console/Commands/LegiScanImport.php | 3 +++ app/Models/LegiScan/State.php | 3 +++ database/factories/LegiScan/StateFactory.php | 3 +++ database/migrations/2022_10_03_200810_create_states_table.php | 3 +++ 4 files changed, 12 insertions(+) diff --git a/app/Console/Commands/LegiScanImport.php b/app/Console/Commands/LegiScanImport.php index 8f8693b..c5de6e2 100644 --- a/app/Console/Commands/LegiScanImport.php +++ b/app/Console/Commands/LegiScanImport.php @@ -49,6 +49,9 @@ protected function translateStates(): static 'id' => $state->state_id, 'name' => $state->state_name, 'abbreviation' => $state->state_abbr, + 'capitol' => $state->capitol, + 'latitude' => $state->latitude, + 'longitude' => $state->longitude, ] ); diff --git a/app/Models/LegiScan/State.php b/app/Models/LegiScan/State.php index bd8748a..f5e9494 100644 --- a/app/Models/LegiScan/State.php +++ b/app/Models/LegiScan/State.php @@ -15,6 +15,9 @@ class State extends Model 'id', 'name', 'abbreviation', + 'capitol', + 'latitude', + 'longitude', ]; public function bills() diff --git a/database/factories/LegiScan/StateFactory.php b/database/factories/LegiScan/StateFactory.php index 6ebd439..f4d06b0 100644 --- a/database/factories/LegiScan/StateFactory.php +++ b/database/factories/LegiScan/StateFactory.php @@ -19,6 +19,9 @@ public function definition() 'id' => (State::all()->max('id') ?? 0) + 1, 'name' => $this->faker->state, 'abbreviation' => $this->faker->stateAbbr, + 'capitol' => $this->faker->city, + 'latitude' => $this->faker->latitude, + 'longitude' => $this->faker->longitude, ]; } } diff --git a/database/migrations/2022_10_03_200810_create_states_table.php b/database/migrations/2022_10_03_200810_create_states_table.php index a5ea945..0b6ef72 100644 --- a/database/migrations/2022_10_03_200810_create_states_table.php +++ b/database/migrations/2022_10_03_200810_create_states_table.php @@ -12,6 +12,9 @@ public function up() $table->unsignedInteger('id'); $table->string('name'); $table->string('abbreviation'); + $table->string('capitol'); + $table->string('latitude'); + $table->string('longitude'); }); } From 768ed3ee5452656e43c7dcadb612e40caf19e214 Mon Sep 17 00:00:00 2001 From: PeterDKC Date: Tue, 4 Oct 2022 14:00:57 -0500 Subject: [PATCH 3/3] Add carry_over and biennium fields --- app/Console/Commands/LegiScanImport.php | 2 + app/Exceptions/InvalidEnumException.php | 9 ++ app/Models/LegiScan/State.php | 9 +- app/Traits/Models/HasEnumProperties.php | 125 ++++++++++++++++++ database/factories/LegiScan/StateFactory.php | 2 + .../2022_10_03_200810_create_states_table.php | 2 + 6 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 app/Exceptions/InvalidEnumException.php create mode 100644 app/Traits/Models/HasEnumProperties.php diff --git a/app/Console/Commands/LegiScanImport.php b/app/Console/Commands/LegiScanImport.php index c5de6e2..1ac9dad 100644 --- a/app/Console/Commands/LegiScanImport.php +++ b/app/Console/Commands/LegiScanImport.php @@ -49,6 +49,8 @@ protected function translateStates(): static 'id' => $state->state_id, 'name' => $state->state_name, 'abbreviation' => $state->state_abbr, + 'biennium' => $state->biennium, + 'carry_over' => $state->carry_over, 'capitol' => $state->capitol, 'latitude' => $state->latitude, 'longitude' => $state->longitude, diff --git a/app/Exceptions/InvalidEnumException.php b/app/Exceptions/InvalidEnumException.php new file mode 100644 index 0000000..937aa08 --- /dev/null +++ b/app/Exceptions/InvalidEnumException.php @@ -0,0 +1,9 @@ +hasMany(Bill::class, 'state_id'); diff --git a/app/Traits/Models/HasEnumProperties.php b/app/Traits/Models/HasEnumProperties.php new file mode 100644 index 0000000..d3695dc --- /dev/null +++ b/app/Traits/Models/HasEnumProperties.php @@ -0,0 +1,125 @@ +hasEnumProperty($field)) { + $property = $instance->getEnumProperty($field); + + return $instance->$property; + } + + return false; + } + + /** + * Check for the presence of a property that starts + * with enum for the provided attribute + * + * @param string $field + * @param mixed $value + * @return $this + * @throws InvalidEnumException + */ + public function setAttribute($field, $value) + { + if ($this->hasEnumProperty($field)) { + if (!$this->isValidEnum($field, $value)) { + throw new InvalidEnumException('Invalid value for ' . static::class . "::$field ($value)"); + } + + if ($this->isKeyedEnum($field, $value)) { + $value = $this->getKeyedEnum($field, $value); + } + } + + return parent::setAttribute($field, $value); + } + + /** + * Gets the expected enum property + * + * @param string $field + * @return string + */ + protected function getEnumProperty(string $field) + { + return 'enum' . Str::plural(Str::studly($field)); + } + + /** + * Gets the enum value by key + * + * @param string $field + * @param mixed $key + * @return mixed + */ + protected function getKeyedEnum(string $field, $key) + { + return static::getEnum($field)[$key]; + } + + /** + * Is an enum property defined for the provided field + * + * @param string $field + * @return bool + */ + protected function hasEnumProperty(string $field) + { + $property = $this->getEnumProperty($field); + + return isset($this->$property) && is_array($this->$property); + } + + /** + * Is the provided value a key in the enum + * + * @param string $field + * @param mixed $key + * @return bool + */ + protected function isKeyedEnum(string $field, $key) + { + return in_array($key, array_keys(static::getEnum($field)), true); + } + + /** + * Is the value a valid enum in any way + * + * @param string $field + * @param mixed $value + * @return bool + */ + protected function isValidEnum(string $field, $value) + { + return $this->isValueEnum($field, $value) || + $this->isKeyedEnum($field, $value); + } + + /** + * Is the provided value in the enum + * + * @param string $field + * @param mixed $value + * @return bool + */ + protected function isValueEnum(string $field, $value) + { + return in_array($value, static::getEnum($field)); + } +} diff --git a/database/factories/LegiScan/StateFactory.php b/database/factories/LegiScan/StateFactory.php index f4d06b0..34525cc 100644 --- a/database/factories/LegiScan/StateFactory.php +++ b/database/factories/LegiScan/StateFactory.php @@ -19,6 +19,8 @@ public function definition() 'id' => (State::all()->max('id') ?? 0) + 1, 'name' => $this->faker->state, 'abbreviation' => $this->faker->stateAbbr, + 'biennium' => rand(0, 1), + 'carry_over' => Arr::random(State::getEnum('carryOver')), 'capitol' => $this->faker->city, 'latitude' => $this->faker->latitude, 'longitude' => $this->faker->longitude, diff --git a/database/migrations/2022_10_03_200810_create_states_table.php b/database/migrations/2022_10_03_200810_create_states_table.php index 0b6ef72..66ad128 100644 --- a/database/migrations/2022_10_03_200810_create_states_table.php +++ b/database/migrations/2022_10_03_200810_create_states_table.php @@ -12,6 +12,8 @@ public function up() $table->unsignedInteger('id'); $table->string('name'); $table->string('abbreviation'); + $table->string('biennium'); + $table->string('carry_over'); $table->string('capitol'); $table->string('latitude'); $table->string('longitude');