feat: presidential results import (#96)

app/Jobs/Presidential241124/Records/FetchRecordsJob.php

@@ -0,0 +1,154 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Jobs\Presidential241124\Records;
+
+use App\Jobs\DeleteTemporaryTableData;
+use App\Jobs\PersistTemporaryTableData;
+use App\Jobs\SchedulableJob;
+use App\Jobs\UpdateElectionRecordsTimestamp;
+use App\Models\County;
+use App\Models\Record;
+use App\Models\Vote;
+use Illuminate\Support\Facades\Bus;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+use Spatie\TemporaryDirectory\TemporaryDirectory;
+
+class FetchRecordsJob extends SchedulableJob
+{
+    public static function name(): string
+    {
+        return 'Prezidențiale 24.11.2024 / Procese Verbale';
+    }
+
+    public function execute(): void
+    {
+        $temporaryDirectory = TemporaryDirectory::make()
+            ->deleteWhenDestroyed();
+
+        $cwd = $temporaryDirectory->path();
+
+        $tmpDisk = Storage::build([
+            'driver' => 'local',
+            'root' => $cwd,
+        ]);
+
+        $tmpDisk->put('turnout.csv', $this->scheduledJob->fetchSource()->resource());
+
+        // Split the CSV by county
+        Process::path($cwd)
+            ->run([
+                config('import.awk_path'),
+                '-F,',
+                'FNR==1 {header = $0; next} !seen[$1]++ {print header > $1".csv"} {print > $1".csv"}',
+                'turnout.csv',
+            ]);
+
+        $tmpDisk->delete('turnout.csv');
+
+        collect($tmpDisk->allFiles())
+            ->each(function (string $file) use ($tmpDisk) {
+                $this->scheduledJob->disk()
+                    ->writeStream(
+                        $this->scheduledJob->getSourcePath($file),
+                        $tmpDisk->readStream($file)
+                    );
+            });
+
+        $counties = County::all();
+
+        $electionName = $this->scheduledJob->election->getFilamentName();
+        $electionId = $this->scheduledJob->election_id;
+
+        $time = now()->toDateTimeString();
+
+        $sourceFiles = collect([
+            '1' => 'AB',
+            '2' => 'AR',
+            '3' => 'AG',
+            '4' => 'BC',
+            '5' => 'BH',
+            '6' => 'BN',
+            '7' => 'BT',
+            '8' => 'BV',
+            '9' => 'BR',
+            '10' => 'BZ',
+            '11' => 'CS',
+            '12' => 'CL',
+            '13' => 'CJ',
+            '14' => 'CT',
+            '15' => 'CV',
+            '16' => 'DB',
+            '17' => 'DJ',
+            '18' => 'GL',
+            '19' => 'GR',
+            '20' => 'GJ',
+            '21' => 'HR',
+            '22' => 'HD',
+            '23' => 'IL',
+            '24' => 'IS',
+            '25' => 'IF',
+            '26' => 'MM',
+            '27' => 'MH',
+            '28' => 'MS',
+            '29' => 'NT',
+            '30' => 'OT',
+            '31' => 'PH',
+            '32' => 'SM',
+            '33' => 'SJ',
+            '34' => 'SB',
+            '35' => 'SV',
+            '36' => 'TR',
+            '37' => 'TM',
+            '38' => 'TL',
+            '39' => 'VS',
+            '40' => 'VL',
+            '41' => 'VN',
+
+            '44' => 'B',
+            '45' => 'B',
+            '46' => 'B',
+            '47' => 'B',
+            '48' => 'B',
+            '49' => 'B',
+        ]);
+
+        $jobs = $sourceFiles
+            ->map(fn (string $countyCode, string $filename) => new ImportCountyRecordsJob($this->scheduledJob, $countyCode, $filename))
+            ->push(new ImportAbroadRecordsJob($this->scheduledJob));
+
+        $persistAndClean = fn () => Bus::chain([
+            new PersistTemporaryTableData(Record::class, $electionId),
+            new DeleteTemporaryTableData(Record::class, $electionId),
+
+            new PersistTemporaryTableData(Vote::class, $electionId),
+            new DeleteTemporaryTableData(Vote::class, $electionId),
+        ])->dispatch();
+
+        Bus::batch($jobs)
+            ->catch($persistAndClean)
+            ->then($persistAndClean)
+            ->then(fn () => UpdateElectionRecordsTimestamp::dispatch($electionId))
+            ->name("$electionName / Rezultate / $time")
+            ->allowFailures()
+            ->dispatch();
+    }
+
+    /**
+     * Get the tags that should be assigned to the job.
+     *
+     * @return array<int, string>
+     */
+    public function tags(): array
+    {
+        return [
+            'import',
+            'records',
+            'scheduled_job:' . $this->scheduledJob->id,
+            'election:' . $this->scheduledJob->election_id,
+            static::name(),
+        ];
+    }
+}

app/Jobs/Presidential241124/Records/ImportAbroadRecordsJob.php

@@ -0,0 +1,136 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Jobs\Presidential241124\Records;
+
+use App\Events\CountryCodeNotFound;
+use App\Exceptions\CountryCodeNotFoundException;
+use App\Exceptions\MissingSourceFileException;
+use App\Models\Country;
+use App\Models\Record;
+use App\Models\ScheduledJob;
+use App\Models\Vote;
+use App\Services\RecordService;
+use Illuminate\Bus\Batchable;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use League\Csv\Reader;
+
+class ImportAbroadRecordsJob implements ShouldQueue
+{
+    use Batchable;
+    use Dispatchable;
+    use InteractsWithQueue;
+    use Queueable;
+    use SerializesModels;
+
+    public ScheduledJob $scheduledJob;
+
+    public function __construct(ScheduledJob $scheduledJob)
+    {
+        $this->scheduledJob = $scheduledJob;
+    }
+
+    public function handle(): void
+    {
+        $disk = $this->scheduledJob->disk();
+        $path = $this->scheduledJob->getSourcePath('43.csv');
+
+        if (! $disk->exists($path)) {
+            throw new MissingSourceFileException($path);
+        }
+
+        $reader = Reader::createFromStream($disk->readStream($path));
+        $reader->setHeaderOffset(0);
+
+        $records = collect();
+
+        $votables = RecordService::generateVotables(
+            $reader->getHeader(),
+            $this->scheduledJob->election_id
+        );
+
+        foreach ($reader->getRecords() as $row) {
+            try {
+                $countryId = $this->getCountryId($row['uat_name']);
+
+                $part = RecordService::getPart($row['report_stage_code']);
+
+                $records->push([
+                    'election_id' => $this->scheduledJob->election_id,
+                    'country_id' => $countryId,
+                    'section' => $row['precinct_nr'],
+                    'part' => $part,
+
+                    'eligible_voters_permanent' => $row['a'],
+                    'eligible_voters_special' => 0,
+
+                    'present_voters_permanent' => $row['b1'],
+                    'present_voters_special' => $row['b2'],
+                    'present_voters_supliment' => $row['b3'],
+                    'present_voters_mail' => 0, //$row['b4'],
+
+                    'votes_valid' => $row['c'],
+                    'votes_null' => $row['d'],
+
+                    'papers_received' => $row['e'],
+                    'papers_unused' => $row['f'],
+
+                    'has_issues' => false,
+                ]);
+
+                $votes = collect();
+                foreach ($votables as $column => $votable) {
+                    $votes->push([
+                        'election_id' => $this->scheduledJob->election_id,
+                        'country_id' => $countryId,
+                        'section' => $row['precinct_nr'],
+                        'part' => $part,
+
+                        'votable_type' => $votable['votable_type'],
+                        'votable_id' => $votable['votable_id'],
+
+                        'votes' => $row[$column],
+                    ]);
+                }
+
+                Vote::saveToTemporaryTable($votes->all());
+            } catch (CountryCodeNotFoundException $th) {
+                CountryCodeNotFound::dispatch($row['uat_name'], $this->scheduledJob->election);
+            }
+        }
+
+        Record::saveToTemporaryTable($records->all());
+    }
+
+    protected function getCountryId(string $name): string
+    {
+        $country = Country::search($name)->first();
+
+        if (! $country) {
+            throw new CountryCodeNotFoundException($name);
+        }
+
+        return $country->id;
+    }
+
+    /**
+     * Get the tags that should be assigned to the job.
+     *
+     * @return array<int, string>
+     */
+    public function tags(): array
+    {
+        return [
+            'import',
+            'records',
+            'scheduled_job:' . $this->scheduledJob->id,
+            'election:' . $this->scheduledJob->election_id,
+            'abroad',
+        ];
+    }
+}