From dcf7ef42624ded7704bee56c098ed4ba674fbfec Mon Sep 17 00:00:00 2001 From: Fityan <63894003+fityannugroho@users.noreply.github.com> Date: Sat, 23 Nov 2024 13:11:08 +0700 Subject: [PATCH] feat: improve seeder (#466) --- prisma/mongodb/schema.prisma | 8 + prisma/mongodb/seeder.ts | 116 ++---------- .../migration.sql | 8 + prisma/mysql/schema.prisma | 8 + .../migration.sql | 8 + prisma/postgresql/schema.prisma | 8 + prisma/seed.ts | 29 ++- prisma/seeder.ts | 165 +++++------------- .../migration.sql | 6 + prisma/sqlite/schema.prisma | 8 + src/common/utils/__tests__/package.spec.ts | 68 ++++++++ src/common/utils/package.ts | 37 ++++ 12 files changed, 226 insertions(+), 243 deletions(-) create mode 100644 prisma/mysql/migrations/20241123043002_create_seeder_logs_table/migration.sql create mode 100644 prisma/postgresql/migrations/20241122141040_create_seeder_logs_table/migration.sql create mode 100644 prisma/sqlite/migrations/20241123043745_create_seeder_logs_table/migration.sql create mode 100644 src/common/utils/__tests__/package.spec.ts create mode 100644 src/common/utils/package.ts diff --git a/prisma/mongodb/schema.prisma b/prisma/mongodb/schema.prisma index 543bc49..a903b34 100644 --- a/prisma/mongodb/schema.prisma +++ b/prisma/mongodb/schema.prisma @@ -52,6 +52,14 @@ model Regency { @@map("regencies") } +model SeederLogs { + id String @id @default(auto()) @map("_id") @db.ObjectId + dataVersion String @map("data_version") + createdAt DateTime @default(now()) @map("created_at") + + @@map("seeder_logs") +} + model Village { id String @id @default(auto()) @map("_id") @db.ObjectId code String @unique diff --git a/prisma/mongodb/seeder.ts b/prisma/mongodb/seeder.ts index 4c32664..b747673 100644 --- a/prisma/mongodb/seeder.ts +++ b/prisma/mongodb/seeder.ts @@ -1,92 +1,23 @@ -import { areArraysEqual } from '@/common/utils/array'; -import { - getDistricts, - getIslands, - getProvinces, - getRegencies, - getVillages, -} from '@/common/utils/data'; import { PrismaClient } from '@prisma/client'; -import { Areas } from 'idn-area-data'; -import { Seeder } from '../seeder'; +import { Area, Seeder } from '../seeder'; export class MongodbSeeder extends Seeder { constructor(prisma: PrismaClient) { super(prisma); } - async hasProvinceChanges(): Promise { - const [newProvinces, oldProvinces] = await Promise.all([ - getProvinces(), - this.prisma.province.findMany(), - ]); + async deleteAreas(area: Area): Promise { + const mongoCollectionMap = { + province: 'provinces', + regency: 'regencies', + district: 'districts', + village: 'villages', + island: 'islands', + }; - return !areArraysEqual(newProvinces, oldProvinces, ['code', 'name']); - } - - async hasRegencyChanges(): Promise { - const [newRegencies, oldRegencies] = await Promise.all([ - getRegencies(), - this.prisma.regency.findMany(), - ]); - - return !areArraysEqual(newRegencies, oldRegencies, [ - 'code', - 'name', - 'provinceCode', - ]); - } - - async hasDistrictChanges(): Promise { - const [newDistricts, oldDistricts] = await Promise.all([ - getDistricts(), - this.prisma.district.findMany(), - ]); - - return !areArraysEqual(newDistricts, oldDistricts, [ - 'code', - 'name', - 'regencyCode', - ]); - } - - async hasIslandChanges(): Promise { - const [newIslands, oldIslands] = await Promise.all([ - getIslands(), - this.prisma.island.findMany(), - ]); - - return !areArraysEqual(newIslands, oldIslands, [ - 'code', - 'name', - 'regencyCode', - 'isPopulated', - 'isOutermostSmall', - ]); - } - - async hasVillageChanges(): Promise { - const [newVillages, oldVillages] = await Promise.all([ - getVillages(), - this.prisma.village.findMany(), - ]); - - return !areArraysEqual(newVillages, oldVillages, [ - 'code', - 'name', - 'districtCode', - ]); - } - - /** - * Delete all data in a collection. - */ - protected async deleteCollection(collection: Areas) { - // Skip TypeScript checking because `$runCommandRaw()` method only available for mongodb provider. - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - return await this.prisma.$runCommandRaw({ - delete: collection, + // @ts-ignore Skip TypeScript checking because `$runCommandRaw()` method only available for mongodb provider. + const res = await this.prisma.$runCommandRaw({ + delete: mongoCollectionMap[area], deletes: [ { q: {}, @@ -94,30 +25,7 @@ export class MongodbSeeder extends Seeder { }, ], }); - } - - async deleteProvinces(): Promise { - const res = await this.deleteCollection('provinces'); - return res.n as number; - } - - async deleteRegencies(): Promise { - const res = await this.deleteCollection('regencies'); - return res.n as number; - } - - async deleteDistricts(): Promise { - const res = await this.deleteCollection('districts'); - return res.n as number; - } - - async deleteVillages(): Promise { - const res = await this.deleteCollection('villages'); - return res.n as number; - } - async deleteIslands(): Promise { - const res = await this.deleteCollection('islands'); return res.n as number; } } diff --git a/prisma/mysql/migrations/20241123043002_create_seeder_logs_table/migration.sql b/prisma/mysql/migrations/20241123043002_create_seeder_logs_table/migration.sql new file mode 100644 index 0000000..4c7c812 --- /dev/null +++ b/prisma/mysql/migrations/20241123043002_create_seeder_logs_table/migration.sql @@ -0,0 +1,8 @@ +-- CreateTable +CREATE TABLE `seeder_logs` ( + `id` INTEGER NOT NULL AUTO_INCREMENT, + `data_version` VARCHAR(255) NOT NULL, + `created_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/prisma/mysql/schema.prisma b/prisma/mysql/schema.prisma index 38bb77b..d35e674 100644 --- a/prisma/mysql/schema.prisma +++ b/prisma/mysql/schema.prisma @@ -49,6 +49,14 @@ model Regency { @@map("regencies") } +model SeederLogs { + id Int @id @default(autoincrement()) + dataVersion String @map("data_version") @db.VarChar(255) + createdAt DateTime @default(now()) @map("created_at") + + @@map("seeder_logs") +} + model Village { code String @id @db.VarChar(10) districtCode String @map("district_code") @db.VarChar(6) diff --git a/prisma/postgresql/migrations/20241122141040_create_seeder_logs_table/migration.sql b/prisma/postgresql/migrations/20241122141040_create_seeder_logs_table/migration.sql new file mode 100644 index 0000000..3b27d79 --- /dev/null +++ b/prisma/postgresql/migrations/20241122141040_create_seeder_logs_table/migration.sql @@ -0,0 +1,8 @@ +-- CreateTable +CREATE TABLE "seeder_logs" ( + "id" SERIAL NOT NULL, + "data_version" VARCHAR(255) NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "seeder_logs_pkey" PRIMARY KEY ("id") +); diff --git a/prisma/postgresql/schema.prisma b/prisma/postgresql/schema.prisma index f0a80bd..33405e7 100644 --- a/prisma/postgresql/schema.prisma +++ b/prisma/postgresql/schema.prisma @@ -49,6 +49,14 @@ model Regency { @@map("regencies") } +model SeederLogs { + id Int @id @default(autoincrement()) + dataVersion String @map("data_version") @db.VarChar(255) + createdAt DateTime @default(now()) @map("created_at") + + @@map("seeder_logs") +} + model Village { code String @id @db.VarChar(10) districtCode String @map("district_code") @db.VarChar(6) diff --git a/prisma/seed.ts b/prisma/seed.ts index 820b943..86da003 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -17,32 +17,29 @@ async function main() { seeder = new MongodbSeeder(prisma); } - const hasDataChanges = timify( - seeder.hasDataChanges.bind(seeder), - 'check-data-changes', - ); - // Skip the seeder if if there are no data changes. console.log('Checking for data changes...\n'); - if (!(await hasDataChanges())) { + if (!(await seeder.hasDataChanges())) { console.log('There are no data changes. Seeder is skipped.'); return; } console.log('Data changes found!'); console.log('Deleting all data...'); - await timify(seeder.deleteVillages.bind(seeder), 'delete-villages')(); - await timify(seeder.deleteIslands.bind(seeder), 'delete-islands')(); - await timify(seeder.deleteDistricts.bind(seeder), 'delete-districts')(); - await timify(seeder.deleteRegencies.bind(seeder), 'delete-regencies')(); - await timify(seeder.deleteProvinces.bind(seeder), 'delete-provinces')(); + await timify(() => seeder.deleteAreas('village'), 'delete-villages')(); + await timify(() => seeder.deleteAreas('district'), 'delete-districts')(); + await timify(() => seeder.deleteAreas('island'), 'delete-islands')(); + await timify(() => seeder.deleteAreas('regency'), 'delete-regencies')(); + await timify(() => seeder.deleteAreas('province'), 'delete-provinces')(); console.log('Inserting all data...'); - await timify(seeder.insertProvinces.bind(seeder), 'insert-provinces')(); - await timify(seeder.insertRegencies.bind(seeder), 'insert-regencies')(); - await timify(seeder.insertDistricts.bind(seeder), 'insert-districts')(); - await timify(seeder.insertIslands.bind(seeder), 'insert-islands')(); - await timify(seeder.insertVillages.bind(seeder), 'insert-villages')(); + await timify(() => seeder.insertAreas('province'), 'insert-provinces')(); + await timify(() => seeder.insertAreas('regency'), 'insert-regencies')(); + await timify(() => seeder.insertAreas('island'), 'insert-islands')(); + await timify(() => seeder.insertAreas('district'), 'insert-districts')(); + await timify(() => seeder.insertAreas('village'), 'insert-villages')(); + + await seeder.generateLog(); } main() diff --git a/prisma/seeder.ts b/prisma/seeder.ts index 8eb5d50..0b94862 100644 --- a/prisma/seeder.ts +++ b/prisma/seeder.ts @@ -1,4 +1,3 @@ -import { areArraysEqual } from '@/common/utils/array'; import { getDistricts, getIslands, @@ -6,150 +5,70 @@ import { getRegencies, getVillages, } from '@/common/utils/data'; +import { getInstalledPackageVersion } from '@/common/utils/package'; import { PrismaClient } from '@prisma/client'; -export class Seeder { - constructor(protected readonly prisma: PrismaClient) {} - - /** - * Check if any provinces data have changed. - */ - async hasProvinceChanges(): Promise { - const [newProvinces, oldProvinces] = await Promise.all([ - getProvinces(), - this.prisma.province.findMany(), - ]); - - return !areArraysEqual(newProvinces, oldProvinces); - } - - /** - * Check if any regencies data have changed. - */ - async hasRegencyChanges(): Promise { - const [newRegencies, oldRegencies] = await Promise.all([ - getRegencies(), - this.prisma.regency.findMany(), - ]); - - return !areArraysEqual(newRegencies, oldRegencies); - } - - /** - * Check if any districts data have changed. - */ - async hasDistrictChanges(): Promise { - const [newDistricts, oldDistricts] = await Promise.all([ - getDistricts(), - this.prisma.district.findMany(), - ]); +export type Area = 'province' | 'regency' | 'district' | 'village' | 'island'; - return !areArraysEqual(newDistricts, oldDistricts); - } - - /** - * Check if any islands data have changed. - */ - async hasIslandChanges(): Promise { - const [newIslands, oldIslands] = await Promise.all([ - getIslands(), - this.prisma.island.findMany(), - ]); - - return !areArraysEqual(newIslands, oldIslands); - } - - /** - * Check if any villages data have changed. - */ - async hasVillageChanges(): Promise { - const [newVillages, oldVillages] = await Promise.all([ - getVillages(), - this.prisma.village.findMany(), - ]); - - return !areArraysEqual(newVillages, oldVillages); +export class Seeder { + constructor(protected readonly prisma: PrismaClient) { + // Bind the methods to the class instance. + this.hasDataChanges = this.hasDataChanges.bind(this); + this.insertAreas = this.insertAreas.bind(this); + this.deleteAreas = this.deleteAreas.bind(this); + this.generateLog = this.generateLog.bind(this); } /** * Check if there are data changes. */ async hasDataChanges(): Promise { - if (await this.hasProvinceChanges()) { - return true; - } + const packageVersion = await getInstalledPackageVersion('idn-area-data'); - if (await this.hasRegencyChanges()) { - return true; + if (!packageVersion) { + throw new Error( + 'idn-area-data package is not installed. Make sure to run `pnpm install` first.', + ); } - if (await this.hasDistrictChanges()) { - return true; - } + const { dataVersion } = + (await this.prisma.seederLogs.findFirst({ + orderBy: { createdAt: 'desc' }, + })) ?? {}; - if (await this.hasIslandChanges()) { - return true; - } - - if (await this.hasVillageChanges()) { - return true; - } - - return false; - } - - async insertProvinces(): Promise { - const provinces = await getProvinces(); - const res = await this.prisma.province.createMany({ data: provinces }); - return res.count; - } - - async insertRegencies(): Promise { - const regencies = await getRegencies(); - const res = await this.prisma.regency.createMany({ data: regencies }); - return res.count; - } - - async insertDistricts(): Promise { - const districts = await getDistricts(); - const res = await this.prisma.district.createMany({ data: districts }); - return res.count; + // Compare the installed version of the package with the latest data version in the database. + return !dataVersion || dataVersion !== packageVersion; } - async insertVillages(): Promise { - const villages = await getVillages(); - const res = await this.prisma.village.createMany({ data: villages }); - return res.count; - } + // Methods to insert data to the database. + async insertAreas(area: Area): Promise { + const data = await { + province: getProvinces, + regency: getRegencies, + district: getDistricts, + village: getVillages, + island: getIslands, + }[area](); - async insertIslands(): Promise { - const islands = await getIslands(); - const res = await this.prisma.island.createMany({ data: islands }); + // @ts-ignore prisma[area] is a valid property since area is one of the valid values. + const res = await this.prisma[area].createMany({ data }); return res.count; } - async deleteProvinces(): Promise { - const res = await this.prisma.province.deleteMany(); + async deleteAreas(area: Area): Promise { + // @ts-ignore prisma[area] is a valid property since area is one of the valid values. + const res = await this.prisma[area].deleteMany(); return res.count; } - async deleteRegencies(): Promise { - const res = await this.prisma.regency.deleteMany(); - return res.count; - } - - async deleteDistricts(): Promise { - const res = await this.prisma.district.deleteMany(); - return res.count; - } - - async deleteVillages(): Promise { - const res = await this.prisma.village.deleteMany(); - return res.count; - } + /** + * Generate a log after the seeder is executed + */ + async generateLog(): Promise { + const packageVersion = await getInstalledPackageVersion('idn-area-data'); - async deleteIslands(): Promise { - const res = await this.prisma.island.deleteMany(); - return res.count; + await this.prisma.seederLogs.create({ + data: { dataVersion: packageVersion }, + }); } } diff --git a/prisma/sqlite/migrations/20241123043745_create_seeder_logs_table/migration.sql b/prisma/sqlite/migrations/20241123043745_create_seeder_logs_table/migration.sql new file mode 100644 index 0000000..725768e --- /dev/null +++ b/prisma/sqlite/migrations/20241123043745_create_seeder_logs_table/migration.sql @@ -0,0 +1,6 @@ +-- CreateTable +CREATE TABLE "seeder_logs" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "data_version" TEXT NOT NULL, + "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/prisma/sqlite/schema.prisma b/prisma/sqlite/schema.prisma index c77862d..0fd78ca 100644 --- a/prisma/sqlite/schema.prisma +++ b/prisma/sqlite/schema.prisma @@ -49,6 +49,14 @@ model Regency { @@map("regencies") } +model SeederLogs { + id Int @id @default(autoincrement()) + dataVersion String @map("data_version") + createdAt DateTime @default(now()) @map("created_at") + + @@map("seeder_logs") +} + model Village { code String @id districtCode String @map("district_code") diff --git a/src/common/utils/__tests__/package.spec.ts b/src/common/utils/__tests__/package.spec.ts new file mode 100644 index 0000000..bcf7092 --- /dev/null +++ b/src/common/utils/__tests__/package.spec.ts @@ -0,0 +1,68 @@ +import { describe, it, expect, vi } from 'vitest'; +import { getInstalledPackageVersion } from '../package'; +import { exec } from 'node:child_process'; + +vi.mock('node:child_process', () => ({ + exec: vi.fn(), +})); + +beforeEach(() => { + vi.resetModules(); +}); + +describe('getInstalledPackageVersion', () => { + it('should return the version of the installed package', async () => { + const mockStdout = JSON.stringify([ + { + dependencies: { 'some-package': { version: '1.0.0' } }, + devDependencies: {}, + }, + ]); + + vi.mocked(exec).mockImplementation((_cmd, callback) => { + // @ts-ignore (mocked) + return callback(null, mockStdout, ''); + }); + + const version = await getInstalledPackageVersion('some-package'); + expect(version).toBe('1.0.0'); + }); + + it('should return undefined if the package is not found', async () => { + const mockStdout = JSON.stringify([ + { + dependencies: {}, + devDependencies: {}, + }, + ]); + + vi.mocked(exec).mockImplementation((_cmd, callback) => { + // @ts-ignore (mocked) + return callback(null, mockStdout, ''); + }); + + const version = await getInstalledPackageVersion('non-existent-package'); + expect(version).toBeUndefined(); + }); + + it('should return undefined if there is an error in stderr', async () => { + vi.mocked(exec).mockImplementation((_cmd, callback) => { + // @ts-ignore + return callback(null, '', 'Some error'); + }); + + const version = await getInstalledPackageVersion('some-package'); + expect(version).toBeUndefined(); + }); + + it('should reject if exec returns an error', async () => { + vi.mocked(exec).mockImplementation((_cmd, callback) => { + // @ts-ignore + return callback(new Error('exec error'), '', ''); + }); + + await expect(getInstalledPackageVersion('some-package')).rejects.toThrow( + 'exec error', + ); + }); +}); diff --git a/src/common/utils/package.ts b/src/common/utils/package.ts new file mode 100644 index 0000000..bd8a796 --- /dev/null +++ b/src/common/utils/package.ts @@ -0,0 +1,37 @@ +import { exec } from 'node:child_process'; + +type Dependency = { + from: string; + version: string; + resolved: string; + path: string; +}; + +export async function getInstalledPackageVersion( + packageName: string, +): Promise { + const { stdout, stderr } = await new Promise<{ + stdout: string; + stderr: string; + }>((resolve, reject) => { + exec('pnpm list --json', (error, stdout, stderr) => { + if (error) { + reject(error); + } + resolve({ stdout, stderr }); + }); + }); + + if (stderr) { + return undefined; + } + + const [{ dependencies, devDependencies }] = JSON.parse(stdout) as [ + { + dependencies: Record; + devDependencies: Record; + }, + ]; + + return { ...dependencies, ...devDependencies }[packageName]?.version; +}