From 31df8380dce1fdf0d5d2e0d13233e4cd08d47848 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Sat, 25 Nov 2023 17:26:25 -0500 Subject: [PATCH 01/18] ci: run PR CI on next branch --- .github/workflows/pull-request.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 5ca52702..ca876455 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -1,8 +1,8 @@ -name: Pull request on main +name: Pull request on main or next on: pull_request: - branches: [ main ] + branches: [ main, next ] jobs: build: From 1859991e4996b6012101b5a65bd2b261ca42f217 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Tue, 21 Nov 2023 15:47:23 -0500 Subject: [PATCH 02/18] feat(cli): move options to commands --- packages/cli/src/index.ts | 152 ++++++++++-------- packages/cli/src/lib/config.ts | 5 +- .../collections/base/directus-collection.ts | 1 - .../collections/dashboards/collection.ts | 1 - .../services/collections/flows/collection.ts | 1 - .../collections/operations/collection.ts | 1 - .../services/collections/panels/collection.ts | 1 - .../collections/permissions/collection.ts | 1 - .../services/collections/roles/collection.ts | 1 - .../collections/settings/collection.ts | 2 - .../collections/webhooks/collection.ts | 2 - 11 files changed, 90 insertions(+), 78 deletions(-) diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 0a44cd4b..1e80a57c 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,5 +1,6 @@ +import 'dotenv/config'; import 'reflect-metadata'; -import { program } from 'commander'; +import { Option, program } from 'commander'; import { disposeContext, initContext, @@ -17,73 +18,96 @@ const defaultDumpPath = Path.join(process.cwd(), 'directus-config'); const defaultSnapshotPath = 'snapshot'; const defaultCollectionsPath = 'collections'; +// Global options +const debugOption = new Option('-d, --debug', 'display more logging').default( + false, +); +const directusUrlOption = new Option( + '-u, --directus-url ', + 'Directus URL', +).env('DIRECTUS_URL'); +const directusTokenOption = new Option( + '-t, --directus-token ', + 'Directus access token', +).env('DIRECTUS_TOKEN'); + +// Shared options +const noSplitOption = new Option( + '--no-split', + 'should the schema snapshot be split into multiple files', +).default(true); +const dumpPathOption = new Option( + '--dump-path ', + 'the base path for the dump, must be an absolute path', +).default(defaultDumpPath); +const collectionsPathOption = new Option( + '--collections-path ', + 'the path for the collections dump, relative to the dump path', +).default(defaultCollectionsPath); +const snapshotPathOption = new Option( + '--snapshot-path ', + 'the path for the schema snapshot dump, relative to the dump path', +).default(defaultSnapshotPath); +const forceOption = new Option( + '-f, --force', + 'force the diff of schema, even if the Directus version is different', +).default(false); + program - .option('-d, --debug', 'display more logging', false) - .option( - '-u, --directus-url ', - 'Directus URL. Can also be set via DIRECTUS_URL env var', - ) - .option( - '-t, --directus-token ', - 'Directus access token. Can also be set via DIRECTUS_TOKEN env var', - ) - .option( - '--no-split', - 'should the schema snapshot be split into multiple files', - true, - ) - .option( - '--dump-path ', - 'the base path for the dump, must be an absolute path', - defaultDumpPath, - ) - .option( - '--collections-path ', - 'the path for the collections dump, relative to the dump path', - defaultCollectionsPath, - ) - .option( - '--snapshot-path ', - 'the path for the schema snapshot dump, relative to the dump path', - defaultSnapshotPath, + .addOption(debugOption) + .addOption(directusUrlOption) + .addOption(directusTokenOption); + +program + .command('pull') + .description('get the schema and collections and store them locally') + .addOption(noSplitOption) + .addOption(dumpPathOption) + .addOption(collectionsPathOption) + .addOption(snapshotPathOption) + .action(wrapAction(runPull)); + +program + .command('diff') + .description( + 'describe the schema and collections diff. Does not modify the database.', ) - .option( - '-f, --force', - 'force the diff of schema, even if the Directus version is different', - false, - ); + .addOption(noSplitOption) + .addOption(dumpPathOption) + .addOption(collectionsPathOption) + .addOption(snapshotPathOption) + .addOption(forceOption) + .action(wrapAction(runDiff)); -registerCommand( - 'pull', - 'get the schema and collections and store them locally', - runPull, -); -registerCommand( - 'diff', - 'describe the schema and collections diff. Does not modify the database.', - runDiff, -); -registerCommand('push', 'push the schema and collections', runPush); -registerCommand('untrack', 'stop tracking of an element', runUntrack) - .option('-c, --collection ', 'the collection of the element') - .option('-i, --id ', 'the id of the element to untrack'); +program + .command('push') + .description('push the schema and collections') + .addOption(noSplitOption) + .addOption(dumpPathOption) + .addOption(collectionsPathOption) + .addOption(snapshotPathOption) + .addOption(forceOption) + .action(wrapAction(runPush)); + +program + .command('untrack') + .description('stop tracking of an element') + .requiredOption( + '-c, --collection ', + 'the collection of the element', + ) + .requiredOption('-i, --id ', 'the id of the element to untrack') + .action(wrapAction(runUntrack)); program.parse(process.argv); -function registerCommand( - name: string, - description: string, - action: (options?: Options) => Promise, -) { - return program - .command(name) - .description(description) - .action(() => { - const options = program.opts() as Options; - return initContext(options as ProgramOptions) - .then(() => action(options)) - .catch(logErrorAndStop) - .then(disposeContext) - .then(logEndAndClose); - }); +function wrapAction(action: (options?: Options) => Promise) { + return (commandOpts: Options) => { + const options: ProgramOptions = { ...program.opts(), ...commandOpts }; + return initContext(options) + .then(() => action(commandOpts)) + .catch(logErrorAndStop) + .then(disposeContext) + .then(logEndAndClose); + }; } diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts index 1e3d79f2..f73b284d 100644 --- a/packages/cli/src/lib/config.ts +++ b/packages/cli/src/lib/config.ts @@ -13,9 +13,8 @@ export interface ProgramOptions { } export function getConfig(options: ProgramOptions) { - const { dumpPath } = options; - const snapshotPath = Path.join(dumpPath, options.snapshotPath); - const collectionsPath = Path.join(dumpPath, options.collectionsPath); + const snapshotPath = Path.join(options.dumpPath, options.snapshotPath); + const collectionsPath = Path.join(options.dumpPath, options.collectionsPath); return { logger: { diff --git a/packages/cli/src/lib/services/collections/base/directus-collection.ts b/packages/cli/src/lib/services/collections/base/directus-collection.ts index d044badb..8560a6c4 100644 --- a/packages/cli/src/lib/services/collections/base/directus-collection.ts +++ b/packages/cli/src/lib/services/collections/base/directus-collection.ts @@ -1,4 +1,3 @@ -import 'dotenv/config'; import { IdMap, IdMapperClient } from './id-mapper-client'; import { DirectusBaseType, diff --git a/packages/cli/src/lib/services/collections/dashboards/collection.ts b/packages/cli/src/lib/services/collections/dashboards/collection.ts index d63f5867..32ec804c 100644 --- a/packages/cli/src/lib/services/collections/dashboards/collection.ts +++ b/packages/cli/src/lib/services/collections/dashboards/collection.ts @@ -1,4 +1,3 @@ -import 'dotenv/config'; import { DirectusCollection } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; diff --git a/packages/cli/src/lib/services/collections/flows/collection.ts b/packages/cli/src/lib/services/collections/flows/collection.ts index 5f5022c5..42b5d5f5 100644 --- a/packages/cli/src/lib/services/collections/flows/collection.ts +++ b/packages/cli/src/lib/services/collections/flows/collection.ts @@ -1,4 +1,3 @@ -import 'dotenv/config'; import { DirectusCollection, WithSyncIdAndWithoutId } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; diff --git a/packages/cli/src/lib/services/collections/operations/collection.ts b/packages/cli/src/lib/services/collections/operations/collection.ts index 9b7c8d00..b6966485 100644 --- a/packages/cli/src/lib/services/collections/operations/collection.ts +++ b/packages/cli/src/lib/services/collections/operations/collection.ts @@ -1,4 +1,3 @@ -import 'dotenv/config'; import { DirectusCollection } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; diff --git a/packages/cli/src/lib/services/collections/panels/collection.ts b/packages/cli/src/lib/services/collections/panels/collection.ts index c59f473f..73f6adb2 100644 --- a/packages/cli/src/lib/services/collections/panels/collection.ts +++ b/packages/cli/src/lib/services/collections/panels/collection.ts @@ -1,4 +1,3 @@ -import 'dotenv/config'; import { DirectusCollection } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; diff --git a/packages/cli/src/lib/services/collections/permissions/collection.ts b/packages/cli/src/lib/services/collections/permissions/collection.ts index 20ea95be..80bb53c0 100644 --- a/packages/cli/src/lib/services/collections/permissions/collection.ts +++ b/packages/cli/src/lib/services/collections/permissions/collection.ts @@ -1,4 +1,3 @@ -import 'dotenv/config'; import { DirectusCollection } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; diff --git a/packages/cli/src/lib/services/collections/roles/collection.ts b/packages/cli/src/lib/services/collections/roles/collection.ts index 436f2cb1..71a1d452 100644 --- a/packages/cli/src/lib/services/collections/roles/collection.ts +++ b/packages/cli/src/lib/services/collections/roles/collection.ts @@ -1,4 +1,3 @@ -import 'dotenv/config'; import { DirectusCollection } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; diff --git a/packages/cli/src/lib/services/collections/settings/collection.ts b/packages/cli/src/lib/services/collections/settings/collection.ts index 1bb09db7..bdd5cd4e 100644 --- a/packages/cli/src/lib/services/collections/settings/collection.ts +++ b/packages/cli/src/lib/services/collections/settings/collection.ts @@ -1,5 +1,3 @@ -import 'dotenv/config'; - import { DirectusCollection } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; diff --git a/packages/cli/src/lib/services/collections/webhooks/collection.ts b/packages/cli/src/lib/services/collections/webhooks/collection.ts index 13167ebd..4bcf2e36 100644 --- a/packages/cli/src/lib/services/collections/webhooks/collection.ts +++ b/packages/cli/src/lib/services/collections/webhooks/collection.ts @@ -1,5 +1,3 @@ -import 'dotenv/config'; - import { DirectusCollection } from '../base'; import pino from 'pino'; import { Inject, Service } from 'typedi'; From c7d4e98f594b08c891ede395fa19d8188759c7a4 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Tue, 21 Nov 2023 22:39:08 -0500 Subject: [PATCH 03/18] feat(cli): manage config with a service --- package-lock.json | 13 +- packages/cli/package.json | 4 +- packages/cli/src/index.ts | 18 +-- packages/cli/src/lib/commands/index.ts | 1 + packages/cli/src/lib/commands/untrack.ts | 16 +-- packages/cli/src/lib/config.ts | 41 ------- packages/cli/src/lib/constants.ts | 4 - packages/cli/src/lib/env.spec.ts | 24 ---- packages/cli/src/lib/helpers.ts | 28 ++--- packages/cli/src/lib/index.ts | 1 - packages/cli/src/lib/loader.ts | 35 +++--- .../collections/base/id-mapper-client.ts | 35 +++--- .../collections/dashboards/data-loader.ts | 9 +- .../dashboards/id-mapper-client.ts | 7 +- .../services/collections/flows/data-loader.ts | 12 +- .../collections/flows/id-mapper-client.ts | 7 +- .../collections/operations/data-loader.ts | 9 +- .../operations/id-mapper-client.ts | 7 +- .../collections/panels/data-loader.ts | 12 +- .../collections/panels/id-mapper-client.ts | 7 +- .../collections/permissions/data-loader.ts | 9 +- .../permissions/id-mapper-client.ts | 7 +- .../services/collections/roles/data-loader.ts | 12 +- .../collections/roles/id-mapper-client.ts | 7 +- .../collections/settings/data-loader.ts | 12 +- .../collections/settings/id-mapper-client.ts | 7 +- .../collections/webhooks/data-loader.ts | 12 +- .../collections/webhooks/id-mapper-client.ts | 7 +- .../cli/src/lib/services/config/config.ts | 116 ++++++++++++++++++ packages/cli/src/lib/services/config/index.ts | 3 + .../cli/src/lib/services/config/interfaces.ts | 22 ++++ .../cli/src/lib/services/config/schema.ts | 48 ++++++++ packages/cli/src/lib/services/index.ts | 1 + .../cli/src/lib/services/migration-client.ts | 13 +- .../lib/services/snapshot/snapshot-client.ts | 13 +- 35 files changed, 352 insertions(+), 227 deletions(-) delete mode 100644 packages/cli/src/lib/config.ts delete mode 100644 packages/cli/src/lib/env.spec.ts create mode 100644 packages/cli/src/lib/services/config/config.ts create mode 100644 packages/cli/src/lib/services/config/index.ts create mode 100644 packages/cli/src/lib/services/config/interfaces.ts create mode 100644 packages/cli/src/lib/services/config/schema.ts diff --git a/package-lock.json b/package-lock.json index 4db7bc9a..96862510 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16274,6 +16274,14 @@ "node": ">=14.17" } }, + "node_modules/typescript-cacheable": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/typescript-cacheable/-/typescript-cacheable-3.0.3.tgz", + "integrity": "sha512-xC6+mFrbxptvjFyaRIOMxWuGeHCihepjacRIeoZc9s3Uve4fmqS1t746zpmSyVOJ7JHwFzqtlGIiTjWLtziWhg==", + "engines": { + "node": ">=14.16.0" + } + }, "node_modules/uglify-js": { "version": "3.17.4", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz", @@ -16894,7 +16902,6 @@ "version": "3.22.4", "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", - "dev": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -16933,7 +16940,9 @@ "pino": "^8.16.1", "pino-pretty": "^10.2.3", "reflect-metadata": "^0.1.13", - "typedi": "^0.10.0" + "typedi": "^0.10.0", + "typescript-cacheable": "^3.0.3", + "zod": "^3.22.4" }, "bin": { "directus-sync": "bin/index.js" diff --git a/packages/cli/package.json b/packages/cli/package.json index ed9726a3..f329471e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -43,6 +43,8 @@ "pino": "^8.16.1", "pino-pretty": "^10.2.3", "reflect-metadata": "^0.1.13", - "typedi": "^0.10.0" + "typedi": "^0.10.0", + "typescript-cacheable": "^3.0.3", + "zod": "^3.22.4" } } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 1e80a57c..c7d4f320 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,12 +1,13 @@ import 'dotenv/config'; import 'reflect-metadata'; -import { Option, program } from 'commander'; +import { Command, Option, program } from 'commander'; import { + CommandName, + CommandsOptions, disposeContext, initContext, logEndAndClose, logErrorAndStop, - ProgramOptions, runDiff, runPull, runPush, @@ -101,11 +102,14 @@ program program.parse(process.argv); -function wrapAction(action: (options?: Options) => Promise) { - return (commandOpts: Options) => { - const options: ProgramOptions = { ...program.opts(), ...commandOpts }; - return initContext(options) - .then(() => action(commandOpts)) +function wrapAction(action: () => Promise) { + return (commandOpts: CommandsOptions[CommandName], command: Command) => { + return initContext( + program.opts(), + command.name() as CommandName, + commandOpts, + ) + .then(action) .catch(logErrorAndStop) .then(disposeContext) .then(logEndAndClose); diff --git a/packages/cli/src/lib/commands/index.ts b/packages/cli/src/lib/commands/index.ts index 672a1e84..0bf1aaa2 100644 --- a/packages/cli/src/lib/commands/index.ts +++ b/packages/cli/src/lib/commands/index.ts @@ -1,3 +1,4 @@ export * from './diff'; export * from './pull'; export * from './push'; +export * from './untrack'; diff --git a/packages/cli/src/lib/commands/untrack.ts b/packages/cli/src/lib/commands/untrack.ts index 36995700..f6de7b80 100644 --- a/packages/cli/src/lib/commands/untrack.ts +++ b/packages/cli/src/lib/commands/untrack.ts @@ -1,19 +1,13 @@ -import { getIdMapperClientByName } from '../services'; +import { ConfigService, getIdMapperClientByName } from '../services'; import { Container } from 'typedi'; import pino from 'pino'; import { LOGGER } from '../constants'; -interface RunUntrackOptions { - collection: string; - id: string; -} - -export async function runUntrack(options?: RunUntrackOptions) { +export async function runUntrack() { const logger: pino.Logger = Container.get(LOGGER); - if (!options) { - throw new Error('Missing options'); - } - const { collection, id } = options; + const config = Container.get(ConfigService); + + const { collection, id } = config.getUntrackConfig(); const idMapper = getIdMapperClientByName(collection); try { await idMapper.removeByLocalId(id); diff --git a/packages/cli/src/lib/config.ts b/packages/cli/src/lib/config.ts deleted file mode 100644 index f73b284d..00000000 --- a/packages/cli/src/lib/config.ts +++ /dev/null @@ -1,41 +0,0 @@ -import Path from 'path'; -import { env } from './helpers'; - -export interface ProgramOptions { - debug: boolean; - split: boolean; - dumpPath: string; - collectionsPath: string; - snapshotPath: string; - force: boolean; - directusUrl?: string; - directusToken?: string; -} - -export function getConfig(options: ProgramOptions) { - const snapshotPath = Path.join(options.dumpPath, options.snapshotPath); - const collectionsPath = Path.join(options.dumpPath, options.collectionsPath); - - return { - logger: { - level: options.debug ? 'debug' : 'info', - }, - collections: { - dumpPath: collectionsPath, - }, - snapshot: { - dumpPath: snapshotPath, - splitFiles: options.split, - force: options.force, - }, - directus: { - url: env('DIRECTUS_URL', options.directusUrl), - token: env('DIRECTUS_TOKEN', options.directusToken), - }, - }; -} - -export type Config = ReturnType; -export type CollectionsConfig = Config['collections']; -export type SnapshotConfig = Config['snapshot']; -export type DirectusConfig = Config['directus']; diff --git a/packages/cli/src/lib/constants.ts b/packages/cli/src/lib/constants.ts index 629902d0..862896bf 100644 --- a/packages/cli/src/lib/constants.ts +++ b/packages/cli/src/lib/constants.ts @@ -1,5 +1 @@ -export const COLLECTIONS_CONFIG = 'collectionsConfig'; -export const SNAPSHOT_CONFIG = 'snapshotConfig'; - -export const DIRECTUS_CONFIG = 'directusConfig'; export const LOGGER = 'logger'; diff --git a/packages/cli/src/lib/env.spec.ts b/packages/cli/src/lib/env.spec.ts deleted file mode 100644 index f7b5509a..00000000 --- a/packages/cli/src/lib/env.spec.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { env } from './helpers'; - -describe('env', () => { - let originalEnv: NodeJS.ProcessEnv; - beforeEach(() => { - originalEnv = process.env; - process.env = {}; - }); - afterEach(() => { - process.env = originalEnv; - }); - it('should return the environment variable', () => { - process.env.TEST = 'test'; - expect(env('TEST')).toEqual('test'); - }); - it('should return the default value', () => { - expect(env('TEST', 'test')).toEqual('test'); - }); - it('should throw an error if the environment variable is not defined', () => { - expect(() => env('TEST')).toThrow( - 'Environment variable TEST is not defined', - ); - }); -}); diff --git a/packages/cli/src/lib/helpers.ts b/packages/cli/src/lib/helpers.ts index ec0e7c3e..47e4e80c 100755 --- a/packages/cli/src/lib/helpers.ts +++ b/packages/cli/src/lib/helpers.ts @@ -7,20 +7,24 @@ import { } from 'fs-extra'; import pino from 'pino'; import { Container } from 'typedi'; -import { Config } from './config'; import path from 'path'; import { LOGGER } from './constants'; +import { ConfigService } from './services'; -export function createDumpFolders(config: Config) { +export function createDumpFolders() { const logger: pino.Logger = Container.get(LOGGER); + const config = Container.get(ConfigService); - if (!existsSync(config.collections.dumpPath)) { + const collectionsConfig = config.getCollectionsConfig(); + if (!existsSync(collectionsConfig.dumpPath)) { logger.info('Create dump folder for collections'); - mkdirpSync(config.collections.dumpPath); + mkdirpSync(collectionsConfig.dumpPath); } - if (!existsSync(config.snapshot.dumpPath)) { + + const snapshotConfig = config.getSnapshotConfig(); + if (!existsSync(snapshotConfig.dumpPath)) { logger.info('Create dump folder for snapshot'); - mkdirpSync(config.snapshot.dumpPath); + mkdirpSync(snapshotConfig.dumpPath); } } @@ -74,15 +78,3 @@ export function loadJsonFilesRecursively(dirPath: string): T[] { } return files; } - -/** - * Returns an environment variable or throws an error if it is not defined. - * Accepts a default value as a second argument. - */ -export function env(name: string, defaultValue?: string): string { - const value = process.env[name] ?? defaultValue; - if (value === undefined) { - throw new Error(`Environment variable ${name} is not defined`); - } - return value; -} diff --git a/packages/cli/src/lib/index.ts b/packages/cli/src/lib/index.ts index 5c14a5d1..74d4ebd8 100644 --- a/packages/cli/src/lib/index.ts +++ b/packages/cli/src/lib/index.ts @@ -1,5 +1,4 @@ export * from './constants'; -export * from './config'; export * from './helpers'; export * from './loader'; export * from './commands'; diff --git a/packages/cli/src/lib/loader.ts b/packages/cli/src/lib/loader.ts index a207d615..de122588 100644 --- a/packages/cli/src/lib/loader.ts +++ b/packages/cli/src/lib/loader.ts @@ -1,9 +1,13 @@ import { + CommandName, + CommandsOptions, + ConfigService, DashboardsCollection, FlowsCollection, OperationsCollection, PanelsCollection, PermissionsCollection, + ProgramOptions, RolesCollection, SettingsCollection, WebhooksCollection, @@ -11,16 +15,14 @@ import { import { createDumpFolders } from './helpers'; import { Container } from 'typedi'; import Logger from 'pino'; -import { getConfig, ProgramOptions } from './config'; -import { - COLLECTIONS_CONFIG, - DIRECTUS_CONFIG, - LOGGER, - SNAPSHOT_CONFIG, -} from './constants'; +import { LOGGER } from './constants'; // eslint-disable-next-line @typescript-eslint/require-await -export async function initContext(options: ProgramOptions) { +export async function initContext( + programOptions: ProgramOptions, + commandName: CommandName, + commandOptions: CommandsOptions[CommandName], +) { // Set temporary logger, in case of error when loading the config Container.set( LOGGER, @@ -34,8 +36,10 @@ export async function initContext(options: ProgramOptions) { level: 'error', }), ); - // Load the config - const config = getConfig(options); + // Get the config service + const config = Container.get(ConfigService); + // Set the options + config.setOptions(programOptions, commandName, commandOptions); // Define the logger Container.set( LOGGER, @@ -46,19 +50,14 @@ export async function initContext(options: ProgramOptions) { colorize: true, }, }, - level: config.logger.level, + level: config.getLoggerConfig().level, }), ); - // Define the configs - Container.set(COLLECTIONS_CONFIG, config.collections); - Container.set(SNAPSHOT_CONFIG, config.snapshot); - Container.set(DIRECTUS_CONFIG, config.directus); - - createDumpFolders(config); + createDumpFolders(); } -export async function disposeContext() { +export function disposeContext() { // Close some services if needed } diff --git a/packages/cli/src/lib/services/collections/base/id-mapper-client.ts b/packages/cli/src/lib/services/collections/base/id-mapper-client.ts index 31fb21f0..e81be6a1 100644 --- a/packages/cli/src/lib/services/collections/base/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/base/id-mapper-client.ts @@ -1,5 +1,5 @@ import createHttpError from 'http-errors'; -import { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; export interface IdMap { id: number; @@ -12,6 +12,10 @@ export interface IdMap { export abstract class IdMapperClient { protected readonly extensionUri = '/directus-extension-sync'; + protected readonly url: string; + + protected readonly token: string; + /** * Cache for id maps */ @@ -24,9 +28,13 @@ export abstract class IdMapperClient { }; constructor( - protected readonly config: DirectusConfig, + protected readonly config: ConfigService, protected readonly table: string, - ) {} + ) { + const { url, token } = config.getDirectusConfig(); + this.url = url; + this.token = token; + } async getBySyncId(syncId: string): Promise { // Try to get from cache @@ -118,19 +126,16 @@ export abstract class IdMapperClient { payload: unknown = undefined, options: RequestInit = {}, ): Promise { - const response = await fetch( - `${this.config.url}${this.extensionUri}${uri}`, - { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${this.config.token}`, - }, - method, - body: payload ? JSON.stringify(payload) : null, - ...options, + const response = await fetch(`${this.url}${this.extensionUri}${uri}`, { + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${this.token}`, }, - ); + method, + body: payload ? JSON.stringify(payload) : null, + ...options, + }); if (!response.ok) { let error; try { diff --git a/packages/cli/src/lib/services/collections/dashboards/data-loader.ts b/packages/cli/src/lib/services/collections/dashboards/data-loader.ts index 8db57abe..7a20f7fd 100644 --- a/packages/cli/src/lib/services/collections/dashboards/data-loader.ts +++ b/packages/cli/src/lib/services/collections/dashboards/data-loader.ts @@ -1,16 +1,15 @@ import { DataLoader } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { DASHBOARDS_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusDashboard } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class DashboardsDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { + constructor(config: ConfigService) { const filePath = path.join( - config.dumpPath, + config.getCollectionsConfig().dumpPath, `${DASHBOARDS_COLLECTION}.json`, ); super(filePath); diff --git a/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts b/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts index 7e896537..2ed17419 100644 --- a/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { DASHBOARDS_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class DashboardsIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, DASHBOARDS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/flows/data-loader.ts b/packages/cli/src/lib/services/collections/flows/data-loader.ts index bb38db3f..c6f2c7cc 100644 --- a/packages/cli/src/lib/services/collections/flows/data-loader.ts +++ b/packages/cli/src/lib/services/collections/flows/data-loader.ts @@ -1,15 +1,17 @@ import { DataLoader } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { FLOWS_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusFlow } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class FlowsDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { - const filePath = path.join(config.dumpPath, `${FLOWS_COLLECTION}.json`); + constructor(config: ConfigService) { + const filePath = path.join( + config.getCollectionsConfig().dumpPath, + `${FLOWS_COLLECTION}.json`, + ); super(filePath); } } diff --git a/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts b/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts index 7836b5aa..a38ccb72 100644 --- a/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { FLOWS_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class FlowsIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, FLOWS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/operations/data-loader.ts b/packages/cli/src/lib/services/collections/operations/data-loader.ts index cc249ba3..47cf1745 100644 --- a/packages/cli/src/lib/services/collections/operations/data-loader.ts +++ b/packages/cli/src/lib/services/collections/operations/data-loader.ts @@ -1,16 +1,15 @@ import { DataLoader, WithSyncIdAndWithoutId } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { OPERATIONS_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusOperation } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class OperationsDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { + constructor(config: ConfigService) { const filePath = path.join( - config.dumpPath, + config.getCollectionsConfig().dumpPath, `${OPERATIONS_COLLECTION}.json`, ); super(filePath); diff --git a/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts b/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts index 1dbab4c5..5c44f370 100644 --- a/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { OPERATIONS_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class OperationsIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, OPERATIONS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/panels/data-loader.ts b/packages/cli/src/lib/services/collections/panels/data-loader.ts index 755c8ba3..ab66d059 100644 --- a/packages/cli/src/lib/services/collections/panels/data-loader.ts +++ b/packages/cli/src/lib/services/collections/panels/data-loader.ts @@ -1,15 +1,17 @@ import { DataLoader } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { PANELS_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusPanel } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class PanelsDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { - const filePath = path.join(config.dumpPath, `${PANELS_COLLECTION}.json`); + constructor(config: ConfigService) { + const filePath = path.join( + config.getCollectionsConfig().dumpPath, + `${PANELS_COLLECTION}.json`, + ); super(filePath); } } diff --git a/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts b/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts index f6f88d49..9aefef94 100644 --- a/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { PANELS_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class PanelsIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, PANELS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/permissions/data-loader.ts b/packages/cli/src/lib/services/collections/permissions/data-loader.ts index 4a864988..9b25a38e 100644 --- a/packages/cli/src/lib/services/collections/permissions/data-loader.ts +++ b/packages/cli/src/lib/services/collections/permissions/data-loader.ts @@ -1,16 +1,15 @@ import { DataLoader, WithSyncIdAndWithoutId } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { PERMISSIONS_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusPermission } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class PermissionsDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { + constructor(config: ConfigService) { const filePath = path.join( - config.dumpPath, + config.getCollectionsConfig().dumpPath, `${PERMISSIONS_COLLECTION}.json`, ); super(filePath); diff --git a/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts b/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts index 3fa4446d..d1eed7ff 100644 --- a/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { PERMISSIONS_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class PermissionsIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, PERMISSIONS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/roles/data-loader.ts b/packages/cli/src/lib/services/collections/roles/data-loader.ts index 9eef4658..e58aa165 100644 --- a/packages/cli/src/lib/services/collections/roles/data-loader.ts +++ b/packages/cli/src/lib/services/collections/roles/data-loader.ts @@ -1,15 +1,17 @@ import { DataLoader } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { ROLES_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusRole } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class RolesDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { - const filePath = path.join(config.dumpPath, `${ROLES_COLLECTION}.json`); + constructor(config: ConfigService) { + const filePath = path.join( + config.getCollectionsConfig().dumpPath, + `${ROLES_COLLECTION}.json`, + ); super(filePath); } } diff --git a/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts b/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts index acbabee7..f34556e9 100644 --- a/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { ROLES_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class RolesIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, ROLES_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/settings/data-loader.ts b/packages/cli/src/lib/services/collections/settings/data-loader.ts index d962d906..1393918b 100644 --- a/packages/cli/src/lib/services/collections/settings/data-loader.ts +++ b/packages/cli/src/lib/services/collections/settings/data-loader.ts @@ -1,16 +1,18 @@ import { DataLoader } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { SETTINGS_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusSettings } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class SettingsDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { - const filePath = path.join(config.dumpPath, `${SETTINGS_COLLECTION}.json`); + constructor(config: ConfigService) { + const filePath = path.join( + config.getCollectionsConfig().dumpPath, + `${SETTINGS_COLLECTION}.json`, + ); super(filePath); } } diff --git a/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts b/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts index 7033b736..8f38862a 100644 --- a/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { SETTINGS_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class SettingsIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, SETTINGS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/webhooks/data-loader.ts b/packages/cli/src/lib/services/collections/webhooks/data-loader.ts index db9d8e49..afd9be0c 100644 --- a/packages/cli/src/lib/services/collections/webhooks/data-loader.ts +++ b/packages/cli/src/lib/services/collections/webhooks/data-loader.ts @@ -1,16 +1,18 @@ import { DataLoader } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { WEBHOOKS_COLLECTION } from './constants'; import path from 'path'; -import type { CollectionsConfig } from '../../../config'; -import { COLLECTIONS_CONFIG } from '../../../constants'; import { DirectusWebhook } from './interfaces'; +import { ConfigService } from '../../config'; @Service() export class WebhooksDataLoader extends DataLoader { - constructor(@Inject(COLLECTIONS_CONFIG) config: CollectionsConfig) { - const filePath = path.join(config.dumpPath, `${WEBHOOKS_COLLECTION}.json`); + constructor(config: ConfigService) { + const filePath = path.join( + config.getCollectionsConfig().dumpPath, + `${WEBHOOKS_COLLECTION}.json`, + ); super(filePath); } } diff --git a/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts b/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts index 8b7f5f91..197872fb 100644 --- a/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts @@ -1,12 +1,11 @@ import { IdMapperClient } from '../base'; -import { Inject, Service } from 'typedi'; +import { Service } from 'typedi'; import { WEBHOOKS_COLLECTION } from './constants'; -import { DIRECTUS_CONFIG } from '../../../constants'; -import type { DirectusConfig } from '../../../config'; +import { ConfigService } from '../../config'; @Service() export class WebhooksIdMapperClient extends IdMapperClient { - constructor(@Inject(DIRECTUS_CONFIG) config: DirectusConfig) { + constructor(config: ConfigService) { super(config, WEBHOOKS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/config/config.ts b/packages/cli/src/lib/services/config/config.ts new file mode 100644 index 00000000..8782a97c --- /dev/null +++ b/packages/cli/src/lib/services/config/config.ts @@ -0,0 +1,116 @@ +import { Service } from 'typedi'; +import { + CommandName, + CommandsOptions, + OptionsName, + OptionsTypes, + ProgramOptions, +} from './interfaces'; +import Path from 'path'; +import { CommandsOptionsSchemas, ProgramOptionsSchema } from './schema'; +import { Cacheable } from 'typescript-cacheable'; + +@Service() +export class ConfigService { + protected programOptions: ProgramOptions | undefined; + + protected commandOptions: CommandsOptions[CommandName] | undefined; + + protected commandName: CommandName | undefined; + + setOptions( + programOptions: ProgramOptions, + commandName: CommandName, + commandOptions: CommandsOptions[CommandName], + ) { + this.programOptions = programOptions; + this.commandName = commandName; + this.commandOptions = commandOptions; + } + + @Cacheable() + getLoggerConfig() { + return { + level: this.getOptions('debug') ? 'debug' : 'info', + }; + } + + @Cacheable() + getCollectionsConfig() { + const dumpPath = this.getOptions('dumpPath'); + const collectionsSubPath = this.getOptions('collectionsPath'); + const collectionsPath = Path.join(dumpPath, collectionsSubPath); + return { + dumpPath: collectionsPath, + }; + } + + @Cacheable() + getSnapshotConfig() { + const dumpPath = this.getOptions('dumpPath'); + const snapshotSubPath = this.getOptions('snapshotPath'); + const snapshotPath = Path.join(dumpPath, snapshotSubPath); + return { + dumpPath: snapshotPath, + splitFiles: this.getOptions('split'), + force: this.getOptions('force', false), + }; + } + + @Cacheable() + getDirectusConfig() { + return { + url: this.getOptions('directusUrl'), + token: this.getOptions('directusToken'), + }; + } + + @Cacheable() + getUntrackConfig() { + return { + collection: this.getOptions('collection'), + id: this.getOptions('id'), + }; + } + + protected getOptions( + name: T, + defaultValue?: OptionsTypes[T], + ): OptionsTypes[T] { + const commandOptions = this.getCommandOptions(); + if (commandOptions[name as keyof typeof commandOptions] !== undefined) { + return commandOptions[ + name as keyof typeof commandOptions + ] as OptionsTypes[T]; + } + const programOptions = this.getGlobalOptions(); + if (programOptions[name as keyof ProgramOptions] !== undefined) { + return programOptions[name as keyof ProgramOptions] as OptionsTypes[T]; + } + if (defaultValue === undefined) { + throw new Error(`missing option ${name}`); + } + return defaultValue; + } + + protected getGlobalOptions() { + if (!this.programOptions) { + throw new Error('program options not set'); + } + return ProgramOptionsSchema.parse(this.programOptions); + } + + protected getCommandOptions() { + if (!this.commandName) { + throw new Error('command name not set'); + } + if (!this.commandOptions) { + throw new Error('command options not set'); + } + const schema = CommandsOptionsSchemas[this.commandName]; + if (!schema) { + throw new Error(`missing schema for command ${this.commandName}`); + } + return schema.parse(this.commandOptions); + } +} diff --git a/packages/cli/src/lib/services/config/index.ts b/packages/cli/src/lib/services/config/index.ts new file mode 100644 index 00000000..b149a0aa --- /dev/null +++ b/packages/cli/src/lib/services/config/index.ts @@ -0,0 +1,3 @@ +export * from './config'; +export * from './interfaces'; +export * from './schema'; diff --git a/packages/cli/src/lib/services/config/interfaces.ts b/packages/cli/src/lib/services/config/interfaces.ts new file mode 100644 index 00000000..c24b74ce --- /dev/null +++ b/packages/cli/src/lib/services/config/interfaces.ts @@ -0,0 +1,22 @@ +import { z } from 'zod'; +import { + CommandsOptionsSchemas, + Options, + OptionsSchema, + ProgramOptionsSchema, +} from './schema'; + +export type ProgramOptions = z.infer; + +export interface CommandsOptions { + pull: z.infer; + diff: z.infer; + push: z.infer; + untrack: z.infer; +} + +export type CommandName = keyof CommandsOptions; + +export type OptionsName = keyof typeof Options; + +export type OptionsTypes = z.infer; diff --git a/packages/cli/src/lib/services/config/schema.ts b/packages/cli/src/lib/services/config/schema.ts new file mode 100644 index 00000000..96222bc5 --- /dev/null +++ b/packages/cli/src/lib/services/config/schema.ts @@ -0,0 +1,48 @@ +import { z } from 'zod'; + +export const Options = { + debug: z.boolean(), + directusUrl: z.string(), + directusToken: z.string(), + split: z.boolean(), + dumpPath: z.string(), + collectionsPath: z.string(), + snapshotPath: z.string(), + force: z.boolean(), + collection: z.string(), + id: z.string(), +}; +export const OptionsSchema = z.object(Options); + +export const ProgramOptionsSchema = z.object({ + debug: Options.debug, + directusUrl: Options.directusUrl, + directusToken: Options.directusToken, +}); + +export const CommandsOptionsSchemas = { + pull: z.object({ + split: Options.split, + dumpPath: Options.dumpPath, + collectionsPath: Options.collectionsPath, + snapshotPath: Options.snapshotPath, + }), + diff: z.object({ + split: Options.split, + dumpPath: Options.dumpPath, + collectionsPath: Options.collectionsPath, + snapshotPath: Options.snapshotPath, + force: Options.force, + }), + push: z.object({ + split: Options.split, + dumpPath: Options.dumpPath, + collectionsPath: Options.collectionsPath, + snapshotPath: Options.snapshotPath, + force: Options.force, + }), + untrack: z.object({ + collection: Options.collection, + id: Options.id, + }), +}; diff --git a/packages/cli/src/lib/services/index.ts b/packages/cli/src/lib/services/index.ts index 04986971..df99cc70 100644 --- a/packages/cli/src/lib/services/index.ts +++ b/packages/cli/src/lib/services/index.ts @@ -1,3 +1,4 @@ +export * from './config'; export * from './migration-client'; export * from './snapshot'; export * from './collections'; diff --git a/packages/cli/src/lib/services/migration-client.ts b/packages/cli/src/lib/services/migration-client.ts index 660efccd..d20a36fd 100644 --- a/packages/cli/src/lib/services/migration-client.ts +++ b/packages/cli/src/lib/services/migration-client.ts @@ -9,8 +9,8 @@ import { } from '@directus/sdk'; import { Inject, Service } from 'typedi'; import pino from 'pino'; -import { DIRECTUS_CONFIG, LOGGER } from '../constants'; -import type { DirectusConfig } from '../config'; +import { LOGGER } from '../constants'; +import { ConfigService } from './config'; @Service() export class MigrationClient { @@ -21,7 +21,7 @@ export class MigrationClient { AuthenticationClient; constructor( - @Inject(DIRECTUS_CONFIG) protected readonly config: DirectusConfig, + protected readonly config: ConfigService, @Inject(LOGGER) protected readonly logger: pino.Logger, ) { this.client = this.createClient(); @@ -48,10 +48,9 @@ export class MigrationClient { } protected createClient() { - const client = createDirectus(this.config.url) - .with(rest()) - .with(authentication()); - client.setToken(this.config.token); + const { url, token } = this.config.getDirectusConfig(); + const client = createDirectus(url).with(rest()).with(authentication()); + client.setToken(token); return client; } } diff --git a/packages/cli/src/lib/services/snapshot/snapshot-client.ts b/packages/cli/src/lib/services/snapshot/snapshot-client.ts index 75f7141a..b73b1268 100644 --- a/packages/cli/src/lib/services/snapshot/snapshot-client.ts +++ b/packages/cli/src/lib/services/snapshot/snapshot-client.ts @@ -2,12 +2,12 @@ import { Inject, Service } from 'typedi'; import { MigrationClient } from '../migration-client'; import { schemaApply, schemaDiff, schemaSnapshot } from '@directus/sdk'; import path from 'path'; -import type { SnapshotConfig } from '../../config'; import { Collection, Field, Relation, Snapshot } from './interfaces'; import { mkdirpSync, readJsonSync, removeSync, writeJsonSync } from 'fs-extra'; -import { LOGGER, SNAPSHOT_CONFIG } from '../../constants'; +import { LOGGER } from '../../constants'; import pino from 'pino'; import { getChildLogger, loadJsonFilesRecursively } from '../../helpers'; +import { ConfigService } from '../config'; interface SnapshotDiffDiff { collections: unknown[]; @@ -32,14 +32,15 @@ export class SnapshotClient { protected readonly logger: pino.Logger; constructor( - @Inject(SNAPSHOT_CONFIG) config: SnapshotConfig, + config: ConfigService, @Inject(LOGGER) baseLogger: pino.Logger, protected readonly migrationClient: MigrationClient, ) { this.logger = getChildLogger(baseLogger, 'snapshot'); - this.dumpPath = config.dumpPath; - this.splitFiles = config.splitFiles; - this.force = config.force; + const { dumpPath, splitFiles, force } = config.getSnapshotConfig(); + this.dumpPath = dumpPath; + this.splitFiles = splitFiles; + this.force = force; } /** From c4959b2100455eccf854911d1d1d4a5f544a9474 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Tue, 21 Nov 2023 22:42:42 -0500 Subject: [PATCH 04/18] chore: optimize imports --- packages/api/src/api/helpers.ts | 2 +- packages/api/src/api/interfaces.ts | 4 ++-- packages/cli/src/index.ts | 2 +- packages/cli/src/lib/loader.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/api/src/api/helpers.ts b/packages/api/src/api/helpers.ts index c8025ec2..ced19b3e 100644 --- a/packages/api/src/api/helpers.ts +++ b/packages/api/src/api/helpers.ts @@ -1,7 +1,7 @@ import { z, ZodError, ZodSchema } from 'zod'; import createError, { isHttpError } from 'http-errors'; import pino from 'pino'; -import { Response, Request, NextFunction } from './interfaces'; +import { NextFunction, Request, Response } from './interfaces'; /** * Helpers to ensure the user is an admin diff --git a/packages/api/src/api/interfaces.ts b/packages/api/src/api/interfaces.ts index 871143dd..018eb7d0 100644 --- a/packages/api/src/api/interfaces.ts +++ b/packages/api/src/api/interfaces.ts @@ -1,7 +1,7 @@ import { - Response, - Request as BaseRequest, NextFunction, + Request as BaseRequest, + Response, } from 'express-serve-static-core'; export interface Request extends BaseRequest { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index c7d4f320..39f2bc74 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -11,9 +11,9 @@ import { runDiff, runPull, runPush, + runUntrack, } from './lib'; import Path from 'path'; -import { runUntrack } from './lib/commands/untrack'; const defaultDumpPath = Path.join(process.cwd(), 'directus-config'); const defaultSnapshotPath = 'snapshot'; diff --git a/packages/cli/src/lib/loader.ts b/packages/cli/src/lib/loader.ts index de122588..caadb979 100644 --- a/packages/cli/src/lib/loader.ts +++ b/packages/cli/src/lib/loader.ts @@ -15,7 +15,7 @@ import { import { createDumpFolders } from './helpers'; import { Container } from 'typedi'; import Logger from 'pino'; -import { LOGGER } from './constants'; +import { LOGGER } from './constants'; // eslint-disable-next-line @typescript-eslint/require-await // eslint-disable-next-line @typescript-eslint/require-await export async function initContext( From 0ac358a8dbfc1cbc4a6b852d1f1e6998626da526 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Sat, 25 Nov 2023 11:46:54 -0500 Subject: [PATCH 05/18] feat: add config loader from file --- packages/cli/.gitignore | 2 + packages/cli/.npmignore | 2 + packages/cli/src/index.ts | 13 ++-- .../lib/services/config/config-file-loader.ts | 62 +++++++++++++++++++ .../cli/src/lib/services/config/config.ts | 31 ++++++++-- packages/cli/src/lib/services/config/index.ts | 1 + .../cli/src/lib/services/config/interfaces.ts | 3 + .../cli/src/lib/services/config/schema.ts | 16 +++++ 8 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 packages/cli/src/lib/services/config/config-file-loader.ts diff --git a/packages/cli/.gitignore b/packages/cli/.gitignore index 630d881f..45b4ebf7 100644 --- a/packages/cli/.gitignore +++ b/packages/cli/.gitignore @@ -1,2 +1,4 @@ directus-config README.md +directus-sync.config.js +directus-sync.config.*.js diff --git a/packages/cli/.npmignore b/packages/cli/.npmignore index 77182054..b18dde73 100644 --- a/packages/cli/.npmignore +++ b/packages/cli/.npmignore @@ -7,3 +7,5 @@ src example.env jest.config.js tsconfig.json +directus-sync.config.js +directus-sync.config.*.js diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 39f2bc74..bc77fdf5 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -13,9 +13,9 @@ import { runPush, runUntrack, } from './lib'; -import Path from 'path'; -const defaultDumpPath = Path.join(process.cwd(), 'directus-config'); +const defaultDumpPath = './directus-config'; +const defaultConfigPath = './directus-sync.config.js'; const defaultSnapshotPath = 'snapshot'; const defaultCollectionsPath = 'collections'; @@ -31,6 +31,10 @@ const directusTokenOption = new Option( '-t, --directus-token ', 'Directus access token', ).env('DIRECTUS_TOKEN'); +const configPathOption = new Option( + '-c, --config-path ', + 'the path to the config file. Required for extended options', +).default(defaultConfigPath); // Shared options const noSplitOption = new Option( @@ -39,7 +43,7 @@ const noSplitOption = new Option( ).default(true); const dumpPathOption = new Option( '--dump-path ', - 'the base path for the dump, must be an absolute path', + 'the base path for the dump', ).default(defaultDumpPath); const collectionsPathOption = new Option( '--collections-path ', @@ -57,7 +61,8 @@ const forceOption = new Option( program .addOption(debugOption) .addOption(directusUrlOption) - .addOption(directusTokenOption); + .addOption(directusTokenOption) + .addOption(configPathOption); program .command('pull') diff --git a/packages/cli/src/lib/services/config/config-file-loader.ts b/packages/cli/src/lib/services/config/config-file-loader.ts new file mode 100644 index 00000000..dcb4a4a0 --- /dev/null +++ b/packages/cli/src/lib/services/config/config-file-loader.ts @@ -0,0 +1,62 @@ +import { ConfigFileOptionsSchema } from './schema'; +import { ConfigFileOptions } from './interfaces'; +import { existsSync } from 'fs-extra'; +import deepmerge from 'deepmerge'; +import Path from 'path'; + +export class ConfigFileLoader { + protected loadedConfig: ConfigFileOptions | undefined; + + protected loadedConfigPaths: string[] = []; + + constructor(protected readonly configPath: string) { + this.loadedConfig = this.loadFromPath(this.configPath); + } + + get(): ConfigFileOptions | undefined { + return this.loadedConfig; + } + + protected loadFromPath(configPath: string): ConfigFileOptions | undefined { + // Check if the config is already loaded + // If the config is already loaded, return undefined to avoid infinite loop + if (this.loadedConfigPaths.includes(configPath)) { + return undefined; + } + // Check if the config exists + if (!existsSync(configPath)) { + return undefined; + } + // eslint-disable-next-line @typescript-eslint/no-var-requires + const rawConfig = require(configPath); + // Validate and return the config + const config = ConfigFileOptionsSchema.parse(rawConfig); + // Add the config to the loaded config + this.loadedConfigPaths.push(configPath); + // Merge with parents + return this.mergeWithParents(config, configPath); + } + + protected mergeWithParents( + config: ConfigFileOptions, + currentPath: string, + ): ConfigFileOptions { + const extendsPath = config.extends; + if (!extendsPath) { + return config; + } + let mergedConfig = config; + for (const parentPath of extendsPath) { + const parentFullPath = Path.resolve( + Path.dirname(currentPath), + parentPath, + ); + const parentConfig = this.loadFromPath(parentFullPath); + if (!parentConfig) { + continue; + } + mergedConfig = deepmerge(parentConfig, mergedConfig); + } + return mergedConfig; + } +} diff --git a/packages/cli/src/lib/services/config/config.ts b/packages/cli/src/lib/services/config/config.ts index 8782a97c..0eda8152 100644 --- a/packages/cli/src/lib/services/config/config.ts +++ b/packages/cli/src/lib/services/config/config.ts @@ -2,6 +2,7 @@ import { Service } from 'typedi'; import { CommandName, CommandsOptions, + ConfigFileOptions, OptionsName, OptionsTypes, ProgramOptions, @@ -9,6 +10,7 @@ import { import Path from 'path'; import { CommandsOptionsSchemas, ProgramOptionsSchema } from './schema'; import { Cacheable } from 'typescript-cacheable'; +import { ConfigFileLoader } from './config-file-loader'; @Service() export class ConfigService { @@ -37,9 +39,9 @@ export class ConfigService { @Cacheable() getCollectionsConfig() { - const dumpPath = this.getOptions('dumpPath'); + const dumpPath = Path.resolve(this.getOptions('dumpPath')); const collectionsSubPath = this.getOptions('collectionsPath'); - const collectionsPath = Path.join(dumpPath, collectionsSubPath); + const collectionsPath = Path.resolve(dumpPath, collectionsSubPath); return { dumpPath: collectionsPath, }; @@ -47,9 +49,9 @@ export class ConfigService { @Cacheable() getSnapshotConfig() { - const dumpPath = this.getOptions('dumpPath'); + const dumpPath = Path.resolve(this.getOptions('dumpPath')); const snapshotSubPath = this.getOptions('snapshotPath'); - const snapshotPath = Path.join(dumpPath, snapshotSubPath); + const snapshotPath = Path.resolve(dumpPath, snapshotSubPath); return { dumpPath: snapshotPath, splitFiles: this.getOptions('split'), @@ -73,10 +75,19 @@ export class ConfigService { }; } + @Cacheable() + getConfigFileLoaderConfig() { + return this.getOptions('configPath'); + } + protected getOptions( name: T, defaultValue?: OptionsTypes[T], ): OptionsTypes[T] { + const fileOptions = this.getFileOptions(); + if (fileOptions && fileOptions[name as keyof ConfigFileOptions]) { + return fileOptions[name as keyof ConfigFileOptions] as OptionsTypes[T]; + } const commandOptions = this.getCommandOptions(); if (commandOptions[name as keyof typeof commandOptions] !== undefined) { return commandOptions[ @@ -93,6 +104,7 @@ export class ConfigService { return defaultValue; } + @Cacheable() protected getGlobalOptions() { if (!this.programOptions) { throw new Error('program options not set'); @@ -100,6 +112,7 @@ export class ConfigService { return ProgramOptionsSchema.parse(this.programOptions); } + @Cacheable() protected getCommandOptions() { if (!this.commandName) { throw new Error('command name not set'); @@ -113,4 +126,14 @@ export class ConfigService { } return schema.parse(this.commandOptions); } + + @Cacheable() + protected getFileOptions(): ConfigFileOptions | undefined { + const globalOptions = this.getGlobalOptions(); + if (!globalOptions.configPath) { + throw new Error('missing config file path'); + } + const configFilePath = Path.resolve(globalOptions.configPath); + return new ConfigFileLoader(configFilePath).get(); + } } diff --git a/packages/cli/src/lib/services/config/index.ts b/packages/cli/src/lib/services/config/index.ts index b149a0aa..df244df9 100644 --- a/packages/cli/src/lib/services/config/index.ts +++ b/packages/cli/src/lib/services/config/index.ts @@ -1,3 +1,4 @@ export * from './config'; +export * from './config-file-loader'; export * from './interfaces'; export * from './schema'; diff --git a/packages/cli/src/lib/services/config/interfaces.ts b/packages/cli/src/lib/services/config/interfaces.ts index c24b74ce..ce96b548 100644 --- a/packages/cli/src/lib/services/config/interfaces.ts +++ b/packages/cli/src/lib/services/config/interfaces.ts @@ -1,6 +1,7 @@ import { z } from 'zod'; import { CommandsOptionsSchemas, + ConfigFileOptionsSchema, Options, OptionsSchema, ProgramOptionsSchema, @@ -20,3 +21,5 @@ export type CommandName = keyof CommandsOptions; export type OptionsName = keyof typeof Options; export type OptionsTypes = z.infer; + +export type ConfigFileOptions = z.infer; diff --git a/packages/cli/src/lib/services/config/schema.ts b/packages/cli/src/lib/services/config/schema.ts index 96222bc5..a8cf6856 100644 --- a/packages/cli/src/lib/services/config/schema.ts +++ b/packages/cli/src/lib/services/config/schema.ts @@ -11,6 +11,7 @@ export const Options = { force: z.boolean(), collection: z.string(), id: z.string(), + configPath: z.string(), }; export const OptionsSchema = z.object(Options); @@ -18,6 +19,7 @@ export const ProgramOptionsSchema = z.object({ debug: Options.debug, directusUrl: Options.directusUrl, directusToken: Options.directusToken, + configPath: Options.configPath, }); export const CommandsOptionsSchemas = { @@ -46,3 +48,17 @@ export const CommandsOptionsSchemas = { id: Options.id, }), }; + +export const ConfigFileOptionsSchema = z.object({ + // Inheritance + extends: z.array(z.string()).optional(), + // Global options + debug: Options.debug.optional(), + directusUrl: Options.directusUrl.optional(), + directusToken: Options.directusToken.optional(), + // Dump config + split: Options.split.optional(), + dumpPath: Options.dumpPath.optional(), + collectionsPath: Options.collectionsPath.optional(), + snapshotPath: Options.snapshotPath.optional(), +}); From a2b29804ef235503170f4bec27ef74eedd61cef1 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Sat, 25 Nov 2023 15:02:18 -0500 Subject: [PATCH 06/18] test: add tests for ConfigFileLoader --- packages/cli/.gitignore | 4 +- packages/cli/.npmignore | 9 +-- .../basic/directus-sync.config.base.base.js | 6 ++ .../basic/directus-sync.config.base.js | 6 ++ .../basic/directus-sync.config.js | 5 ++ .../directus-sync.config.base.js | 5 ++ .../dependency-loop/directus-sync.config.js | 5 ++ .../directus-sync.config.js | 5 ++ .../with-extra/directus-sync.config.js | 7 ++ .../config/config-file-loader.spec.ts | 69 +++++++++++++++++++ packages/cli/tsconfig.base.json | 2 +- 11 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 packages/cli/test/files/config-loader/basic/directus-sync.config.base.base.js create mode 100644 packages/cli/test/files/config-loader/basic/directus-sync.config.base.js create mode 100644 packages/cli/test/files/config-loader/basic/directus-sync.config.js create mode 100644 packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.base.js create mode 100644 packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.js create mode 100644 packages/cli/test/files/config-loader/missing-dependency/directus-sync.config.js create mode 100644 packages/cli/test/files/config-loader/with-extra/directus-sync.config.js create mode 100644 packages/cli/test/services/config/config-file-loader.spec.ts diff --git a/packages/cli/.gitignore b/packages/cli/.gitignore index 45b4ebf7..3581b486 100644 --- a/packages/cli/.gitignore +++ b/packages/cli/.gitignore @@ -1,4 +1,2 @@ -directus-config +directus-config/ README.md -directus-sync.config.js -directus-sync.config.*.js diff --git a/packages/cli/.npmignore b/packages/cli/.npmignore index b18dde73..583a99dd 100644 --- a/packages/cli/.npmignore +++ b/packages/cli/.npmignore @@ -1,11 +1,12 @@ -node_modules -directus-config -src +node_modules/ +directus-config/ +src/ +test/ .DS_Store .env .gitignore example.env jest.config.js tsconfig.json -directus-sync.config.js +test/files/config-loader/basic/directus-sync.config.js directus-sync.config.*.js diff --git a/packages/cli/test/files/config-loader/basic/directus-sync.config.base.base.js b/packages/cli/test/files/config-loader/basic/directus-sync.config.base.base.js new file mode 100644 index 00000000..e0fd7aa0 --- /dev/null +++ b/packages/cli/test/files/config-loader/basic/directus-sync.config.base.base.js @@ -0,0 +1,6 @@ +module.exports = { + debug: false, + split: false, + dumpPath: './dump', + directusUrl: 'http://localhost:8055', +}; diff --git a/packages/cli/test/files/config-loader/basic/directus-sync.config.base.js b/packages/cli/test/files/config-loader/basic/directus-sync.config.base.js new file mode 100644 index 00000000..413a2b88 --- /dev/null +++ b/packages/cli/test/files/config-loader/basic/directus-sync.config.base.js @@ -0,0 +1,6 @@ +module.exports = { + extends: ['./directus-sync.config.base.base.js'], + debug: false, + split: false, + directusToken: 'token', +}; diff --git a/packages/cli/test/files/config-loader/basic/directus-sync.config.js b/packages/cli/test/files/config-loader/basic/directus-sync.config.js new file mode 100644 index 00000000..ba3222f0 --- /dev/null +++ b/packages/cli/test/files/config-loader/basic/directus-sync.config.js @@ -0,0 +1,5 @@ +module.exports = { + extends: ['./directus-sync.config.base.js'], + debug: true, + split: true, +}; diff --git a/packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.base.js b/packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.base.js new file mode 100644 index 00000000..4d944894 --- /dev/null +++ b/packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.base.js @@ -0,0 +1,5 @@ +module.exports = { + extends: ['./directus-sync.config.js'], + debug: false, + split: false, +}; diff --git a/packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.js b/packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.js new file mode 100644 index 00000000..ba3222f0 --- /dev/null +++ b/packages/cli/test/files/config-loader/dependency-loop/directus-sync.config.js @@ -0,0 +1,5 @@ +module.exports = { + extends: ['./directus-sync.config.base.js'], + debug: true, + split: true, +}; diff --git a/packages/cli/test/files/config-loader/missing-dependency/directus-sync.config.js b/packages/cli/test/files/config-loader/missing-dependency/directus-sync.config.js new file mode 100644 index 00000000..9af8e3a1 --- /dev/null +++ b/packages/cli/test/files/config-loader/missing-dependency/directus-sync.config.js @@ -0,0 +1,5 @@ +module.exports = { + extends: ['./directus-sync.config.wrong.js'], + debug: true, + split: true, +}; diff --git a/packages/cli/test/files/config-loader/with-extra/directus-sync.config.js b/packages/cli/test/files/config-loader/with-extra/directus-sync.config.js new file mode 100644 index 00000000..e664a0d5 --- /dev/null +++ b/packages/cli/test/files/config-loader/with-extra/directus-sync.config.js @@ -0,0 +1,7 @@ +module.exports = { + debug: true, + split: true, + extra: { + foo: 'bar', + }, +}; diff --git a/packages/cli/test/services/config/config-file-loader.spec.ts b/packages/cli/test/services/config/config-file-loader.spec.ts new file mode 100644 index 00000000..f5fe3b4d --- /dev/null +++ b/packages/cli/test/services/config/config-file-loader.spec.ts @@ -0,0 +1,69 @@ +import { ConfigFileLoader } from '../../../src/lib'; +import Path from 'path'; + +describe('ConfigFileLoader', () => { + it('should be created with a wrong file path', () => { + const configFileLoader = new ConfigFileLoader('/wrong-path'); + expect(configFileLoader.get()).toBeUndefined(); + }); + + it('should be created with a correct file path', () => { + const configFileLoader = new ConfigFileLoader( + Path.resolve('test/files/config-loader/basic/directus-sync.config.js'), + ); + expect(configFileLoader.get()).toBeDefined(); + }); + it('should merge config recursively', () => { + const configFileLoader = new ConfigFileLoader( + Path.resolve('test/files/config-loader/basic/directus-sync.config.js'), + ); + expect(configFileLoader.get()).toEqual({ + dumpPath: './dump', + extends: [ + './directus-sync.config.base.base.js', + './directus-sync.config.base.js', + ], + directusUrl: 'http://localhost:8055', + directusToken: 'token', + debug: true, + split: true, + }); + }); + it('should exclude extra properties', () => { + const configFileLoader = new ConfigFileLoader( + Path.resolve( + 'test/files/config-loader/with-extra/directus-sync.config.js', + ), + ); + expect(configFileLoader.get()).toEqual({ + debug: true, + split: true, + }); + }); + + it('should deals with dependency loops', () => { + const configFileLoader = new ConfigFileLoader( + Path.resolve( + 'test/files/config-loader/dependency-loop/directus-sync.config.js', + ), + ); + expect(configFileLoader.get()).toEqual({ + extends: ['./directus-sync.config.js', './directus-sync.config.base.js'], + debug: true, + split: true, + }); + }); + + it('should deals with missing dependencies', () => { + const configFileLoader = new ConfigFileLoader( + Path.resolve( + 'test/files/config-loader/missing-dependency/directus-sync.config.js', + ), + ); + expect(configFileLoader.get()).toEqual({ + extends: ['./directus-sync.config.wrong.js'], + debug: true, + split: true, + }); + }); +}); diff --git a/packages/cli/tsconfig.base.json b/packages/cli/tsconfig.base.json index 208e2fe2..80ca4afa 100644 --- a/packages/cli/tsconfig.base.json +++ b/packages/cli/tsconfig.base.json @@ -29,5 +29,5 @@ "rootDir": "./src", "outDir": "./dist" }, - "include": ["./src/**/*.ts"] + "include": ["./src/**/*.ts", "./test/**/*.ts"] } From f97b74f77ac820d2de74e384c15697147d50d5b5 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Sat, 25 Nov 2023 15:18:54 -0500 Subject: [PATCH 07/18] feat: format zod errors --- packages/cli/src/lib/helpers.ts | 23 ++++++++++ .../lib/services/config/config-file-loader.ts | 3 +- .../cli/src/lib/services/config/config.ts | 9 +++- packages/cli/test/helpers.spec.ts | 42 +++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 packages/cli/test/helpers.spec.ts diff --git a/packages/cli/src/lib/helpers.ts b/packages/cli/src/lib/helpers.ts index 47e4e80c..f2c3bb31 100755 --- a/packages/cli/src/lib/helpers.ts +++ b/packages/cli/src/lib/helpers.ts @@ -5,6 +5,7 @@ import { readJsonSync, statSync, } from 'fs-extra'; +import { z, ZodError, ZodSchema } from 'zod'; import pino from 'pino'; import { Container } from 'typedi'; import path from 'path'; @@ -78,3 +79,25 @@ export function loadJsonFilesRecursively(dirPath: string): T[] { } return files; } + +/** + * Validate an object against a zod schema and format the error if it fails + */ +export function zodParse( + payload: unknown, + schema: T, + errorContext?: string, +): z.infer { + try { + return schema.parse(payload); + } catch (error) { + const message = + error instanceof ZodError + ? error.issues + .map((e) => `[${e.path.join(',')}] ${e.message}`) + .join('. ') + : (error as string); + const fullMessage = errorContext ? `${errorContext}: ${message}` : message; + throw new Error(fullMessage); + } +} diff --git a/packages/cli/src/lib/services/config/config-file-loader.ts b/packages/cli/src/lib/services/config/config-file-loader.ts index dcb4a4a0..e63b3c0b 100644 --- a/packages/cli/src/lib/services/config/config-file-loader.ts +++ b/packages/cli/src/lib/services/config/config-file-loader.ts @@ -3,6 +3,7 @@ import { ConfigFileOptions } from './interfaces'; import { existsSync } from 'fs-extra'; import deepmerge from 'deepmerge'; import Path from 'path'; +import { zodParse } from '../../helpers'; export class ConfigFileLoader { protected loadedConfig: ConfigFileOptions | undefined; @@ -30,7 +31,7 @@ export class ConfigFileLoader { // eslint-disable-next-line @typescript-eslint/no-var-requires const rawConfig = require(configPath); // Validate and return the config - const config = ConfigFileOptionsSchema.parse(rawConfig); + const config = zodParse(rawConfig, ConfigFileOptionsSchema, 'Config file'); // Add the config to the loaded config this.loadedConfigPaths.push(configPath); // Merge with parents diff --git a/packages/cli/src/lib/services/config/config.ts b/packages/cli/src/lib/services/config/config.ts index 0eda8152..9d4b1d55 100644 --- a/packages/cli/src/lib/services/config/config.ts +++ b/packages/cli/src/lib/services/config/config.ts @@ -11,6 +11,7 @@ import Path from 'path'; import { CommandsOptionsSchemas, ProgramOptionsSchema } from './schema'; import { Cacheable } from 'typescript-cacheable'; import { ConfigFileLoader } from './config-file-loader'; +import { zodParse } from '../../helpers'; @Service() export class ConfigService { @@ -109,7 +110,11 @@ export class ConfigService { if (!this.programOptions) { throw new Error('program options not set'); } - return ProgramOptionsSchema.parse(this.programOptions); + return zodParse( + this.programOptions, + ProgramOptionsSchema, + 'Global options', + ); } @Cacheable() @@ -124,7 +129,7 @@ export class ConfigService { if (!schema) { throw new Error(`missing schema for command ${this.commandName}`); } - return schema.parse(this.commandOptions); + return zodParse(this.commandOptions, schema, 'Command options'); } @Cacheable() diff --git a/packages/cli/test/helpers.spec.ts b/packages/cli/test/helpers.spec.ts new file mode 100644 index 00000000..15106dfb --- /dev/null +++ b/packages/cli/test/helpers.spec.ts @@ -0,0 +1,42 @@ +import { z } from 'zod'; +import { zodParse } from '../src/lib'; + +describe('zodParse', () => { + it('should throw error if payload is invalid', () => { + const payload = { + name: 'John Doe', + email: 'john@doe.com', + validated: false, + address: '123, Main Street, New York, NY', + }; + + const schema = z.object({ + name: z.string().min(3).max(50), + email: z.string().email(), + phone: z.string().min(10).max(10), + address: z.string().min(10).max(100), + }); + + expect(() => zodParse(payload, schema, 'User details')).toThrowError( + 'User details: [phone] Required', + ); + // With no error context + expect(() => zodParse(payload, schema)).toThrowError('[phone] Required'); + }); + + it('should return payload if payload is valid', () => { + const payload = { + name: 'John Doe', + email: 'john@doe.com', + address: '123, Main Street, New York, NY', + }; + + const schema = z.object({ + name: z.string().min(3).max(50), + email: z.string().email(), + address: z.string().min(10).max(100), + }); + + expect(zodParse(payload, schema)).toEqual(payload); + }); +}); From fd6229dbf953c67ccb22b15a3b27be4939254886 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Sat, 25 Nov 2023 17:24:36 -0500 Subject: [PATCH 08/18] feat: merge options from default to CLI options --- README.md | 40 +++++++- packages/cli/src/index.ts | 63 +++++++----- packages/cli/src/lib/loader.ts | 10 +- .../cli/src/lib/services/config/config.ts | 99 +++++++------------ .../src/lib/services/config/default-config.ts | 14 +++ packages/cli/src/lib/services/config/index.ts | 5 +- .../cli/src/lib/services/config/interfaces.ts | 19 +--- .../cli/src/lib/services/config/schema.ts | 62 +++--------- 8 files changed, 151 insertions(+), 161 deletions(-) create mode 100644 packages/cli/src/lib/services/config/default-config.ts diff --git a/README.md b/README.md index a952c7eb..f78d3d68 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,22 @@ npx directus-sync untrack --collection --id Removes tracking from an element within Directus. You must specify the collection and the ID of the element you wish to stop tracking. -## Global Options +## Available options + +Options are merged from the following sources, in order of precedence: + +1. CLI arguments +2. Environment variables +3. Configuration file +4. Default values + +### CLI and environment variables These options can be used with any command to configure the operation of `directus-sync`: +- `-c, --config-path ` + Change the path to the config file. The default is `"./directus-sync.config.js"`. + - `-d, --debug` Display additional logging information. Useful for debugging or verifying what `directus-sync` is doing under the hood. @@ -94,6 +106,32 @@ These options can be used with any command to configure the operation of `direct - `-h, --help` Display help information for the `directus-sync` commands. +### Configuration file + +The `directus-sync` CLI also supports a configuration file. This file is optional. If it is not provided, the CLI will +use the default values for the options. + +The default path for the configuration file is `./directus-sync.config.js`. You can change this path using the +`--config-path` option. + +The configuration file can extend another configuration file using the `extends` property. + +This is an example of a configuration file: + +```javascript +// ./directus-sync.config.js +module.exports = { + extends: ['./directus-sync.config.base.js'], + debug: true, + directusUrl: 'https://directus.example.com', + directusToken: 'my-directus-token', + split: true, + dumpPath: './directus-config', + collectionsPath: 'collections', + snapshotPath: 'snapshot', +}; +``` + ### Tracked Elements `directus-sync` tracks the following Directus collections: diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index bc77fdf5..6a4eac2f 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,9 +1,8 @@ import 'dotenv/config'; import 'reflect-metadata'; -import { Command, Option, program } from 'commander'; +import { Option, program } from 'commander'; import { - CommandName, - CommandsOptions, + DefaultConfig, disposeContext, initContext, logEndAndClose, @@ -14,14 +13,10 @@ import { runUntrack, } from './lib'; -const defaultDumpPath = './directus-config'; -const defaultConfigPath = './directus-sync.config.js'; -const defaultSnapshotPath = 'snapshot'; -const defaultCollectionsPath = 'collections'; - // Global options -const debugOption = new Option('-d, --debug', 'display more logging').default( - false, +const debugOption = new Option( + '-d, --debug', + `display more logging (default "${DefaultConfig.debug}")`, ); const directusUrlOption = new Option( '-u, --directus-url ', @@ -33,30 +28,30 @@ const directusTokenOption = new Option( ).env('DIRECTUS_TOKEN'); const configPathOption = new Option( '-c, --config-path ', - 'the path to the config file. Required for extended options', -).default(defaultConfigPath); + `the path to the config file. Required for extended options (default "${DefaultConfig.configPath}")`, +); // Shared options const noSplitOption = new Option( '--no-split', - 'should the schema snapshot be split into multiple files', -).default(true); + `should the schema snapshot be split into multiple files (default "${DefaultConfig.split}")`, +); const dumpPathOption = new Option( '--dump-path ', - 'the base path for the dump', -).default(defaultDumpPath); + `the base path for the dump (default "${DefaultConfig.dumpPath}")`, +); const collectionsPathOption = new Option( '--collections-path ', - 'the path for the collections dump, relative to the dump path', -).default(defaultCollectionsPath); + `the path for the collections dump, relative to the dump path (default "${DefaultConfig.collectionsPath}")`, +); const snapshotPathOption = new Option( '--snapshot-path ', - 'the path for the schema snapshot dump, relative to the dump path', -).default(defaultSnapshotPath); + `the path for the schema snapshot dump, relative to the dump path (default "${DefaultConfig.snapshotPath}")`, +); const forceOption = new Option( '-f, --force', - 'force the diff of schema, even if the Directus version is different', -).default(false); + `force the diff of schema, even if the Directus version is different (default "${DefaultConfig.force}")`, +); program .addOption(debugOption) @@ -107,12 +102,28 @@ program program.parse(process.argv); +/** + * Remove some default values from the program options that overrides the config file + */ +function cleanProgramOptions(programOptions: Record) { + return programOptions; +} + +/** + * Remove some default values from the command options that overrides the config file + */ +function cleanCommandOptions(commandOptions: Record) { + if (commandOptions.split === true) { + delete commandOptions.split; + } + return commandOptions; +} + function wrapAction(action: () => Promise) { - return (commandOpts: CommandsOptions[CommandName], command: Command) => { + return (commandOpts: Record) => { return initContext( - program.opts(), - command.name() as CommandName, - commandOpts, + cleanProgramOptions(program.opts()), + cleanCommandOptions(commandOpts), ) .then(action) .catch(logErrorAndStop) diff --git a/packages/cli/src/lib/loader.ts b/packages/cli/src/lib/loader.ts index caadb979..e06e6c3b 100644 --- a/packages/cli/src/lib/loader.ts +++ b/packages/cli/src/lib/loader.ts @@ -1,13 +1,10 @@ import { - CommandName, - CommandsOptions, ConfigService, DashboardsCollection, FlowsCollection, OperationsCollection, PanelsCollection, PermissionsCollection, - ProgramOptions, RolesCollection, SettingsCollection, WebhooksCollection, @@ -19,9 +16,8 @@ import { LOGGER } from './constants'; // eslint-disable-next-line @typescript-es // eslint-disable-next-line @typescript-eslint/require-await export async function initContext( - programOptions: ProgramOptions, - commandName: CommandName, - commandOptions: CommandsOptions[CommandName], + programOptions: object, + commandOptions: object, ) { // Set temporary logger, in case of error when loading the config Container.set( @@ -39,7 +35,7 @@ export async function initContext( // Get the config service const config = Container.get(ConfigService); // Set the options - config.setOptions(programOptions, commandName, commandOptions); + config.setOptions(programOptions, commandOptions); // Define the logger Container.set( LOGGER, diff --git a/packages/cli/src/lib/services/config/config.ts b/packages/cli/src/lib/services/config/config.ts index 9d4b1d55..8aeb364a 100644 --- a/packages/cli/src/lib/services/config/config.ts +++ b/packages/cli/src/lib/services/config/config.ts @@ -1,33 +1,24 @@ import { Service } from 'typedi'; -import { - CommandName, - CommandsOptions, - ConfigFileOptions, - OptionsName, - OptionsTypes, - ProgramOptions, -} from './interfaces'; +import { ConfigFileOptions, OptionName, Options } from './interfaces'; import Path from 'path'; -import { CommandsOptionsSchemas, ProgramOptionsSchema } from './schema'; import { Cacheable } from 'typescript-cacheable'; import { ConfigFileLoader } from './config-file-loader'; import { zodParse } from '../../helpers'; +import deepmerge from 'deepmerge'; +import { DefaultConfig } from './default-config'; +import { OptionsSchema } from './schema'; @Service() export class ConfigService { - protected programOptions: ProgramOptions | undefined; + protected programOptions: Partial | undefined; - protected commandOptions: CommandsOptions[CommandName] | undefined; - - protected commandName: CommandName | undefined; + protected commandOptions: Partial | undefined; setOptions( - programOptions: ProgramOptions, - commandName: CommandName, - commandOptions: CommandsOptions[CommandName], + programOptions: Partial, + commandOptions: Partial, ) { this.programOptions = programOptions; - this.commandName = commandName; this.commandOptions = commandOptions; } @@ -56,7 +47,7 @@ export class ConfigService { return { dumpPath: snapshotPath, splitFiles: this.getOptions('split'), - force: this.getOptions('force', false), + force: this.getOptions('force'), }; } @@ -81,64 +72,46 @@ export class ConfigService { return this.getOptions('configPath'); } - protected getOptions( + protected getOptions( name: T, - defaultValue?: OptionsTypes[T], - ): OptionsTypes[T] { - const fileOptions = this.getFileOptions(); - if (fileOptions && fileOptions[name as keyof ConfigFileOptions]) { - return fileOptions[name as keyof ConfigFileOptions] as OptionsTypes[T]; - } - const commandOptions = this.getCommandOptions(); - if (commandOptions[name as keyof typeof commandOptions] !== undefined) { - return commandOptions[ - name as keyof typeof commandOptions - ] as OptionsTypes[T]; - } - const programOptions = this.getGlobalOptions(); - if (programOptions[name as keyof ProgramOptions] !== undefined) { - return programOptions[name as keyof ProgramOptions] as OptionsTypes[T]; + defaultValue?: Options[T], + ): Required[T] { + const options = this.flattenOptions(); + if (options[name] !== undefined) { + return options[name] as Required[T]; } if (defaultValue === undefined) { throw new Error(`missing option ${name}`); } - return defaultValue; - } - - @Cacheable() - protected getGlobalOptions() { - if (!this.programOptions) { - throw new Error('program options not set'); - } - return zodParse( - this.programOptions, - ProgramOptionsSchema, - 'Global options', - ); + return defaultValue as Required[T]; } + /** + * Options overridden in this order: + * 1. Default options + * 2. Config file options + * 3. Program options + * 4. Command options + */ @Cacheable() - protected getCommandOptions() { - if (!this.commandName) { - throw new Error('command name not set'); - } - if (!this.commandOptions) { - throw new Error('command options not set'); - } - const schema = CommandsOptionsSchemas[this.commandName]; - if (!schema) { - throw new Error(`missing schema for command ${this.commandName}`); - } - return zodParse(this.commandOptions, schema, 'Command options'); + protected flattenOptions() { + let options = {}; + options = deepmerge(options, DefaultConfig); + options = deepmerge(options, this.getFileOptions() ?? {}); + options = deepmerge(options, this.programOptions ?? {}); + options = deepmerge(options, this.commandOptions ?? {}); + // Validate options + return zodParse(options, OptionsSchema, 'Options parsing'); } @Cacheable() protected getFileOptions(): ConfigFileOptions | undefined { - const globalOptions = this.getGlobalOptions(); - if (!globalOptions.configPath) { + const configPath = + this.programOptions?.configPath ?? DefaultConfig.configPath; + if (!configPath) { throw new Error('missing config file path'); } - const configFilePath = Path.resolve(globalOptions.configPath); - return new ConfigFileLoader(configFilePath).get(); + const configFullPath = Path.resolve(configPath); + return new ConfigFileLoader(configFullPath).get(); } } diff --git a/packages/cli/src/lib/services/config/default-config.ts b/packages/cli/src/lib/services/config/default-config.ts new file mode 100644 index 00000000..963ad1b7 --- /dev/null +++ b/packages/cli/src/lib/services/config/default-config.ts @@ -0,0 +1,14 @@ +import { Options } from './interfaces'; + +export const DefaultConfig: Partial = { + // Global + configPath: './directus-sync.config.js', + debug: false, + // Pull, diff, push + dumpPath: './directus-config', + collectionsPath: 'collections', + snapshotPath: 'snapshot', + split: true, + // Diff, push + force: false, +}; diff --git a/packages/cli/src/lib/services/config/index.ts b/packages/cli/src/lib/services/config/index.ts index df244df9..a1becd8c 100644 --- a/packages/cli/src/lib/services/config/index.ts +++ b/packages/cli/src/lib/services/config/index.ts @@ -1,4 +1,5 @@ +export * from './schema'; +export * from './interfaces'; +export * from './default-config'; export * from './config'; export * from './config-file-loader'; -export * from './interfaces'; -export * from './schema'; diff --git a/packages/cli/src/lib/services/config/interfaces.ts b/packages/cli/src/lib/services/config/interfaces.ts index ce96b548..cdb519d4 100644 --- a/packages/cli/src/lib/services/config/interfaces.ts +++ b/packages/cli/src/lib/services/config/interfaces.ts @@ -1,25 +1,12 @@ import { z } from 'zod'; import { - CommandsOptionsSchemas, ConfigFileOptionsSchema, - Options, + OptionsFields, OptionsSchema, - ProgramOptionsSchema, } from './schema'; -export type ProgramOptions = z.infer; +export type OptionName = keyof typeof OptionsFields; -export interface CommandsOptions { - pull: z.infer; - diff: z.infer; - push: z.infer; - untrack: z.infer; -} - -export type CommandName = keyof CommandsOptions; - -export type OptionsName = keyof typeof Options; - -export type OptionsTypes = z.infer; +export type Options = z.infer; export type ConfigFileOptions = z.infer; diff --git a/packages/cli/src/lib/services/config/schema.ts b/packages/cli/src/lib/services/config/schema.ts index a8cf6856..2c9c29d1 100644 --- a/packages/cli/src/lib/services/config/schema.ts +++ b/packages/cli/src/lib/services/config/schema.ts @@ -1,64 +1,34 @@ import { z } from 'zod'; -export const Options = { +export const OptionsFields = { + // Global + configPath: z.string(), debug: z.boolean(), directusUrl: z.string(), directusToken: z.string(), + // Pull, diff, push split: z.boolean(), dumpPath: z.string(), collectionsPath: z.string(), snapshotPath: z.string(), + // Diff, push force: z.boolean(), - collection: z.string(), - id: z.string(), - configPath: z.string(), -}; -export const OptionsSchema = z.object(Options); - -export const ProgramOptionsSchema = z.object({ - debug: Options.debug, - directusUrl: Options.directusUrl, - directusToken: Options.directusToken, - configPath: Options.configPath, -}); - -export const CommandsOptionsSchemas = { - pull: z.object({ - split: Options.split, - dumpPath: Options.dumpPath, - collectionsPath: Options.collectionsPath, - snapshotPath: Options.snapshotPath, - }), - diff: z.object({ - split: Options.split, - dumpPath: Options.dumpPath, - collectionsPath: Options.collectionsPath, - snapshotPath: Options.snapshotPath, - force: Options.force, - }), - push: z.object({ - split: Options.split, - dumpPath: Options.dumpPath, - collectionsPath: Options.collectionsPath, - snapshotPath: Options.snapshotPath, - force: Options.force, - }), - untrack: z.object({ - collection: Options.collection, - id: Options.id, - }), + // Untrack + collection: z.string().optional(), + id: z.string().optional(), }; +export const OptionsSchema = z.object(OptionsFields); export const ConfigFileOptionsSchema = z.object({ // Inheritance extends: z.array(z.string()).optional(), // Global options - debug: Options.debug.optional(), - directusUrl: Options.directusUrl.optional(), - directusToken: Options.directusToken.optional(), + debug: OptionsFields.debug.optional(), + directusUrl: OptionsFields.directusUrl.optional(), + directusToken: OptionsFields.directusToken.optional(), // Dump config - split: Options.split.optional(), - dumpPath: Options.dumpPath.optional(), - collectionsPath: Options.collectionsPath.optional(), - snapshotPath: Options.snapshotPath.optional(), + split: OptionsFields.split.optional(), + dumpPath: OptionsFields.dumpPath.optional(), + collectionsPath: OptionsFields.collectionsPath.optional(), + snapshotPath: OptionsFields.snapshotPath.optional(), }); From b695c9ec43b55defae4eedd70bb2ff327501479f Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Sat, 25 Nov 2023 20:06:20 -0500 Subject: [PATCH 09/18] chore: move tests --- packages/cli/{test => src/lib}/helpers.spec.ts | 2 +- .../lib}/services/config/config-file-loader.spec.ts | 2 +- packages/cli/tsconfig.base.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename packages/cli/{test => src/lib}/helpers.spec.ts (96%) rename packages/cli/{test => src/lib}/services/config/config-file-loader.spec.ts (97%) diff --git a/packages/cli/test/helpers.spec.ts b/packages/cli/src/lib/helpers.spec.ts similarity index 96% rename from packages/cli/test/helpers.spec.ts rename to packages/cli/src/lib/helpers.spec.ts index 15106dfb..98e91a60 100644 --- a/packages/cli/test/helpers.spec.ts +++ b/packages/cli/src/lib/helpers.spec.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { zodParse } from '../src/lib'; +import { zodParse } from './helpers'; describe('zodParse', () => { it('should throw error if payload is invalid', () => { diff --git a/packages/cli/test/services/config/config-file-loader.spec.ts b/packages/cli/src/lib/services/config/config-file-loader.spec.ts similarity index 97% rename from packages/cli/test/services/config/config-file-loader.spec.ts rename to packages/cli/src/lib/services/config/config-file-loader.spec.ts index f5fe3b4d..d9de1036 100644 --- a/packages/cli/test/services/config/config-file-loader.spec.ts +++ b/packages/cli/src/lib/services/config/config-file-loader.spec.ts @@ -1,4 +1,4 @@ -import { ConfigFileLoader } from '../../../src/lib'; +import { ConfigFileLoader } from './config-file-loader'; import Path from 'path'; describe('ConfigFileLoader', () => { diff --git a/packages/cli/tsconfig.base.json b/packages/cli/tsconfig.base.json index 80ca4afa..208e2fe2 100644 --- a/packages/cli/tsconfig.base.json +++ b/packages/cli/tsconfig.base.json @@ -29,5 +29,5 @@ "rootDir": "./src", "outDir": "./dist" }, - "include": ["./src/**/*.ts", "./test/**/*.ts"] + "include": ["./src/**/*.ts"] } From 04b1b8336ed9938441aed055bc6b181bd9351d84 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Sat, 25 Nov 2023 20:08:44 -0500 Subject: [PATCH 10/18] chore: ignore directus-config in prettier config --- .prettierignore | 1 + README.md | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.prettierignore b/.prettierignore index 16acd49d..8776457c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ node_modules dist package-lock.json +directus-config diff --git a/README.md b/README.md index f78d3d68..bf6ba268 100644 --- a/README.md +++ b/README.md @@ -121,14 +121,14 @@ This is an example of a configuration file: ```javascript // ./directus-sync.config.js module.exports = { - extends: ['./directus-sync.config.base.js'], - debug: true, - directusUrl: 'https://directus.example.com', - directusToken: 'my-directus-token', - split: true, - dumpPath: './directus-config', - collectionsPath: 'collections', - snapshotPath: 'snapshot', + extends: ['./directus-sync.config.base.js'], + debug: true, + directusUrl: 'https://directus.example.com', + directusToken: 'my-directus-token', + split: true, + dumpPath: './directus-config', + collectionsPath: 'collections', + snapshotPath: 'snapshot', }; ``` From be605e918f4476b65b8e6ca1162175454cbb2061 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Wed, 29 Nov 2023 23:08:53 -0500 Subject: [PATCH 11/18] feat: auth with email and password (#13) * feat: allow authentication with email/password * docs: add config example directus-sync.config.js --- README.md | 14 ++++ packages/cli/.gitignore | 1 + packages/cli/.npmignore | 3 +- packages/cli/src/index.ts | 10 +++ .../services/collections/base/data-client.ts | 8 +-- .../collections/base/id-mapper-client.ts | 32 +++++---- .../dashboards/id-mapper-client.ts | 6 +- .../collections/flows/id-mapper-client.ts | 6 +- .../operations/id-mapper-client.ts | 6 +- .../collections/panels/id-mapper-client.ts | 6 +- .../permissions/id-mapper-client.ts | 6 +- .../collections/roles/id-mapper-client.ts | 6 +- .../collections/settings/id-mapper-client.ts | 6 +- .../collections/webhooks/id-mapper-client.ts | 6 +- .../cli/src/lib/services/config/config.ts | 65 ++++++++++++------- .../cli/src/lib/services/config/helpers.ts | 10 +++ packages/cli/src/lib/services/config/index.ts | 1 + .../cli/src/lib/services/config/interfaces.ts | 11 ++++ .../cli/src/lib/services/config/schema.ts | 6 +- .../cli/src/lib/services/migration-client.ts | 47 ++++++++++---- .../lib/services/snapshot/snapshot-client.ts | 6 +- 21 files changed, 178 insertions(+), 84 deletions(-) create mode 100644 packages/cli/src/lib/services/config/helpers.ts diff --git a/README.md b/README.md index bf6ba268..6c93e77d 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,11 @@ Moreover, `directus-sync` organizes backups into multiple files, significantly i easier to track and review changes. This thoughtful separation facilitates a smoother version control process, allowing for targeted updates and clearer oversight of your Directus configurations. +# Requirements + +- Node.js 18 or higher +- `directus-extension-sync` installed on your Directus instance. See the [installation instructions](#dependency-directus-extension-sync). + # Usage The CLI is available using the `npx` command. @@ -86,6 +91,13 @@ These options can be used with any command to configure the operation of `direct - `-t, --directus-token ` Provide the Directus access token. Alternatively, set the `DIRECTUS_TOKEN` environment variable. + If provided, the `directus-email` and `directus-password` options are ignored. + +- `-e, --directus-email ` + Provide the Directus email. Alternatively, set the `DIRECTUS_ADMIN_EMAIL` environment variable. + +- `-p, --directus-password ` + Provide the Directus password. Alternatively, set the `DIRECTUS_ADMIN_PASSWORD` environment variable. - `--no-split` Indicates whether the schema snapshot should be split into multiple files. By default, snapshots are split. @@ -125,6 +137,8 @@ module.exports = { debug: true, directusUrl: 'https://directus.example.com', directusToken: 'my-directus-token', + directusEmail: 'admin@example.com', // ignored if directusToken is provided + directusPassword: 'my-directus-password', // ignored if directusToken is provided split: true, dumpPath: './directus-config', collectionsPath: 'collections', diff --git a/packages/cli/.gitignore b/packages/cli/.gitignore index 3581b486..4133cbc3 100644 --- a/packages/cli/.gitignore +++ b/packages/cli/.gitignore @@ -1,2 +1,3 @@ directus-config/ README.md +directus-sync.config.js diff --git a/packages/cli/.npmignore b/packages/cli/.npmignore index 583a99dd..769d18fe 100644 --- a/packages/cli/.npmignore +++ b/packages/cli/.npmignore @@ -8,5 +8,4 @@ test/ example.env jest.config.js tsconfig.json -test/files/config-loader/basic/directus-sync.config.js -directus-sync.config.*.js +directus-sync.config.js diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 6a4eac2f..7cd30117 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -26,6 +26,14 @@ const directusTokenOption = new Option( '-t, --directus-token ', 'Directus access token', ).env('DIRECTUS_TOKEN'); +const directusEmailOption = new Option( + '-e, --directus-email ', + 'Directus user email', +).env('DIRECTUS_ADMIN_EMAIL'); +const directusPasswordOption = new Option( + '-p, --directus-password ', + 'Directus user password', +).env('DIRECTUS_ADMIN_PASSWORD'); const configPathOption = new Option( '-c, --config-path ', `the path to the config file. Required for extended options (default "${DefaultConfig.configPath}")`, @@ -57,6 +65,8 @@ program .addOption(debugOption) .addOption(directusUrlOption) .addOption(directusTokenOption) + .addOption(directusEmailOption) + .addOption(directusPasswordOption) .addOption(configPathOption); program diff --git a/packages/cli/src/lib/services/collections/base/data-client.ts b/packages/cli/src/lib/services/collections/base/data-client.ts index 28fa2523..88d313dd 100644 --- a/packages/cli/src/lib/services/collections/base/data-client.ts +++ b/packages/cli/src/lib/services/collections/base/data-client.ts @@ -19,7 +19,7 @@ export abstract class DataClient { async query( query: Query, ): Promise { - const directus = this.migrationClient.get(); + const directus = await this.migrationClient.get(); const response = await directus.request( await this.getQueryCommand(query), ); @@ -39,7 +39,7 @@ export abstract class DataClient { * Remove the id and the syncId from the item before inserting it. */ async create(item: WithoutIdAndSyncId): Promise { - const directus = this.migrationClient.get(); + const directus = await this.migrationClient.get(); return await directus.request(await this.getInsertCommand(item)); } @@ -51,7 +51,7 @@ export abstract class DataClient { itemId: DirectusId, diffItem: Partial>, ): Promise { - const directus = this.migrationClient.get(); + const directus = await this.migrationClient.get(); return await directus.request( await this.getUpdateCommand(itemId, diffItem), ); @@ -62,7 +62,7 @@ export abstract class DataClient { * The id is the local id. */ async delete(itemId: DirectusId): Promise { - const directus = this.migrationClient.get(); + const directus = await this.migrationClient.get(); return await directus.request(await this.getDeleteCommand(itemId)); } diff --git a/packages/cli/src/lib/services/collections/base/id-mapper-client.ts b/packages/cli/src/lib/services/collections/base/id-mapper-client.ts index e81be6a1..cfc92f49 100644 --- a/packages/cli/src/lib/services/collections/base/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/base/id-mapper-client.ts @@ -1,5 +1,6 @@ import createHttpError from 'http-errors'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; +import { Cacheable } from 'typescript-cacheable'; export interface IdMap { id: number; @@ -12,10 +13,6 @@ export interface IdMap { export abstract class IdMapperClient { protected readonly extensionUri = '/directus-extension-sync'; - protected readonly url: string; - - protected readonly token: string; - /** * Cache for id maps */ @@ -28,13 +25,9 @@ export abstract class IdMapperClient { }; constructor( - protected readonly config: ConfigService, + protected readonly migrationClient: MigrationClient, protected readonly table: string, - ) { - const { url, token } = config.getDirectusConfig(); - this.url = url; - this.token = token; - } + ) {} async getBySyncId(syncId: string): Promise { // Try to get from cache @@ -126,11 +119,12 @@ export abstract class IdMapperClient { payload: unknown = undefined, options: RequestInit = {}, ): Promise { - const response = await fetch(`${this.url}${this.extensionUri}${uri}`, { + const { url, token } = await this.getUrlAndToken(); + const response = await fetch(`${url}${this.extensionUri}${uri}`, { headers: { 'Content-Type': 'application/json', Accept: 'application/json', - Authorization: `Bearer ${this.token}`, + Authorization: `Bearer ${token}`, }, method, body: payload ? JSON.stringify(payload) : null, @@ -161,6 +155,18 @@ export abstract class IdMapperClient { } } + @Cacheable() + protected async getUrlAndToken() { + const directus = await this.migrationClient.get(); + //Remove trailing slash + const url = directus.url.toString().replace(/\/$/, ''); + const token = await directus.getToken(); + if (!token) { + throw new Error('Cannot get token from Directus'); + } + return { url, token }; + } + protected addToCache(idMap: IdMap) { this.cache.bySyncId.set(idMap.sync_id, idMap); this.cache.byLocalId.set(idMap.local_id, idMap); diff --git a/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts b/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts index 2ed17419..4cc7027d 100644 --- a/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/dashboards/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { DASHBOARDS_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class DashboardsIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, DASHBOARDS_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, DASHBOARDS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts b/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts index a38ccb72..9f8c4a2f 100644 --- a/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/flows/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { FLOWS_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class FlowsIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, FLOWS_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, FLOWS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts b/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts index 5c44f370..2850e693 100644 --- a/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/operations/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { OPERATIONS_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class OperationsIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, OPERATIONS_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, OPERATIONS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts b/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts index 9aefef94..8b28f1b9 100644 --- a/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/panels/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { PANELS_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class PanelsIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, PANELS_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, PANELS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts b/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts index d1eed7ff..499e3b6a 100644 --- a/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/permissions/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { PERMISSIONS_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class PermissionsIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, PERMISSIONS_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, PERMISSIONS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts b/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts index f34556e9..7ecb04ff 100644 --- a/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/roles/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { ROLES_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class RolesIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, ROLES_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, ROLES_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts b/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts index 8f38862a..43c1f4c8 100644 --- a/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/settings/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { SETTINGS_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class SettingsIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, SETTINGS_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, SETTINGS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts b/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts index 197872fb..bf6e784b 100644 --- a/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts +++ b/packages/cli/src/lib/services/collections/webhooks/id-mapper-client.ts @@ -1,11 +1,11 @@ import { IdMapperClient } from '../base'; import { Service } from 'typedi'; import { WEBHOOKS_COLLECTION } from './constants'; -import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class WebhooksIdMapperClient extends IdMapperClient { - constructor(config: ConfigService) { - super(config, WEBHOOKS_COLLECTION); + constructor(migrationClient: MigrationClient) { + super(migrationClient, WEBHOOKS_COLLECTION); } } diff --git a/packages/cli/src/lib/services/config/config.ts b/packages/cli/src/lib/services/config/config.ts index 8aeb364a..94f84b81 100644 --- a/packages/cli/src/lib/services/config/config.ts +++ b/packages/cli/src/lib/services/config/config.ts @@ -1,5 +1,11 @@ import { Service } from 'typedi'; -import { ConfigFileOptions, OptionName, Options } from './interfaces'; +import { + ConfigFileOptions, + DirectusConfigWithCredentials, + DirectusConfigWithToken, + OptionName, + Options, +} from './interfaces'; import Path from 'path'; import { Cacheable } from 'typescript-cacheable'; import { ConfigFileLoader } from './config-file-loader'; @@ -25,14 +31,14 @@ export class ConfigService { @Cacheable() getLoggerConfig() { return { - level: this.getOptions('debug') ? 'debug' : 'info', + level: this.requireOptions('debug') ? 'debug' : 'info', }; } @Cacheable() getCollectionsConfig() { - const dumpPath = Path.resolve(this.getOptions('dumpPath')); - const collectionsSubPath = this.getOptions('collectionsPath'); + const dumpPath = Path.resolve(this.requireOptions('dumpPath')); + const collectionsSubPath = this.requireOptions('collectionsPath'); const collectionsPath = Path.resolve(dumpPath, collectionsSubPath); return { dumpPath: collectionsPath, @@ -41,49 +47,58 @@ export class ConfigService { @Cacheable() getSnapshotConfig() { - const dumpPath = Path.resolve(this.getOptions('dumpPath')); - const snapshotSubPath = this.getOptions('snapshotPath'); + const dumpPath = Path.resolve(this.requireOptions('dumpPath')); + const snapshotSubPath = this.requireOptions('snapshotPath'); const snapshotPath = Path.resolve(dumpPath, snapshotSubPath); return { dumpPath: snapshotPath, - splitFiles: this.getOptions('split'), - force: this.getOptions('force'), + splitFiles: this.requireOptions('split'), + force: this.requireOptions('force'), }; } + /** + * Returns the Directus config, either with a token or with an email/password + */ @Cacheable() - getDirectusConfig() { - return { - url: this.getOptions('directusUrl'), - token: this.getOptions('directusToken'), - }; + getDirectusConfig(): DirectusConfigWithToken | DirectusConfigWithCredentials { + const url = this.requireOptions('directusUrl'); + const token = this.getOptions('directusToken'); + if (token) { + return { url, token }; + } + + const email = this.requireOptions('directusEmail'); + const password = this.requireOptions('directusPassword'); + return { url, email, password }; } @Cacheable() getUntrackConfig() { return { - collection: this.getOptions('collection'), - id: this.getOptions('id'), + collection: this.requireOptions('collection'), + id: this.requireOptions('id'), }; } @Cacheable() getConfigFileLoaderConfig() { - return this.getOptions('configPath'); + return this.requireOptions('configPath'); } - protected getOptions( + protected getOptions(name: T): Options[T] | undefined { + const options = this.flattenOptions(); + return options[name]; + } + + protected requireOptions( name: T, - defaultValue?: Options[T], ): Required[T] { - const options = this.flattenOptions(); - if (options[name] !== undefined) { - return options[name] as Required[T]; - } - if (defaultValue === undefined) { - throw new Error(`missing option ${name}`); + const value = this.getOptions(name); + if (value !== undefined) { + return value as Required[T]; } - return defaultValue as Required[T]; + throw new Error(`Missing option ${name}`); } /** diff --git a/packages/cli/src/lib/services/config/helpers.ts b/packages/cli/src/lib/services/config/helpers.ts new file mode 100644 index 00000000..65b80be5 --- /dev/null +++ b/packages/cli/src/lib/services/config/helpers.ts @@ -0,0 +1,10 @@ +import { + DirectusConfigWithCredentials, + DirectusConfigWithToken, +} from './interfaces'; + +export function isDirectusConfigWithToken( + config: DirectusConfigWithToken | DirectusConfigWithCredentials, +): config is DirectusConfigWithToken { + return (config as DirectusConfigWithToken).token !== undefined; +} diff --git a/packages/cli/src/lib/services/config/index.ts b/packages/cli/src/lib/services/config/index.ts index a1becd8c..d7b375a4 100644 --- a/packages/cli/src/lib/services/config/index.ts +++ b/packages/cli/src/lib/services/config/index.ts @@ -3,3 +3,4 @@ export * from './interfaces'; export * from './default-config'; export * from './config'; export * from './config-file-loader'; +export * from './helpers'; diff --git a/packages/cli/src/lib/services/config/interfaces.ts b/packages/cli/src/lib/services/config/interfaces.ts index cdb519d4..da20747d 100644 --- a/packages/cli/src/lib/services/config/interfaces.ts +++ b/packages/cli/src/lib/services/config/interfaces.ts @@ -10,3 +10,14 @@ export type OptionName = keyof typeof OptionsFields; export type Options = z.infer; export type ConfigFileOptions = z.infer; + +interface DirectusConfigBase { + url: string; +} +export interface DirectusConfigWithToken extends DirectusConfigBase { + token: string; +} +export interface DirectusConfigWithCredentials extends DirectusConfigBase { + email: string; + password: string; +} diff --git a/packages/cli/src/lib/services/config/schema.ts b/packages/cli/src/lib/services/config/schema.ts index 2c9c29d1..fb502a15 100644 --- a/packages/cli/src/lib/services/config/schema.ts +++ b/packages/cli/src/lib/services/config/schema.ts @@ -5,7 +5,9 @@ export const OptionsFields = { configPath: z.string(), debug: z.boolean(), directusUrl: z.string(), - directusToken: z.string(), + directusToken: z.string().optional(), + directusEmail: z.string().optional(), + directusPassword: z.string().optional(), // Pull, diff, push split: z.boolean(), dumpPath: z.string(), @@ -26,6 +28,8 @@ export const ConfigFileOptionsSchema = z.object({ debug: OptionsFields.debug.optional(), directusUrl: OptionsFields.directusUrl.optional(), directusToken: OptionsFields.directusToken.optional(), + directusEmail: OptionsFields.directusEmail.optional(), + directusPassword: OptionsFields.directusPassword.optional(), // Dump config split: OptionsFields.split.optional(), dumpPath: OptionsFields.dumpPath.optional(), diff --git a/packages/cli/src/lib/services/migration-client.ts b/packages/cli/src/lib/services/migration-client.ts index d20a36fd..2409436d 100644 --- a/packages/cli/src/lib/services/migration-client.ts +++ b/packages/cli/src/lib/services/migration-client.ts @@ -10,24 +10,27 @@ import { import { Inject, Service } from 'typedi'; import pino from 'pino'; import { LOGGER } from '../constants'; -import { ConfigService } from './config'; +import { ConfigService, isDirectusConfigWithToken } from './config'; @Service() export class MigrationClient { protected adminRoleId: string | undefined; - protected readonly client: DirectusClient & - RestClient & - AuthenticationClient; + protected client: + | (DirectusClient & + RestClient & + AuthenticationClient) + | undefined; constructor( protected readonly config: ConfigService, @Inject(LOGGER) protected readonly logger: pino.Logger, - ) { - this.client = this.createClient(); - } + ) {} - get() { + async get() { + if (!this.client) { + this.client = await this.createClient(); + } return this.client; } @@ -36,7 +39,7 @@ export class MigrationClient { */ async getAdminRoleId() { if (!this.adminRoleId) { - const directus = this.get(); + const directus = await this.get(); const { role } = await directus.request( readMe({ fields: ['role'], @@ -47,9 +50,29 @@ export class MigrationClient { return this.adminRoleId; } - protected createClient() { - const { url, token } = this.config.getDirectusConfig(); - const client = createDirectus(url).with(rest()).with(authentication()); + protected async createClient() { + const config = this.config.getDirectusConfig(); + const client = createDirectus(config.url) + .with(rest()) + .with(authentication()); + + // If the token is already set, return it + let token: string; + if (isDirectusConfigWithToken(config)) { + token = config.token; + } + // Otherwise, login and return the token + else { + const response = await client.login(config.email, config.password); + + // Check if the token is defined + if (!response.access_token) { + throw new Error('Cannot login to Directus'); + } + + token = response.access_token; + } + client.setToken(token); return client; } diff --git a/packages/cli/src/lib/services/snapshot/snapshot-client.ts b/packages/cli/src/lib/services/snapshot/snapshot-client.ts index b73b1268..44c76d30 100644 --- a/packages/cli/src/lib/services/snapshot/snapshot-client.ts +++ b/packages/cli/src/lib/services/snapshot/snapshot-client.ts @@ -64,7 +64,7 @@ export class SnapshotClient { if (!diff?.diff) { this.logger.info('No changes to apply'); } else { - const directus = this.migrationClient.get(); + const directus = await this.migrationClient.get(); await directus.request(schemaApply(diff)); this.logger.info('Changes applied'); } @@ -114,7 +114,7 @@ export class SnapshotClient { * Get the snapshot from the Directus instance. */ protected async getSnapshot() { - const directus = this.migrationClient.get(); + const directus = await this.migrationClient.get(); return await directus.request(schemaSnapshot()); // Get better types } @@ -184,7 +184,7 @@ export class SnapshotClient { * Get the diff from Directus instance */ protected async diffSnapshot() { - const directus = this.migrationClient.get(); + const directus = await this.migrationClient.get(); const snapshot = this.loadData(); return await directus.request(schemaDiff(snapshot, this.force)); } From 55789c0ea09eca9472809fb8861f0d41a61cb06c Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Mon, 22 Jan 2024 15:41:58 -0500 Subject: [PATCH 12/18] chore: merge version 3.1 from main --- package-lock.json | 4 ++-- packages/api/CHANGELOG.md | 10 ++++++++++ packages/api/package-lock.json | 4 ++-- packages/api/package.json | 2 +- packages/cli/CHANGELOG.md | 10 ++++++++++ packages/cli/package-lock.json | 4 ++-- packages/cli/package.json | 2 +- packages/cli/src/lib/helpers.ts | 3 +++ 8 files changed, 31 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 96862510..af135a70 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16908,7 +16908,7 @@ }, "packages/api": { "name": "directus-extension-sync", - "version": "0.2.0", + "version": "0.3.1", "license": "MIT", "devDependencies": { "@directus/extensions-sdk": "10.1.13", @@ -16926,7 +16926,7 @@ }, "packages/cli": { "name": "directus-sync", - "version": "0.2.0", + "version": "0.3.1", "license": "MIT", "dependencies": { "@directus/sdk": "^12.0.1", diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index ef0c4ce1..b6c24ce5 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 0.3.1 (2024-01-16) + +**Note:** Version bump only for package directus-extension-sync + +# 0.3.0 (2023-11-30) + +### Features + +- configuration improvement ([#14](https://github.com/tractr/directus-sync/issues/14)) ([5c1ec08](https://github.com/tractr/directus-sync/commit/5c1ec0824da689774463cf0b24ca40245c4e072a)), closes [#13](https://github.com/tractr/directus-sync/issues/13) + # 0.2.0 (2023-11-21) ### Features diff --git a/packages/api/package-lock.json b/packages/api/package-lock.json index e06af26f..69ec9959 100644 --- a/packages/api/package-lock.json +++ b/packages/api/package-lock.json @@ -1,12 +1,12 @@ { "name": "directus-extension-sync", - "version": "0.2.0", + "version": "0.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "directus-extension-sync", - "version": "0.2.0", + "version": "0.3.1", "license": "MIT", "devDependencies": { "@directus/extensions-sdk": "10.1.13", diff --git a/packages/api/package.json b/packages/api/package.json index e701849f..5074f3d3 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -2,7 +2,7 @@ "name": "directus-extension-sync", "description": "This extension exposes routes to manage id mapping for Directus sync.", "icon": "extension", - "version": "0.2.0", + "version": "0.3.1", "author": "Edouard Demotes-Mainard ", "repository": { "type": "git", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 1c2b116d..e7cf0b79 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 0.3.1 (2024-01-16) + +**Note:** Version bump only for package directus-sync + +# 0.3.0 (2023-11-30) + +### Features + +- configuration improvement ([#14](https://github.com/tractr/directus-sync/issues/14)) ([5c1ec08](https://github.com/tractr/directus-sync/commit/5c1ec0824da689774463cf0b24ca40245c4e072a)), closes [#13](https://github.com/tractr/directus-sync/issues/13) + # 0.2.0 (2023-11-21) ### Features diff --git a/packages/cli/package-lock.json b/packages/cli/package-lock.json index 35e747eb..4e044b10 100644 --- a/packages/cli/package-lock.json +++ b/packages/cli/package-lock.json @@ -1,12 +1,12 @@ { "name": "directus-sync", - "version": "0.2.0", + "version": "0.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "directus-sync", - "version": "0.2.0", + "version": "0.3.1", "license": "MIT", "dependencies": { "@directus/sdk": "^12.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index f329471e..45ad59ef 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "directus-sync", - "version": "0.2.0", + "version": "0.3.1", "description": "This is a CLI tool to sync schema and configuration from one Directus instance to another.", "main": "index.js", "bin": { diff --git a/packages/cli/src/lib/helpers.ts b/packages/cli/src/lib/helpers.ts index f2c3bb31..41f2ce65 100755 --- a/packages/cli/src/lib/helpers.ts +++ b/packages/cli/src/lib/helpers.ts @@ -67,6 +67,9 @@ export function getChildLogger( */ export function loadJsonFilesRecursively(dirPath: string): T[] { const files: T[] = []; + if (!existsSync(dirPath)) { + return files; + } const fileNames = readdirSync(dirPath); for (const fileName of fileNames) { const filePath = path.join(dirPath, fileName); From a6e541828de0b19e1d5bb5389158274c550ceff7 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Mon, 22 Jan 2024 16:10:10 -0500 Subject: [PATCH 13/18] feat: preserve flow ids on pull and push (#20) * feat: configuration improvement (#14) * ci: run PR CI on next branch * feat(cli): move options to commands * feat(cli): manage config with a service * chore: optimize imports * feat: add config loader from file * test: add tests for ConfigFileLoader * feat: format zod errors * feat: merge options from default to CLI options * chore: move tests * chore: ignore directus-config in prettier config * feat: auth with email and password (#13) * feat: allow authentication with email/password * docs: add config example directus-sync.config.js * chore(release): bump version [skip ci] - directus-extension-sync@0.3.0 - directus-sync@0.3.0 * feature: treat non-existent directories as empty (#18) * chore(release): bump version [skip ci] - directus-extension-sync@0.3.1 - directus-sync@0.3.1 * feat: preserve id of flows --------- Co-authored-by: tractr-bot Co-authored-by: Evgeniy <88397573+bgenia@users.noreply.github.com> --- .../services/collections/base/data-client.ts | 5 +++-- .../collections/base/directus-collection.ts | 22 ++++++++++++++----- .../services/collections/flows/collection.ts | 2 ++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/lib/services/collections/base/data-client.ts b/packages/cli/src/lib/services/collections/base/data-client.ts index 88d313dd..94f3985d 100644 --- a/packages/cli/src/lib/services/collections/base/data-client.ts +++ b/packages/cli/src/lib/services/collections/base/data-client.ts @@ -4,6 +4,7 @@ import { DirectusId, Query, WithoutIdAndSyncId, + WithoutSyncId, } from './interfaces'; import { MigrationClient } from '../../migration-client'; @@ -38,7 +39,7 @@ export abstract class DataClient { * Inserts data into the target collection using the rest API. * Remove the id and the syncId from the item before inserting it. */ - async create(item: WithoutIdAndSyncId): Promise { + async create(item: WithoutSyncId): Promise { const directus = await this.migrationClient.get(); return await directus.request(await this.getInsertCommand(item)); } @@ -73,7 +74,7 @@ export abstract class DataClient { | Promise>; protected abstract getInsertCommand( - item: WithoutIdAndSyncId, + item: WithoutSyncId, ): | RestCommand | Promise>; diff --git a/packages/cli/src/lib/services/collections/base/directus-collection.ts b/packages/cli/src/lib/services/collections/base/directus-collection.ts index 8560a6c4..a455856f 100644 --- a/packages/cli/src/lib/services/collections/base/directus-collection.ts +++ b/packages/cli/src/lib/services/collections/base/directus-collection.ts @@ -1,9 +1,10 @@ import { IdMap, IdMapperClient } from './id-mapper-client'; import { DirectusBaseType, + DirectusId, UpdateItem, WithoutId, - WithoutIdAndSyncId, + WithoutSyncId, WithSyncId, WithSyncIdAndWithoutId, } from './interfaces'; @@ -24,6 +25,12 @@ export abstract class DirectusCollection< protected abstract readonly enableUpdate: boolean; protected abstract readonly enableDelete: boolean; + /** + * If true, the ids of the items will be used as sync ids. + * This allows to restore the same ids as the original table. + */ + protected readonly preserveIds: boolean = false; + constructor( protected readonly logger: pino.Logger, protected readonly dataDiffer: DataDiffer, @@ -131,7 +138,10 @@ export abstract class DirectusCollection< if (syncId) { output.push({ ...item, _syncId: syncId.sync_id }); } else { - const newSyncId = await this.idMapper.create(item.id); + const newSyncId = await this.idMapper.create( + item.id, + this.preserveIds ? item.id.toString() : undefined, + ); output.push({ ...item, _syncId: newSyncId }); } } @@ -163,9 +173,11 @@ export abstract class DirectusCollection< // If the id mapping was successful, create the item const { _syncId, ...rest } = mappedItem; - const newItem = await this.dataClient.create( - rest as unknown as WithoutIdAndSyncId, - ); + const createPayload = rest as unknown as WithoutSyncId; + if (this.preserveIds) { + createPayload.id = _syncId as DirectusId; + } + const newItem = await this.dataClient.create(createPayload); this.logger.debug(sourceItem, `Created item`); // Create new entry in the id mapper diff --git a/packages/cli/src/lib/services/collections/flows/collection.ts b/packages/cli/src/lib/services/collections/flows/collection.ts index 42b5d5f5..35ea06c3 100644 --- a/packages/cli/src/lib/services/collections/flows/collection.ts +++ b/packages/cli/src/lib/services/collections/flows/collection.ts @@ -17,6 +17,8 @@ export class FlowsCollection extends DirectusCollection { protected readonly enableUpdate = true; protected readonly enableDelete = true; + protected readonly preserveIds = true; + constructor( @Inject(LOGGER) baseLogger: pino.Logger, dataDiffer: FlowsDataDiffer, From 9a1a3a07540858d564e8f5f1b7a92fb3a146bc97 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Tue, 23 Jan 2024 17:51:37 -0500 Subject: [PATCH 14/18] feat: add hooks (#21) * feat(cli): add version command * feat: add hooks schema in zod * feat: add transform data hooks for onLoad and onSave events * feat: add onDump ,onSave and onLoad hooks * feat: provides Directus client in hooks * docs: add hooks examples * docs: add warning about onDump * feat: add onQuery hook * feat: import pull process * docs: add onQuery and mermaid * docs: add onQuery and mermaid * docs: improve mermaid --- README.md | 178 +++++++++++++++++- packages/cli/src/index.ts | 1 + .../services/collections/base/data-differ.ts | 3 +- .../services/collections/base/data-loader.ts | 24 ++- .../collections/base/directus-collection.ts | 40 +++- .../collections/dashboards/collection.ts | 6 + .../collections/dashboards/data-loader.ts | 6 +- .../services/collections/flows/collection.ts | 6 + .../services/collections/flows/data-loader.ts | 6 +- .../collections/operations/collection.ts | 6 + .../collections/operations/data-loader.ts | 6 +- .../services/collections/panels/collection.ts | 6 + .../collections/panels/data-loader.ts | 6 +- .../collections/permissions/collection.ts | 6 + .../collections/permissions/data-loader.ts | 6 +- .../services/collections/roles/collection.ts | 6 + .../services/collections/roles/data-loader.ts | 6 +- .../collections/settings/collection.ts | 6 + .../collections/settings/data-loader.ts | 6 +- .../collections/webhooks/collection.ts | 6 + .../collections/webhooks/data-loader.ts | 6 +- .../cli/src/lib/services/config/config.ts | 11 ++ .../cli/src/lib/services/config/interfaces.ts | 26 ++- .../cli/src/lib/services/config/schema.ts | 22 +++ 24 files changed, 371 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 6c93e77d..168303cd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ for targeted updates and clearer oversight of your Directus configurations. # Requirements - Node.js 18 or higher -- `directus-extension-sync` installed on your Directus instance. See the [installation instructions](#dependency-directus-extension-sync). +- `directus-extension-sync` installed on your Directus instance. See + the [installation instructions](#dependency-directus-extension-sync). # Usage @@ -146,6 +147,181 @@ module.exports = { }; ``` +### Hooks + +In addition to the CLI commands, `directus-sync` also supports hooks. Hooks are JavaScript functions that are executed +at specific points during the synchronization process. They can be used to transform the data coming from Directus or +going to Directus. + +Hooks are defined in the configuration file using the `hooks` property. Under this property, you can define the +collection +name and the hook function to be executed. +Available collection names are: `dashboards`, `flows`, `operations`, `panels`, `permissions`, `roles`, `settings`, +and `webhooks`. + +For each collection, available hook functions are: `onQuery`, `onLoad`, `onSave`, and `onDump`. +These can be asynchronous functions. + +During the `pull` command: + +- `onQuery` is executed just before the query is sent to Directus for get elements. It receives the query object as parameter and must + return the query object. The second parameter is the Directus client. +- `onDump` is executed just after the data is retrieved from Directus and before it is saved to the dump files. The data + is the raw data received from Directus. The second parameter is the Directus client. It must return the data to be + saved to the dump files. +- `onSave` is executed just before the cleaned data is saved to the dump files. The "cleaned" data is the data without + the columns that are ignored by `directus-sync` (such as `user_updated`) and with the relations replaced by the + SyncIDs. The first parameter is the cleaned data and the second parameter is the Directus client. It must return the + data to be saved to the dump files. + +During the `push` command: + +- `onLoad` is executed just after the data is loaded from the dump files. The data is the cleaned data, as described + above. The first parameter is the data coming from the JSON file and the second parameter is the Directus client. + It must return the data. + +#### Simple example + +Here is an example of a configuration file with hooks: + +```javascript +// ./directus-sync.config.js +module.exports = { + hooks: { + flows: { + onDump: (flows) => { + return flows.map((flow) => { + flow.name = `🧊 ${flow.name}`; + return flow; + }); + }, + onSave: (flows) => { + return flows.map((flow) => { + flow.name = `🔥 ${flow.name}`; + return flow; + }); + }, + onLoad: (flows) => { + return flows.map((flow) => { + flow.name = flow.name.replace('🔥 ', ''); + return flow; + }); + }, + }, + }, +}; +``` + +> [!WARNING] +> The dump hook is called after the mapping of the SyncIDs. This means that the data received by the hook is already +> tracked. If you filter out some elements, they will be deleted during the `push` command. + +#### Filtering out elements + +You can use `onQuery` hook to filter out elements. This hook is executed just before the query is sent to Directus, during the `pull` command. + +In the example below, the flows and operations whose name starts with `Test:` are filtered out and will not be tracked. + +```javascript +// ./directus-sync.config.js +const testPrefix = 'Test:'; + +module.exports = { + hooks: { + flows: { + onQuery: (query, client) => { + query.filter = { + ...query.filter, + name: { _nstarts_with: testPrefix }, + }; + return query; + }, + }, + operations: { + onQuery: (query, client) => { + query.filter = { + ...query.filter, + flow: { name: { _nstarts_with: testPrefix } }, + }; + return query; + }, + }, + }, +}; +``` + +> [!WARNING] +> Directus-Sync may alter the query after this hook. For example, for `roles`, the query excludes the `admin` role. + +#### Using the Directus client + +The example below shows how to disable the flows whose name starts with `Test:` and add the flow name to the operation. + +```javascript +const { readFlow } = require('@directus/sdk'); + +const testPrefix = 'Test:'; + +module.exports = { + hooks: { + flows: { + onDump: (flows) => { + return flows.map((flow) => { + flow.status = flow.name.startsWith(testPrefix) + ? 'inactive' + : 'active'; + }); + }, + }, + operations: { + onDump: async (operations, client) => { + for (const operation of operations) { + const flow = await client.request(readFlow(operation.flow)); + if (flow) { + operation.name = `${flow.name}: ${operation.name}`; + } + } + return operations; + }, + }, + }, +}; +``` + +### Lifecycle & hooks + +#### `Pull` command + +```mermaid +flowchart + subgraph Pull[Get elements - for each collection] + direction TB + B[Create query for all elements] + -->|onQuery hook|C[Add collection-specific filters] + -->D[Get elements from Directus] + -->E[Get or create SyncId for each element. Start tracking] + -->F[Remove original Id of each element] + -->|onDump hook|G[Keep elements in memory] + end + subgraph Post[Link elements - for each collection] + direction TB + H[Get all elements from memory] + --> I[Replace relations ids by SyncIds] + --> J[Remove ignore fields] + --> K[Sort elements] + -->|onSave hook|L[Save to JSON file] + end + A[Pull command] --> Pull --> Post --> Z[End] +``` + +#### `Diff` command + +**Coming soon** + +#### `Push` command + +**Coming soon** + ### Tracked Elements `directus-sync` tracks the following Directus collections: diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 7cd30117..c455f3da 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -62,6 +62,7 @@ const forceOption = new Option( ); program + .version(process.env.npm_package_version ?? 'unknown') .addOption(debugOption) .addOption(directusUrlOption) .addOption(directusTokenOption) diff --git a/packages/cli/src/lib/services/collections/base/data-differ.ts b/packages/cli/src/lib/services/collections/base/data-differ.ts index 478378a1..7e1842a6 100644 --- a/packages/cli/src/lib/services/collections/base/data-differ.ts +++ b/packages/cli/src/lib/services/collections/base/data-differ.ts @@ -36,7 +36,7 @@ export abstract class DataDiffer { * Returns the diff between the dump and the target table. */ async getDiff() { - const sourceData = this.dataLoader.getSourceData(); + const sourceData = await this.dataLoader.getSourceData(); const toCreate: WithSyncIdAndWithoutId[] = []; const toUpdate: UpdateItem[] = []; @@ -81,7 +81,6 @@ export abstract class DataDiffer { const idMap = await this.idMapper.getBySyncId(sourceItem._syncId); if (idMap) { const targetItem = await this.dataClient - .query({ filter: { id: idMap.local_id } } as Query) .then((items) => items[0]) .catch(() => { diff --git a/packages/cli/src/lib/services/collections/base/data-loader.ts b/packages/cli/src/lib/services/collections/base/data-loader.ts index 178bcbdc..f7d6c91c 100644 --- a/packages/cli/src/lib/services/collections/base/data-loader.ts +++ b/packages/cli/src/lib/services/collections/base/data-loader.ts @@ -1,26 +1,40 @@ import { DirectusBaseType, WithSyncIdAndWithoutId } from './interfaces'; import { readJsonSync, writeJsonSync } from 'fs-extra'; +import { Hooks } from '../../config'; +import { MigrationClient } from '../../migration-client'; export abstract class DataLoader { - constructor(protected readonly filePath: string) {} + constructor( + protected readonly filePath: string, + protected readonly migrationClient: MigrationClient, + protected readonly hooks: Hooks, + ) {} /** * Returns the source data from the dump file, using readFileSync * and passes it through the data transformer. */ - getSourceData(): WithSyncIdAndWithoutId[] { - return readJsonSync( + async getSourceData(): Promise[]> { + const { onLoad } = this.hooks; + const loadedData = readJsonSync( this.filePath, ) as WithSyncIdAndWithoutId[]; + return onLoad + ? await onLoad(loadedData, await this.migrationClient.get()) + : loadedData; } /** * Save the data to the dump file. The data is passed through the data transformer. */ - saveData(data: WithSyncIdAndWithoutId[]) { + async saveData(data: WithSyncIdAndWithoutId[]) { // Sort data by _syncId to avoid git changes data.sort(this.getSortFunction()); - writeJsonSync(this.filePath, data, { spaces: 2 }); + const { onSave } = this.hooks; + const transformedData = onSave + ? await onSave(data, await this.migrationClient.get()) + : data; + writeJsonSync(this.filePath, transformedData, { spaces: 2 }); } /** diff --git a/packages/cli/src/lib/services/collections/base/directus-collection.ts b/packages/cli/src/lib/services/collections/base/directus-collection.ts index a455856f..44efb465 100644 --- a/packages/cli/src/lib/services/collections/base/directus-collection.ts +++ b/packages/cli/src/lib/services/collections/base/directus-collection.ts @@ -2,6 +2,7 @@ import { IdMap, IdMapperClient } from './id-mapper-client'; import { DirectusBaseType, DirectusId, + Query, UpdateItem, WithoutId, WithoutSyncId, @@ -13,6 +14,8 @@ import { DataLoader } from './data-loader'; import { DataDiffer } from './data-differ'; import pino from 'pino'; import { DataMapper } from './data-mapper'; +import { Hooks } from '../../config'; +import { MigrationClient } from '../../migration-client'; /** * This class is responsible for merging the data from a dump to a target table. @@ -31,6 +34,11 @@ export abstract class DirectusCollection< */ protected readonly preserveIds: boolean = false; + /** + * Used to keep data in memory between the pull and the postProcessPull. + */ + protected tempData: WithSyncIdAndWithoutId[] = []; + constructor( protected readonly logger: pino.Logger, protected readonly dataDiffer: DataDiffer, @@ -38,16 +46,23 @@ export abstract class DirectusCollection< protected readonly dataClient: DataClient, protected readonly dataMapper: DataMapper, protected readonly idMapper: IdMapperClient, + protected readonly migrationClient: MigrationClient, + protected readonly hooks: Hooks, ) {} /** * Pull data from a table to a JSON file */ async pull() { - const items = await this.dataClient.query({ limit: -1 }); + const baseQuery: Query = { limit: -1 }; + const { onQuery } = this.hooks; + const transformedQuery = onQuery + ? await onQuery(baseQuery, await this.migrationClient.get()) + : baseQuery; + const items = await this.dataClient.query(transformedQuery); const mappedItems = await this.mapIdsOfItems(items); const itemsWithoutIds = this.removeIdsOfItems(mappedItems); - this.dataLoader.saveData(itemsWithoutIds); + await this.setTempData(itemsWithoutIds); this.logger.debug(`Pulled ${mappedItems.length} items.`); } @@ -55,10 +70,10 @@ export abstract class DirectusCollection< * This methods will change ids to sync ids and add users placeholders. */ async postProcessPull() { - const items = this.dataLoader.getSourceData(); + const items = this.getTempData(); const mappedItems = await this.dataMapper.mapIdsToSyncIdAndRemoveIgnoredFields(items); - this.dataLoader.saveData(mappedItems); + await this.dataLoader.saveData(mappedItems); this.logger.debug(`Post-processed ${mappedItems.length} items.`); } @@ -126,6 +141,23 @@ export abstract class DirectusCollection< this.idMapper.clearCache(); } + /** + * Temporary store the data in memory. + */ + protected async setTempData(data: WithSyncIdAndWithoutId[]) { + const { onDump } = this.hooks; + this.tempData = onDump + ? await onDump(data, await this.migrationClient.get()) + : data; + } + + /** + * Returns the data stored in memory. + */ + protected getTempData(): WithSyncIdAndWithoutId[] { + return this.tempData; + } + /** * For each item, try to get the mapped id if exists from the idMapper, or create it if not. */ diff --git a/packages/cli/src/lib/services/collections/dashboards/collection.ts b/packages/cli/src/lib/services/collections/dashboards/collection.ts index 32ec804c..3e4fe2c2 100644 --- a/packages/cli/src/lib/services/collections/dashboards/collection.ts +++ b/packages/cli/src/lib/services/collections/dashboards/collection.ts @@ -10,6 +10,8 @@ import { DASHBOARDS_COLLECTION } from './constants'; import { DashboardsDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusDashboard } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class DashboardsCollection extends DirectusCollection { @@ -24,6 +26,8 @@ export class DashboardsCollection extends DirectusCollection dataClient: DashboardsDataClient, dataMapper: DashboardsDataMapper, idMapper: DashboardsIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, ) { super( getChildLogger(baseLogger, DASHBOARDS_COLLECTION), @@ -32,6 +36,8 @@ export class DashboardsCollection extends DirectusCollection dataClient, dataMapper, idMapper, + migrationClient, + config.getHooksConfig(DASHBOARDS_COLLECTION), ); } } diff --git a/packages/cli/src/lib/services/collections/dashboards/data-loader.ts b/packages/cli/src/lib/services/collections/dashboards/data-loader.ts index 7a20f7fd..847e64f7 100644 --- a/packages/cli/src/lib/services/collections/dashboards/data-loader.ts +++ b/packages/cli/src/lib/services/collections/dashboards/data-loader.ts @@ -4,14 +4,16 @@ import { DASHBOARDS_COLLECTION } from './constants'; import path from 'path'; import { DirectusDashboard } from './interfaces'; import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class DashboardsDataLoader extends DataLoader { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${DASHBOARDS_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(DASHBOARDS_COLLECTION); + super(filePath, migrationClient, hooks); } } diff --git a/packages/cli/src/lib/services/collections/flows/collection.ts b/packages/cli/src/lib/services/collections/flows/collection.ts index 35ea06c3..bbbc9da6 100644 --- a/packages/cli/src/lib/services/collections/flows/collection.ts +++ b/packages/cli/src/lib/services/collections/flows/collection.ts @@ -10,6 +10,8 @@ import { FLOWS_COLLECTION } from './constants'; import { FlowsDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusFlow } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class FlowsCollection extends DirectusCollection { @@ -26,6 +28,8 @@ export class FlowsCollection extends DirectusCollection { dataClient: FlowsDataClient, dataMapper: FlowsDataMapper, idMapper: FlowsIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, ) { super( getChildLogger(baseLogger, FLOWS_COLLECTION), @@ -34,6 +38,8 @@ export class FlowsCollection extends DirectusCollection { dataClient, dataMapper, idMapper, + migrationClient, + config.getHooksConfig(FLOWS_COLLECTION), ); } diff --git a/packages/cli/src/lib/services/collections/flows/data-loader.ts b/packages/cli/src/lib/services/collections/flows/data-loader.ts index c6f2c7cc..588c7692 100644 --- a/packages/cli/src/lib/services/collections/flows/data-loader.ts +++ b/packages/cli/src/lib/services/collections/flows/data-loader.ts @@ -4,14 +4,16 @@ import { FLOWS_COLLECTION } from './constants'; import path from 'path'; import { DirectusFlow } from './interfaces'; import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class FlowsDataLoader extends DataLoader { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${FLOWS_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(FLOWS_COLLECTION); + super(filePath, migrationClient, hooks); } } diff --git a/packages/cli/src/lib/services/collections/operations/collection.ts b/packages/cli/src/lib/services/collections/operations/collection.ts index b6966485..04933f08 100644 --- a/packages/cli/src/lib/services/collections/operations/collection.ts +++ b/packages/cli/src/lib/services/collections/operations/collection.ts @@ -10,6 +10,8 @@ import { OPERATIONS_COLLECTION } from './constants'; import { OperationsDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusOperation } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class OperationsCollection extends DirectusCollection { @@ -24,6 +26,8 @@ export class OperationsCollection extends DirectusCollection dataClient: OperationsDataClient, dataMapper: OperationsDataMapper, idMapper: OperationsIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, ) { super( getChildLogger(baseLogger, OPERATIONS_COLLECTION), @@ -32,6 +36,8 @@ export class OperationsCollection extends DirectusCollection dataClient, dataMapper, idMapper, + migrationClient, + config.getHooksConfig(OPERATIONS_COLLECTION), ); } } diff --git a/packages/cli/src/lib/services/collections/operations/data-loader.ts b/packages/cli/src/lib/services/collections/operations/data-loader.ts index 47cf1745..7a1c0cfb 100644 --- a/packages/cli/src/lib/services/collections/operations/data-loader.ts +++ b/packages/cli/src/lib/services/collections/operations/data-loader.ts @@ -4,15 +4,17 @@ import { OPERATIONS_COLLECTION } from './constants'; import path from 'path'; import { DirectusOperation } from './interfaces'; import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class OperationsDataLoader extends DataLoader { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${OPERATIONS_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(OPERATIONS_COLLECTION); + super(filePath, migrationClient, hooks); } protected getSortFunction(): ( diff --git a/packages/cli/src/lib/services/collections/panels/collection.ts b/packages/cli/src/lib/services/collections/panels/collection.ts index 73f6adb2..c604efbd 100644 --- a/packages/cli/src/lib/services/collections/panels/collection.ts +++ b/packages/cli/src/lib/services/collections/panels/collection.ts @@ -10,6 +10,8 @@ import { PANELS_COLLECTION } from './constants'; import { PanelsDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusPanel } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class PanelsCollection extends DirectusCollection { @@ -24,6 +26,8 @@ export class PanelsCollection extends DirectusCollection { dataClient: PanelsDataClient, dataMapper: PanelsDataMapper, idMapper: PanelsIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, ) { super( getChildLogger(baseLogger, PANELS_COLLECTION), @@ -32,6 +36,8 @@ export class PanelsCollection extends DirectusCollection { dataClient, dataMapper, idMapper, + migrationClient, + config.getHooksConfig(PANELS_COLLECTION), ); } } diff --git a/packages/cli/src/lib/services/collections/panels/data-loader.ts b/packages/cli/src/lib/services/collections/panels/data-loader.ts index ab66d059..3790243b 100644 --- a/packages/cli/src/lib/services/collections/panels/data-loader.ts +++ b/packages/cli/src/lib/services/collections/panels/data-loader.ts @@ -4,14 +4,16 @@ import { PANELS_COLLECTION } from './constants'; import path from 'path'; import { DirectusPanel } from './interfaces'; import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class PanelsDataLoader extends DataLoader { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${PANELS_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(PANELS_COLLECTION); + super(filePath, migrationClient, hooks); } } diff --git a/packages/cli/src/lib/services/collections/permissions/collection.ts b/packages/cli/src/lib/services/collections/permissions/collection.ts index 80bb53c0..dec402eb 100644 --- a/packages/cli/src/lib/services/collections/permissions/collection.ts +++ b/packages/cli/src/lib/services/collections/permissions/collection.ts @@ -10,6 +10,8 @@ import { PERMISSIONS_COLLECTION } from './constants'; import { PermissionsDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusPermission } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class PermissionsCollection extends DirectusCollection { @@ -24,6 +26,8 @@ export class PermissionsCollection extends DirectusCollection { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${PERMISSIONS_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(PERMISSIONS_COLLECTION); + super(filePath, migrationClient, hooks); } protected getSortFunction(): ( diff --git a/packages/cli/src/lib/services/collections/roles/collection.ts b/packages/cli/src/lib/services/collections/roles/collection.ts index 71a1d452..3669a2c8 100644 --- a/packages/cli/src/lib/services/collections/roles/collection.ts +++ b/packages/cli/src/lib/services/collections/roles/collection.ts @@ -10,6 +10,8 @@ import { ROLES_COLLECTION } from './constants'; import { RolesDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusRole } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class RolesCollection extends DirectusCollection { @@ -24,6 +26,8 @@ export class RolesCollection extends DirectusCollection { dataClient: RolesDataClient, dataMapper: RolesDataMapper, idMapper: RolesIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, ) { super( getChildLogger(baseLogger, ROLES_COLLECTION), @@ -32,6 +36,8 @@ export class RolesCollection extends DirectusCollection { dataClient, dataMapper, idMapper, + migrationClient, + config.getHooksConfig(ROLES_COLLECTION), ); } } diff --git a/packages/cli/src/lib/services/collections/roles/data-loader.ts b/packages/cli/src/lib/services/collections/roles/data-loader.ts index e58aa165..4f1e1ac2 100644 --- a/packages/cli/src/lib/services/collections/roles/data-loader.ts +++ b/packages/cli/src/lib/services/collections/roles/data-loader.ts @@ -4,14 +4,16 @@ import { ROLES_COLLECTION } from './constants'; import path from 'path'; import { DirectusRole } from './interfaces'; import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class RolesDataLoader extends DataLoader { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${ROLES_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(ROLES_COLLECTION); + super(filePath, migrationClient, hooks); } } diff --git a/packages/cli/src/lib/services/collections/settings/collection.ts b/packages/cli/src/lib/services/collections/settings/collection.ts index bdd5cd4e..629f751a 100644 --- a/packages/cli/src/lib/services/collections/settings/collection.ts +++ b/packages/cli/src/lib/services/collections/settings/collection.ts @@ -10,6 +10,8 @@ import { SETTINGS_COLLECTION } from './constants'; import { SettingsDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusSettings } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class SettingsCollection extends DirectusCollection { @@ -24,6 +26,8 @@ export class SettingsCollection extends DirectusCollection { dataClient: SettingsDataClient, dataMapper: SettingsDataMapper, idMapper: SettingsIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, ) { super( getChildLogger(baseLogger, SETTINGS_COLLECTION), @@ -32,6 +36,8 @@ export class SettingsCollection extends DirectusCollection { dataClient, dataMapper, idMapper, + migrationClient, + config.getHooksConfig(SETTINGS_COLLECTION), ); } } diff --git a/packages/cli/src/lib/services/collections/settings/data-loader.ts b/packages/cli/src/lib/services/collections/settings/data-loader.ts index 1393918b..a2484493 100644 --- a/packages/cli/src/lib/services/collections/settings/data-loader.ts +++ b/packages/cli/src/lib/services/collections/settings/data-loader.ts @@ -5,14 +5,16 @@ import { SETTINGS_COLLECTION } from './constants'; import path from 'path'; import { DirectusSettings } from './interfaces'; import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class SettingsDataLoader extends DataLoader { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${SETTINGS_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(SETTINGS_COLLECTION); + super(filePath, migrationClient, hooks); } } diff --git a/packages/cli/src/lib/services/collections/webhooks/collection.ts b/packages/cli/src/lib/services/collections/webhooks/collection.ts index 4bcf2e36..2f29c7ba 100644 --- a/packages/cli/src/lib/services/collections/webhooks/collection.ts +++ b/packages/cli/src/lib/services/collections/webhooks/collection.ts @@ -10,6 +10,8 @@ import { WEBHOOKS_COLLECTION } from './constants'; import { WebhooksDataMapper } from './data-mapper'; import { LOGGER } from '../../../constants'; import { DirectusWebhook } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class WebhooksCollection extends DirectusCollection { @@ -24,6 +26,8 @@ export class WebhooksCollection extends DirectusCollection { dataClient: WebhooksDataClient, dataMapper: WebhooksDataMapper, idMapper: WebhooksIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, ) { super( getChildLogger(baseLogger, WEBHOOKS_COLLECTION), @@ -32,6 +36,8 @@ export class WebhooksCollection extends DirectusCollection { dataClient, dataMapper, idMapper, + migrationClient, + config.getHooksConfig(WEBHOOKS_COLLECTION), ); } } diff --git a/packages/cli/src/lib/services/collections/webhooks/data-loader.ts b/packages/cli/src/lib/services/collections/webhooks/data-loader.ts index afd9be0c..72d558ee 100644 --- a/packages/cli/src/lib/services/collections/webhooks/data-loader.ts +++ b/packages/cli/src/lib/services/collections/webhooks/data-loader.ts @@ -5,14 +5,16 @@ import { WEBHOOKS_COLLECTION } from './constants'; import path from 'path'; import { DirectusWebhook } from './interfaces'; import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; @Service() export class WebhooksDataLoader extends DataLoader { - constructor(config: ConfigService) { + constructor(config: ConfigService, migrationClient: MigrationClient) { const filePath = path.join( config.getCollectionsConfig().dumpPath, `${WEBHOOKS_COLLECTION}.json`, ); - super(filePath); + const hooks = config.getHooksConfig(WEBHOOKS_COLLECTION); + super(filePath, migrationClient, hooks); } } diff --git a/packages/cli/src/lib/services/config/config.ts b/packages/cli/src/lib/services/config/config.ts index 94f84b81..a30beca3 100644 --- a/packages/cli/src/lib/services/config/config.ts +++ b/packages/cli/src/lib/services/config/config.ts @@ -3,6 +3,8 @@ import { ConfigFileOptions, DirectusConfigWithCredentials, DirectusConfigWithToken, + HookCollectionName, + Hooks, OptionName, Options, } from './interfaces'; @@ -86,6 +88,15 @@ export class ConfigService { return this.requireOptions('configPath'); } + @Cacheable() + getHooksConfig(collection: HookCollectionName): Hooks { + const hooks = this.getOptions('hooks'); + if (!hooks) { + return {}; + } + return (hooks[collection] ?? {}) as Hooks; + } + protected getOptions(name: T): Options[T] | undefined { const options = this.flattenOptions(); return options[name]; diff --git a/packages/cli/src/lib/services/config/interfaces.ts b/packages/cli/src/lib/services/config/interfaces.ts index da20747d..71018553 100644 --- a/packages/cli/src/lib/services/config/interfaces.ts +++ b/packages/cli/src/lib/services/config/interfaces.ts @@ -1,9 +1,12 @@ -import { z } from 'zod'; -import { +import type { z } from 'zod'; +import type { ConfigFileOptionsSchema, OptionsFields, + OptionsHooksSchema, OptionsSchema, } from './schema'; +import type { MigrationClient } from '../migration-client'; +import type { DirectusBaseType, Query } from '../collections'; export type OptionName = keyof typeof OptionsFields; @@ -11,6 +14,25 @@ export type Options = z.infer; export type ConfigFileOptions = z.infer; +export type HookCollectionName = keyof typeof OptionsHooksSchema.shape; + +export type TransformDataFunction = ( + data: T[], + directusClient: Awaited>, +) => T[] | Promise; + +export type TransformQueryFunction = >( + query: T, + directusClient: Awaited>, +) => T | Promise; + +export interface Hooks { + onLoad?: TransformDataFunction; + onDump?: TransformDataFunction; + onSave?: TransformDataFunction; + onQuery?: TransformQueryFunction; +} + interface DirectusConfigBase { url: string; } diff --git a/packages/cli/src/lib/services/config/schema.ts b/packages/cli/src/lib/services/config/schema.ts index fb502a15..18161b0a 100644 --- a/packages/cli/src/lib/services/config/schema.ts +++ b/packages/cli/src/lib/services/config/schema.ts @@ -1,5 +1,23 @@ import { z } from 'zod'; +export const HooksSchema = z.object({ + onLoad: z.function().optional(), + onDump: z.function().optional(), + onSave: z.function().optional(), + onQuery: z.function().optional(), +}); + +export const OptionsHooksSchema = z.object({ + dashboards: HooksSchema.optional(), + flows: HooksSchema.optional(), + operations: HooksSchema.optional(), + panels: HooksSchema.optional(), + permissions: HooksSchema.optional(), + roles: HooksSchema.optional(), + settings: HooksSchema.optional(), + webhooks: HooksSchema.optional(), +}); + export const OptionsFields = { // Global configPath: z.string(), @@ -18,6 +36,8 @@ export const OptionsFields = { // Untrack collection: z.string().optional(), id: z.string().optional(), + // Hooks + hooks: OptionsHooksSchema.optional(), }; export const OptionsSchema = z.object(OptionsFields); @@ -35,4 +55,6 @@ export const ConfigFileOptionsSchema = z.object({ dumpPath: OptionsFields.dumpPath.optional(), collectionsPath: OptionsFields.collectionsPath.optional(), snapshotPath: OptionsFields.snapshotPath.optional(), + // Hooks config + hooks: OptionsHooksSchema.optional(), }); From 0eea13166530ddd52bfacc47bceab40ab2d27d6c Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Tue, 23 Jan 2024 18:01:03 -0500 Subject: [PATCH 15/18] feat: handle translations collection (#22) --- README.md | 5 ++- packages/cli/src/lib/loader.ts | 2 + .../cli/src/lib/services/collections/index.ts | 1 + .../collections/translations/collection.ts | 43 +++++++++++++++++++ .../collections/translations/constants.ts | 1 + .../collections/translations/data-client.ts | 36 ++++++++++++++++ .../collections/translations/data-differ.ts | 31 +++++++++++++ .../collections/translations/data-loader.ts | 20 +++++++++ .../collections/translations/data-mapper.ts | 15 +++++++ .../translations/id-mapper-client.ts | 11 +++++ .../collections/translations/index.ts | 8 ++++ .../collections/translations/interfaces.ts | 4 ++ .../cli/src/lib/services/config/schema.ts | 1 + 13 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/lib/services/collections/translations/collection.ts create mode 100644 packages/cli/src/lib/services/collections/translations/constants.ts create mode 100644 packages/cli/src/lib/services/collections/translations/data-client.ts create mode 100644 packages/cli/src/lib/services/collections/translations/data-differ.ts create mode 100644 packages/cli/src/lib/services/collections/translations/data-loader.ts create mode 100644 packages/cli/src/lib/services/collections/translations/data-mapper.ts create mode 100644 packages/cli/src/lib/services/collections/translations/id-mapper-client.ts create mode 100644 packages/cli/src/lib/services/collections/translations/index.ts create mode 100644 packages/cli/src/lib/services/collections/translations/interfaces.ts diff --git a/README.md b/README.md index 168303cd..47200d76 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ going to Directus. Hooks are defined in the configuration file using the `hooks` property. Under this property, you can define the collection name and the hook function to be executed. -Available collection names are: `dashboards`, `flows`, `operations`, `panels`, `permissions`, `roles`, `settings`, +Available collection names are: `dashboards`, `flows`, `operations`, `panels`, `permissions`, `roles`, `settings`, `translations`, and `webhooks`. For each collection, available hook functions are: `onQuery`, `onLoad`, `onSave`, and `onDump`. @@ -333,6 +333,7 @@ flowchart - permissions - roles - settings +- translations - webhooks For these collections, data changes are committed to the code, allowing for replication on other Directus instances. A @@ -362,7 +363,7 @@ configurations and schema within Directus. Here is a step-by-step explanation of Upon execution of the `pull` command, `directus-sync` will: 1. Scan the specified Directus collections, which include dashboards, flows, operations, panels, permissions, roles, - settings, and webhooks. + settings, translations and webhooks. 2. Assign a SyncID to each element within these collections if it doesn't already have one. 3. Commit the data of these collections into code, allowing for versioning and tracking of configuration changes. diff --git a/packages/cli/src/lib/loader.ts b/packages/cli/src/lib/loader.ts index e06e6c3b..df13e488 100644 --- a/packages/cli/src/lib/loader.ts +++ b/packages/cli/src/lib/loader.ts @@ -7,6 +7,7 @@ import { PermissionsCollection, RolesCollection, SettingsCollection, + TranslationsCollection, WebhooksCollection, } from './services'; import { createDumpFolders } from './helpers'; @@ -62,6 +63,7 @@ export function loadCollections() { // The collections are populated in the same order return [ Container.get(SettingsCollection), + Container.get(TranslationsCollection), Container.get(WebhooksCollection), Container.get(FlowsCollection), Container.get(OperationsCollection), diff --git a/packages/cli/src/lib/services/collections/index.ts b/packages/cli/src/lib/services/collections/index.ts index 1fa2bc22..aba7a3ca 100644 --- a/packages/cli/src/lib/services/collections/index.ts +++ b/packages/cli/src/lib/services/collections/index.ts @@ -1,6 +1,7 @@ export * from './base'; export * from './flows'; export * from './settings'; +export * from './translations'; export * from './webhooks'; export * from './operations'; export * from './roles'; diff --git a/packages/cli/src/lib/services/collections/translations/collection.ts b/packages/cli/src/lib/services/collections/translations/collection.ts new file mode 100644 index 00000000..a07d4218 --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/collection.ts @@ -0,0 +1,43 @@ +import { DirectusCollection } from '../base'; +import pino from 'pino'; +import { Inject, Service } from 'typedi'; +import { TranslationsDataLoader } from './data-loader'; +import { TranslationsDataClient } from './data-client'; +import { TranslationsIdMapperClient } from './id-mapper-client'; +import { TranslationsDataDiffer } from './data-differ'; +import { getChildLogger } from '../../../helpers'; +import { TRANSLATIONS_COLLECTION } from './constants'; +import { TranslationsDataMapper } from './data-mapper'; +import { LOGGER } from '../../../constants'; +import { DirectusTranslation } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; + +@Service() +export class TranslationsCollection extends DirectusCollection { + protected readonly enableCreate = true; + protected readonly enableUpdate = true; + protected readonly enableDelete = true; + + constructor( + @Inject(LOGGER) baseLogger: pino.Logger, + dataDiffer: TranslationsDataDiffer, + dataLoader: TranslationsDataLoader, + dataClient: TranslationsDataClient, + dataMapper: TranslationsDataMapper, + idMapper: TranslationsIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, + ) { + super( + getChildLogger(baseLogger, TRANSLATIONS_COLLECTION), + dataDiffer, + dataLoader, + dataClient, + dataMapper, + idMapper, + migrationClient, + config.getHooksConfig(TRANSLATIONS_COLLECTION), + ); + } +} diff --git a/packages/cli/src/lib/services/collections/translations/constants.ts b/packages/cli/src/lib/services/collections/translations/constants.ts new file mode 100644 index 00000000..4432d974 --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/constants.ts @@ -0,0 +1 @@ +export const TRANSLATIONS_COLLECTION = 'translations'; diff --git a/packages/cli/src/lib/services/collections/translations/data-client.ts b/packages/cli/src/lib/services/collections/translations/data-client.ts new file mode 100644 index 00000000..3b9dfd61 --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/data-client.ts @@ -0,0 +1,36 @@ +import { DataClient, Query, WithoutIdAndSyncId } from '../base'; +import { + createTranslation, + deleteTranslation, + readTranslations, + updateTranslation, +} from '@directus/sdk'; +import { Service } from 'typedi'; +import { MigrationClient } from '../../migration-client'; +import { DirectusTranslation } from './interfaces'; + +@Service() +export class TranslationsDataClient extends DataClient { + constructor(migrationClient: MigrationClient) { + super(migrationClient); + } + + protected getDeleteCommand(itemId: number) { + return deleteTranslation(itemId); + } + + protected getInsertCommand(item: WithoutIdAndSyncId) { + return createTranslation(item); + } + + protected getQueryCommand(query: Query) { + return readTranslations(query); + } + + protected getUpdateCommand( + itemId: number, + diffItem: Partial>, + ) { + return updateTranslation(itemId, diffItem); + } +} diff --git a/packages/cli/src/lib/services/collections/translations/data-differ.ts b/packages/cli/src/lib/services/collections/translations/data-differ.ts new file mode 100644 index 00000000..4a6edc7f --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/data-differ.ts @@ -0,0 +1,31 @@ +import { DataDiffer } from '../base'; + +import { Inject, Service } from 'typedi'; +import { TRANSLATIONS_COLLECTION } from './constants'; +import pino from 'pino'; +import { TranslationsDataLoader } from './data-loader'; +import { TranslationsDataClient } from './data-client'; +import { TranslationsIdMapperClient } from './id-mapper-client'; +import { getChildLogger } from '../../../helpers'; +import { TranslationsDataMapper } from './data-mapper'; +import { LOGGER } from '../../../constants'; +import { DirectusTranslation } from './interfaces'; + +@Service() +export class TranslationsDataDiffer extends DataDiffer { + constructor( + @Inject(LOGGER) baseLogger: pino.Logger, + dataLoader: TranslationsDataLoader, + dataClient: TranslationsDataClient, + dataMapper: TranslationsDataMapper, + idMapper: TranslationsIdMapperClient, + ) { + super( + getChildLogger(baseLogger, TRANSLATIONS_COLLECTION), + dataLoader, + dataClient, + dataMapper, + idMapper, + ); + } +} diff --git a/packages/cli/src/lib/services/collections/translations/data-loader.ts b/packages/cli/src/lib/services/collections/translations/data-loader.ts new file mode 100644 index 00000000..1cf184e9 --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/data-loader.ts @@ -0,0 +1,20 @@ +import { DataLoader } from '../base'; + +import { Service } from 'typedi'; +import { TRANSLATIONS_COLLECTION } from './constants'; +import path from 'path'; +import { DirectusTranslation } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; + +@Service() +export class TranslationsDataLoader extends DataLoader { + constructor(config: ConfigService, migrationClient: MigrationClient) { + const filePath = path.join( + config.getCollectionsConfig().dumpPath, + `${TRANSLATIONS_COLLECTION}.json`, + ); + const hooks = config.getHooksConfig(TRANSLATIONS_COLLECTION); + super(filePath, migrationClient, hooks); + } +} diff --git a/packages/cli/src/lib/services/collections/translations/data-mapper.ts b/packages/cli/src/lib/services/collections/translations/data-mapper.ts new file mode 100644 index 00000000..73be526d --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/data-mapper.ts @@ -0,0 +1,15 @@ +import { DataMapper } from '../base'; + +import { Inject, Service } from 'typedi'; +import pino from 'pino'; +import { getChildLogger } from '../../../helpers'; +import { LOGGER } from '../../../constants'; +import { TRANSLATIONS_COLLECTION } from './constants'; +import { DirectusTranslation } from './interfaces'; + +@Service() +export class TranslationsDataMapper extends DataMapper { + constructor(@Inject(LOGGER) baseLogger: pino.Logger) { + super(getChildLogger(baseLogger, TRANSLATIONS_COLLECTION)); + } +} diff --git a/packages/cli/src/lib/services/collections/translations/id-mapper-client.ts b/packages/cli/src/lib/services/collections/translations/id-mapper-client.ts new file mode 100644 index 00000000..18e12a88 --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/id-mapper-client.ts @@ -0,0 +1,11 @@ +import { IdMapperClient } from '../base'; +import { Service } from 'typedi'; +import { TRANSLATIONS_COLLECTION } from './constants'; +import { MigrationClient } from '../../migration-client'; + +@Service() +export class TranslationsIdMapperClient extends IdMapperClient { + constructor(migrationClient: MigrationClient) { + super(migrationClient, TRANSLATIONS_COLLECTION); + } +} diff --git a/packages/cli/src/lib/services/collections/translations/index.ts b/packages/cli/src/lib/services/collections/translations/index.ts new file mode 100644 index 00000000..203e6c76 --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/index.ts @@ -0,0 +1,8 @@ +export * from './interfaces'; +export * from './constants'; +export * from './collection'; +export * from './data-client'; +export * from './data-differ'; +export * from './data-mapper'; +export * from './data-loader'; +export * from './id-mapper-client'; diff --git a/packages/cli/src/lib/services/collections/translations/interfaces.ts b/packages/cli/src/lib/services/collections/translations/interfaces.ts new file mode 100644 index 00000000..45a07899 --- /dev/null +++ b/packages/cli/src/lib/services/collections/translations/interfaces.ts @@ -0,0 +1,4 @@ +import { DirectusTranslation as BaseDirectusTranslation } from '@directus/sdk'; +import { BaseSchema } from '../base'; + +export type DirectusTranslation = BaseDirectusTranslation; diff --git a/packages/cli/src/lib/services/config/schema.ts b/packages/cli/src/lib/services/config/schema.ts index 18161b0a..40c06f34 100644 --- a/packages/cli/src/lib/services/config/schema.ts +++ b/packages/cli/src/lib/services/config/schema.ts @@ -15,6 +15,7 @@ export const OptionsHooksSchema = z.object({ permissions: HooksSchema.optional(), roles: HooksSchema.optional(), settings: HooksSchema.optional(), + translations: HooksSchema.optional(), webhooks: HooksSchema.optional(), }); From 024f62bdfd8da41fc2285ebc96dd628a40497c89 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Wed, 31 Jan 2024 11:54:42 -0500 Subject: [PATCH 16/18] feat: handle folders collection (#27) --- packages/cli/src/lib/loader.ts | 2 + .../collections/folders/collection.ts | 45 +++++++++++++++++++ .../services/collections/folders/constants.ts | 1 + .../collections/folders/data-client.ts | 36 +++++++++++++++ .../collections/folders/data-differ.ts | 31 +++++++++++++ .../collections/folders/data-loader.ts | 20 +++++++++ .../collections/folders/data-mapper.ts | 19 ++++++++ .../collections/folders/id-mapper-client.ts | 11 +++++ .../lib/services/collections/folders/index.ts | 8 ++++ .../collections/folders/interfaces.ts | 4 ++ .../cli/src/lib/services/collections/index.ts | 1 + .../cli/src/lib/services/config/schema.ts | 1 + 12 files changed, 179 insertions(+) create mode 100644 packages/cli/src/lib/services/collections/folders/collection.ts create mode 100644 packages/cli/src/lib/services/collections/folders/constants.ts create mode 100644 packages/cli/src/lib/services/collections/folders/data-client.ts create mode 100644 packages/cli/src/lib/services/collections/folders/data-differ.ts create mode 100644 packages/cli/src/lib/services/collections/folders/data-loader.ts create mode 100644 packages/cli/src/lib/services/collections/folders/data-mapper.ts create mode 100644 packages/cli/src/lib/services/collections/folders/id-mapper-client.ts create mode 100644 packages/cli/src/lib/services/collections/folders/index.ts create mode 100644 packages/cli/src/lib/services/collections/folders/interfaces.ts diff --git a/packages/cli/src/lib/loader.ts b/packages/cli/src/lib/loader.ts index df13e488..da1b8be4 100644 --- a/packages/cli/src/lib/loader.ts +++ b/packages/cli/src/lib/loader.ts @@ -2,6 +2,7 @@ import { ConfigService, DashboardsCollection, FlowsCollection, + FoldersCollection, OperationsCollection, PanelsCollection, PermissionsCollection, @@ -63,6 +64,7 @@ export function loadCollections() { // The collections are populated in the same order return [ Container.get(SettingsCollection), + Container.get(FoldersCollection), Container.get(TranslationsCollection), Container.get(WebhooksCollection), Container.get(FlowsCollection), diff --git a/packages/cli/src/lib/services/collections/folders/collection.ts b/packages/cli/src/lib/services/collections/folders/collection.ts new file mode 100644 index 00000000..b0f57cc8 --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/collection.ts @@ -0,0 +1,45 @@ +import { DirectusCollection } from '../base'; +import pino from 'pino'; +import { Inject, Service } from 'typedi'; +import { FoldersDataLoader } from './data-loader'; +import { FoldersDataClient } from './data-client'; +import { FoldersIdMapperClient } from './id-mapper-client'; +import { FoldersDataDiffer } from './data-differ'; +import { getChildLogger } from '../../../helpers'; +import { FOLDERS_COLLECTION } from './constants'; +import { FoldersDataMapper } from './data-mapper'; +import { LOGGER } from '../../../constants'; +import { DirectusFolder } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; + +@Service() +export class FoldersCollection extends DirectusCollection { + protected readonly enableCreate = true; + protected readonly enableUpdate = true; + protected readonly enableDelete = true; + + protected readonly preserveIds = true; + + constructor( + @Inject(LOGGER) baseLogger: pino.Logger, + dataDiffer: FoldersDataDiffer, + dataLoader: FoldersDataLoader, + dataClient: FoldersDataClient, + dataMapper: FoldersDataMapper, + idMapper: FoldersIdMapperClient, + config: ConfigService, + migrationClient: MigrationClient, + ) { + super( + getChildLogger(baseLogger, FOLDERS_COLLECTION), + dataDiffer, + dataLoader, + dataClient, + dataMapper, + idMapper, + migrationClient, + config.getHooksConfig(FOLDERS_COLLECTION), + ); + } +} diff --git a/packages/cli/src/lib/services/collections/folders/constants.ts b/packages/cli/src/lib/services/collections/folders/constants.ts new file mode 100644 index 00000000..1c0ccd3e --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/constants.ts @@ -0,0 +1 @@ +export const FOLDERS_COLLECTION = 'folders'; diff --git a/packages/cli/src/lib/services/collections/folders/data-client.ts b/packages/cli/src/lib/services/collections/folders/data-client.ts new file mode 100644 index 00000000..05acdc49 --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/data-client.ts @@ -0,0 +1,36 @@ +import { DataClient, Query, WithoutIdAndSyncId } from '../base'; +import { + createFolder, + deleteFolder, + readFolders, + updateFolder, +} from '@directus/sdk'; +import { Service } from 'typedi'; +import { MigrationClient } from '../../migration-client'; +import { DirectusFolder } from './interfaces'; + +@Service() +export class FoldersDataClient extends DataClient { + constructor(migrationClient: MigrationClient) { + super(migrationClient); + } + + protected getDeleteCommand(itemId: string) { + return deleteFolder(itemId); + } + + protected getInsertCommand(item: WithoutIdAndSyncId) { + return createFolder(item); + } + + protected getQueryCommand(query: Query) { + return readFolders(query); + } + + protected getUpdateCommand( + itemId: string, + diffItem: Partial>, + ) { + return updateFolder(itemId, diffItem); + } +} diff --git a/packages/cli/src/lib/services/collections/folders/data-differ.ts b/packages/cli/src/lib/services/collections/folders/data-differ.ts new file mode 100644 index 00000000..f9821ced --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/data-differ.ts @@ -0,0 +1,31 @@ +import { DataDiffer } from '../base'; + +import { Inject, Service } from 'typedi'; +import { FOLDERS_COLLECTION } from './constants'; +import pino from 'pino'; +import { FoldersDataLoader } from './data-loader'; +import { FoldersDataClient } from './data-client'; +import { FoldersIdMapperClient } from './id-mapper-client'; +import { getChildLogger } from '../../../helpers'; +import { FoldersDataMapper } from './data-mapper'; +import { LOGGER } from '../../../constants'; +import { DirectusFolder } from './interfaces'; + +@Service() +export class FoldersDataDiffer extends DataDiffer { + constructor( + @Inject(LOGGER) baseLogger: pino.Logger, + dataLoader: FoldersDataLoader, + dataClient: FoldersDataClient, + dataMapper: FoldersDataMapper, + idMapper: FoldersIdMapperClient, + ) { + super( + getChildLogger(baseLogger, FOLDERS_COLLECTION), + dataLoader, + dataClient, + dataMapper, + idMapper, + ); + } +} diff --git a/packages/cli/src/lib/services/collections/folders/data-loader.ts b/packages/cli/src/lib/services/collections/folders/data-loader.ts new file mode 100644 index 00000000..3e3ff436 --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/data-loader.ts @@ -0,0 +1,20 @@ +import { DataLoader } from '../base'; + +import { Service } from 'typedi'; +import { FOLDERS_COLLECTION } from './constants'; +import path from 'path'; +import { DirectusFolder } from './interfaces'; +import { ConfigService } from '../../config'; +import { MigrationClient } from '../../migration-client'; + +@Service() +export class FoldersDataLoader extends DataLoader { + constructor(config: ConfigService, migrationClient: MigrationClient) { + const filePath = path.join( + config.getCollectionsConfig().dumpPath, + `${FOLDERS_COLLECTION}.json`, + ); + const hooks = config.getHooksConfig(FOLDERS_COLLECTION); + super(filePath, migrationClient, hooks); + } +} diff --git a/packages/cli/src/lib/services/collections/folders/data-mapper.ts b/packages/cli/src/lib/services/collections/folders/data-mapper.ts new file mode 100644 index 00000000..c550b6f2 --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/data-mapper.ts @@ -0,0 +1,19 @@ +import { DataMapper, IdMappers } from '../base'; + +import { Container, Inject, Service } from 'typedi'; +import pino from 'pino'; +import { getChildLogger } from '../../../helpers'; +import { LOGGER } from '../../../constants'; +import { FOLDERS_COLLECTION } from './constants'; +import { DirectusFolder } from './interfaces'; +import { FoldersIdMapperClient } from './id-mapper-client'; + +@Service() +export class FoldersDataMapper extends DataMapper { + protected idMappers: IdMappers = { + parent: Container.get(FoldersIdMapperClient), + }; + constructor(@Inject(LOGGER) baseLogger: pino.Logger) { + super(getChildLogger(baseLogger, FOLDERS_COLLECTION)); + } +} diff --git a/packages/cli/src/lib/services/collections/folders/id-mapper-client.ts b/packages/cli/src/lib/services/collections/folders/id-mapper-client.ts new file mode 100644 index 00000000..6d789f40 --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/id-mapper-client.ts @@ -0,0 +1,11 @@ +import { IdMapperClient } from '../base'; +import { Service } from 'typedi'; +import { FOLDERS_COLLECTION } from './constants'; +import { MigrationClient } from '../../migration-client'; + +@Service() +export class FoldersIdMapperClient extends IdMapperClient { + constructor(migrationClient: MigrationClient) { + super(migrationClient, FOLDERS_COLLECTION); + } +} diff --git a/packages/cli/src/lib/services/collections/folders/index.ts b/packages/cli/src/lib/services/collections/folders/index.ts new file mode 100644 index 00000000..203e6c76 --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/index.ts @@ -0,0 +1,8 @@ +export * from './interfaces'; +export * from './constants'; +export * from './collection'; +export * from './data-client'; +export * from './data-differ'; +export * from './data-mapper'; +export * from './data-loader'; +export * from './id-mapper-client'; diff --git a/packages/cli/src/lib/services/collections/folders/interfaces.ts b/packages/cli/src/lib/services/collections/folders/interfaces.ts new file mode 100644 index 00000000..d385dea3 --- /dev/null +++ b/packages/cli/src/lib/services/collections/folders/interfaces.ts @@ -0,0 +1,4 @@ +import { DirectusFolder as BaseDirectusFolder } from '@directus/sdk'; +import { BaseSchema } from '../base'; + +export type DirectusFolder = BaseDirectusFolder; diff --git a/packages/cli/src/lib/services/collections/index.ts b/packages/cli/src/lib/services/collections/index.ts index aba7a3ca..4a690a26 100644 --- a/packages/cli/src/lib/services/collections/index.ts +++ b/packages/cli/src/lib/services/collections/index.ts @@ -1,5 +1,6 @@ export * from './base'; export * from './flows'; +export * from './folders'; export * from './settings'; export * from './translations'; export * from './webhooks'; diff --git a/packages/cli/src/lib/services/config/schema.ts b/packages/cli/src/lib/services/config/schema.ts index 9ff64185..a996c38e 100644 --- a/packages/cli/src/lib/services/config/schema.ts +++ b/packages/cli/src/lib/services/config/schema.ts @@ -10,6 +10,7 @@ export const HooksSchema = z.object({ export const OptionsHooksSchema = z.object({ dashboards: HooksSchema.optional(), flows: HooksSchema.optional(), + folders: HooksSchema.optional(), operations: HooksSchema.optional(), panels: HooksSchema.optional(), permissions: HooksSchema.optional(), From 6b61104e6fc221fa884c5f3c1282d42b6ef4f2b5 Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Wed, 31 Jan 2024 12:08:56 -0500 Subject: [PATCH 17/18] docs: add folders in README.md --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e80aa4b8..539d3234 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ going to Directus. Hooks are defined in the configuration file using the `hooks` property. Under this property, you can define the collection name and the hook function to be executed. -Available collection names are: `dashboards`, `flows`, `operations`, `panels`, `permissions`, `roles`, `settings`, `translations`, +Available collection names are: `dashboards`, `flows`, `folders`, `operations`, `panels`, `permissions`, `roles`, `settings`, `translations`, and `webhooks`. For each collection, available hook functions are: `onQuery`, `onLoad`, `onSave`, and `onDump`. @@ -329,6 +329,7 @@ flowchart - dashboards - flows +- folders - operations - panels - permissions @@ -363,7 +364,7 @@ configurations and schema within Directus. Here is a step-by-step explanation of Upon execution of the `pull` command, `directus-sync` will: -1. Scan the specified Directus collections, which include dashboards, flows, operations, panels, permissions, roles, +1. Scan the specified Directus collections, which include dashboards, flows, folders, operations, panels, permissions, roles, settings, translations and webhooks. 2. Assign a SyncID to each element within these collections if it doesn't already have one. 3. Commit the data of these collections into code, allowing for versioning and tracking of configuration changes. @@ -371,6 +372,10 @@ Upon execution of the `pull` command, `directus-sync` will: This SyncID tagging facilitates the replication of configurations across different instances of Directus while maintaining the integrity and links between different entities. +> [!NOTE] +> The original IDs of the flows are preserved to maintain the URLs of the `webhook` type flows. +> The original IDs of the folders are preserved to maintain the associations with fields of the `file` and `image` types. + ### Mapping Table Since it's not possible to add tags directly to entities within Directus, `directus-sync` uses a mapping table that From 85a61acb4407ac46bffc7ab064dec0ebba32d2de Mon Sep 17 00:00:00 2001 From: Edouard Demotes Date: Wed, 31 Jan 2024 12:17:28 -0500 Subject: [PATCH 18/18] fix: allow missing collection JSON (#28) --- .../cli/src/lib/services/collections/base/data-loader.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/lib/services/collections/base/data-loader.ts b/packages/cli/src/lib/services/collections/base/data-loader.ts index f7d6c91c..873cd3b2 100644 --- a/packages/cli/src/lib/services/collections/base/data-loader.ts +++ b/packages/cli/src/lib/services/collections/base/data-loader.ts @@ -16,9 +16,8 @@ export abstract class DataLoader { */ async getSourceData(): Promise[]> { const { onLoad } = this.hooks; - const loadedData = readJsonSync( - this.filePath, - ) as WithSyncIdAndWithoutId[]; + const loadedData: WithSyncIdAndWithoutId[] = + readJsonSync(this.filePath, { throws: false }) || []; return onLoad ? await onLoad(loadedData, await this.migrationClient.get()) : loadedData;