feat: add hooks (#21)

* feat(cli): add version command

* feat: add hooks schema in zod

* feat: add transform data hooks for onLoad and onSave events

* feat: add onDump ,onSave and onLoad hooks

* feat: provides Directus client in hooks

* docs: add hooks examples

* docs: add warning about onDump

* feat: add onQuery hook

* feat: import pull process

* docs: add onQuery and mermaid

* docs: add onQuery and mermaid

* docs: improve mermaid

README.md

@@ -16,7 +16,8 @@ for targeted updates and clearer oversight of your Directus configurations.
 # Requirements
 
 - Node.js 18 or higher
-- `directus-extension-sync` installed on your Directus instance. See the [installation instructions](#dependency-directus-extension-sync).
+- `directus-extension-sync` installed on your Directus instance. See
+  the [installation instructions](#dependency-directus-extension-sync).
 
 # Usage
 
@@ -146,6 +147,181 @@ module.exports = {
 };
 ```
 
+### Hooks
+
+In addition to the CLI commands, `directus-sync` also supports hooks. Hooks are JavaScript functions that are executed
+at specific points during the synchronization process. They can be used to transform the data coming from Directus or
+going to Directus.
+
+Hooks are defined in the configuration file using the `hooks` property. Under this property, you can define the
+collection
+name and the hook function to be executed.
+Available collection names are: `dashboards`, `flows`, `operations`, `panels`, `permissions`, `roles`, `settings`,
+and `webhooks`.
+
+For each collection, available hook functions are: `onQuery`, `onLoad`, `onSave`, and `onDump`.
+These can be asynchronous functions.
+
+During the `pull` command:
+
+- `onQuery` is executed just before the query is sent to Directus for get elements. It receives the query object as parameter and must
+  return the query object. The second parameter is the Directus client.
+- `onDump` is executed just after the data is retrieved from Directus and before it is saved to the dump files. The data
+  is the raw data received from Directus. The second parameter is the Directus client. It must return the data to be
+  saved to the dump files.
+- `onSave` is executed just before the cleaned data is saved to the dump files. The "cleaned" data is the data without
+  the columns that are ignored by `directus-sync` (such as `user_updated`) and with the relations replaced by the
+  SyncIDs. The first parameter is the cleaned data and the second parameter is the Directus client. It must return the
+  data to be saved to the dump files.
+
+During the `push` command:
+
+- `onLoad` is executed just after the data is loaded from the dump files. The data is the cleaned data, as described
+  above. The first parameter is the data coming from the JSON file and the second parameter is the Directus client.
+  It must return the data.
+
+#### Simple example
+
+Here is an example of a configuration file with hooks:
+
+```javascript
+// ./directus-sync.config.js
+module.exports = {
+  hooks: {
+    flows: {
+      onDump: (flows) => {
+        return flows.map((flow) => {
+          flow.name = `🧊 ${flow.name}`;
+          return flow;
+        });
+      },
+      onSave: (flows) => {
+        return flows.map((flow) => {
+          flow.name = `🔥 ${flow.name}`;
+          return flow;
+        });
+      },
+      onLoad: (flows) => {
+        return flows.map((flow) => {
+          flow.name = flow.name.replace('🔥 ', '');
+          return flow;
+        });
+      },
+    },
+  },
+};
+```
+
+> [!WARNING]  
+> The dump hook is called after the mapping of the SyncIDs. This means that the data received by the hook is already
+> tracked. If you filter out some elements, they will be deleted during the `push` command.
+
+#### Filtering out elements
+
+You can use `onQuery` hook to filter out elements. This hook is executed just before the query is sent to Directus, during the `pull` command.
+
+In the example below, the flows and operations whose name starts with `Test:` are filtered out and will not be tracked.
+
+```javascript
+// ./directus-sync.config.js
+const testPrefix = 'Test:';
+
+module.exports = {
+  hooks: {
+    flows: {
+      onQuery: (query, client) => {
+        query.filter = {
+          ...query.filter,
+          name: { _nstarts_with: testPrefix },
+        };
+        return query;
+      },
+    },
+    operations: {
+      onQuery: (query, client) => {
+        query.filter = {
+          ...query.filter,
+          flow: { name: { _nstarts_with: testPrefix } },
+        };
+        return query;
+      },
+    },
+  },
+};
+```
+
+> [!WARNING]
+> Directus-Sync may alter the query after this hook. For example, for `roles`, the query excludes the `admin` role.
+
+#### Using the Directus client
+
+The example below shows how to disable the flows whose name starts with `Test:` and add the flow name to the operation.
+
+```javascript
+const { readFlow } = require('@directus/sdk');
+
+const testPrefix = 'Test:';
+
+module.exports = {
+  hooks: {
+    flows: {
+      onDump: (flows) => {
+        return flows.map((flow) => {
+          flow.status = flow.name.startsWith(testPrefix)
+            ? 'inactive'
+            : 'active';
+        });
+      },
+    },
+    operations: {
+      onDump: async (operations, client) => {
+        for (const operation of operations) {
+          const flow = await client.request(readFlow(operation.flow));
+          if (flow) {
+            operation.name = `${flow.name}: ${operation.name}`;
+          }
+        }
+        return operations;
+      },
+    },
+  },
+};
+```
+
+### Lifecycle & hooks
+
+#### `Pull` command
+
+```mermaid
+flowchart
+  subgraph Pull[Get elements - for each collection]
+    direction TB
+		B[Create query for all elements]
+    -->|onQuery hook|C[Add collection-specific filters]
+    -->D[Get elements from Directus]
+    -->E[Get or create SyncId for each element. Start tracking]
+    -->F[Remove original Id of each element]
+    -->|onDump hook|G[Keep elements in memory]
+  end
+	subgraph Post[Link elements - for each collection]
+    direction TB
+		H[Get all elements from memory]
+    --> I[Replace relations ids by SyncIds]
+    --> J[Remove ignore fields]
+    --> K[Sort elements]
+    -->|onSave hook|L[Save to JSON file]
+  end
+ A[Pull command] --> Pull --> Post --> Z[End]
+```
+
+#### `Diff` command
+
+**Coming soon**
+
+#### `Push` command
+
+**Coming soon**
+
 ### Tracked Elements
 
 `directus-sync` tracks the following Directus collections:

packages/cli/src/index.ts

@@ -62,6 +62,7 @@ const forceOption = new Option(
 );
 
 program
+  .version(process.env.npm_package_version ?? 'unknown')
   .addOption(debugOption)
   .addOption(directusUrlOption)
   .addOption(directusTokenOption)

packages/cli/src/lib/services/collections/base/data-differ.ts

@@ -36,7 +36,7 @@ export abstract class DataDiffer<DirectusType extends DirectusBaseType> {
    * Returns the diff between the dump and the target table.
    */
   async getDiff() {
-    const sourceData = this.dataLoader.getSourceData();
+    const sourceData = await this.dataLoader.getSourceData();
 
     const toCreate: WithSyncIdAndWithoutId<DirectusType>[] = [];
     const toUpdate: UpdateItem<DirectusType>[] = [];
@@ -81,7 +81,6 @@ export abstract class DataDiffer<DirectusType extends DirectusBaseType> {
     const idMap = await this.idMapper.getBySyncId(sourceItem._syncId);
     if (idMap) {
       const targetItem = await this.dataClient
-
         .query({ filter: { id: idMap.local_id } } as Query<DirectusType>)
         .then((items) => items[0])
         .catch(() => {

packages/cli/src/lib/services/collections/base/data-loader.ts

@@ -1,26 +1,40 @@
 import { DirectusBaseType, WithSyncIdAndWithoutId } from './interfaces';
 import { readJsonSync, writeJsonSync } from 'fs-extra';
+import { Hooks } from '../../config';
+import { MigrationClient } from '../../migration-client';
 
 export abstract class DataLoader<DirectusType extends DirectusBaseType> {
-  constructor(protected readonly filePath: string) {}
+  constructor(
+    protected readonly filePath: string,
+    protected readonly migrationClient: MigrationClient,
+    protected readonly hooks: Hooks,
+  ) {}
 
   /**
    * Returns the source data from the dump file, using readFileSync
    * and passes it through the data transformer.
    */
-  getSourceData(): WithSyncIdAndWithoutId<DirectusType>[] {
-    return readJsonSync(
+  async getSourceData(): Promise<WithSyncIdAndWithoutId<DirectusType>[]> {
+    const { onLoad } = this.hooks;
+    const loadedData = readJsonSync(
       this.filePath,
     ) as WithSyncIdAndWithoutId<DirectusType>[];
+    return onLoad
+      ? await onLoad(loadedData, await this.migrationClient.get())
+      : loadedData;
   }
 
   /**
    * Save the data to the dump file. The data is passed through the data transformer.
    */
-  saveData(data: WithSyncIdAndWithoutId<DirectusType>[]) {
+  async saveData(data: WithSyncIdAndWithoutId<DirectusType>[]) {
     // Sort data by _syncId to avoid git changes
     data.sort(this.getSortFunction());
-    writeJsonSync(this.filePath, data, { spaces: 2 });
+    const { onSave } = this.hooks;
+    const transformedData = onSave
+      ? await onSave(data, await this.migrationClient.get())
+      : data;
+    writeJsonSync(this.filePath, transformedData, { spaces: 2 });
   }
 
   /**

packages/cli/src/lib/services/collections/base/directus-collection.ts

@@ -2,6 +2,7 @@ import { IdMap, IdMapperClient } from './id-mapper-client';
 import {
   DirectusBaseType,
   DirectusId,
+  Query,
   UpdateItem,
   WithoutId,
   WithoutSyncId,
@@ -13,6 +14,8 @@ import { DataLoader } from './data-loader';
 import { DataDiffer } from './data-differ';
 import pino from 'pino';
 import { DataMapper } from './data-mapper';
+import { Hooks } from '../../config';
+import { MigrationClient } from '../../migration-client';
 
 /**
  * This class is responsible for merging the data from a dump to a target table.
@@ -31,34 +34,46 @@ export abstract class DirectusCollection<
    */
   protected readonly preserveIds: boolean = false;
 
+  /**
+   * Used to keep data in memory between the pull and the postProcessPull.
+   */
+  protected tempData: WithSyncIdAndWithoutId<DirectusType>[] = [];
+
   constructor(
     protected readonly logger: pino.Logger,
     protected readonly dataDiffer: DataDiffer<DirectusType>,
     protected readonly dataLoader: DataLoader<DirectusType>,
     protected readonly dataClient: DataClient<DirectusType>,
     protected readonly dataMapper: DataMapper<DirectusType>,
     protected readonly idMapper: IdMapperClient,
+    protected readonly migrationClient: MigrationClient,
+    protected readonly hooks: Hooks,
   ) {}
 
   /**
    * Pull data from a table to a JSON file
    */
   async pull() {
-    const items = await this.dataClient.query({ limit: -1 });
+    const baseQuery: Query<DirectusType> = { limit: -1 };
+    const { onQuery } = this.hooks;
+    const transformedQuery = onQuery
+      ? await onQuery(baseQuery, await this.migrationClient.get())
+      : baseQuery;
+    const items = await this.dataClient.query(transformedQuery);
     const mappedItems = await this.mapIdsOfItems(items);
     const itemsWithoutIds = this.removeIdsOfItems(mappedItems);
-    this.dataLoader.saveData(itemsWithoutIds);
+    await this.setTempData(itemsWithoutIds);
     this.logger.debug(`Pulled ${mappedItems.length} items.`);
   }
 
   /**
    * This methods will change ids to sync ids and add users placeholders.
    */
   async postProcessPull() {
-    const items = this.dataLoader.getSourceData();
+    const items = this.getTempData();
     const mappedItems =
       await this.dataMapper.mapIdsToSyncIdAndRemoveIgnoredFields(items);
-    this.dataLoader.saveData(mappedItems);
+    await this.dataLoader.saveData(mappedItems);
     this.logger.debug(`Post-processed ${mappedItems.length} items.`);
   }
 
@@ -126,6 +141,23 @@ export abstract class DirectusCollection<
     this.idMapper.clearCache();
   }
 
+  /**
+   * Temporary store the data in memory.
+   */
+  protected async setTempData(data: WithSyncIdAndWithoutId<DirectusType>[]) {
+    const { onDump } = this.hooks;
+    this.tempData = onDump
+      ? await onDump(data, await this.migrationClient.get())
+      : data;
+  }
+
+  /**
+   * Returns the data stored in memory.
+   */
+  protected getTempData(): WithSyncIdAndWithoutId<DirectusType>[] {
+    return this.tempData;
+  }
+
   /**
    * For each item, try to get the mapped id if exists from the idMapper, or create it if not.
    */

packages/cli/src/lib/services/collections/dashboards/collection.ts

@@ -10,6 +10,8 @@ import { DASHBOARDS_COLLECTION } from './constants';
 import { DashboardsDataMapper } from './data-mapper';
 import { LOGGER } from '../../../constants';
 import { DirectusDashboard } from './interfaces';
+import { ConfigService } from '../../config';
+import { MigrationClient } from '../../migration-client';
 
 @Service()
 export class DashboardsCollection extends DirectusCollection<DirectusDashboard> {
@@ -24,6 +26,8 @@ export class DashboardsCollection extends DirectusCollection<DirectusDashboard>
     dataClient: DashboardsDataClient,
     dataMapper: DashboardsDataMapper,
     idMapper: DashboardsIdMapperClient,
+    config: ConfigService,
+    migrationClient: MigrationClient,
   ) {
     super(
       getChildLogger(baseLogger, DASHBOARDS_COLLECTION),
@@ -32,6 +36,8 @@ export class DashboardsCollection extends DirectusCollection<DirectusDashboard>
       dataClient,
       dataMapper,
       idMapper,
+      migrationClient,
+      config.getHooksConfig(DASHBOARDS_COLLECTION),
     );
   }
 }