feat(datasource-customizer): implement gmail-style search (#780)

.codeclimate.yml

@@ -1,10 +1,11 @@
-version: '2' # required to adjust maintainability checks
+version: "2" # required to adjust maintainability checks
 
 checks:
   method-complexity:
     enabled: true
     config:
       threshold: 10
 exclude_patterns:
-  - 'packages/_example/'
-  - '**/test/'
+  - "packages/_example/"
+  - "**/test/"
+  - "packages/datasource-customizer/src/decorators/search/generated-parser/*.ts"

.github/workflows/build.yml

@@ -79,7 +79,7 @@ jobs:
         with:
           node-version: 18.14.0
       - name: Start docker containers
-        if: ${{ matrix.package == 'datasource-mongoose' || matrix.package == 'datasource-sql' }}
+        if: ${{ matrix.package == 'datasource-mongoose' || matrix.package == 'datasource-sql' || matrix.package == 'datasource-sequelize' }}
         run: docker-compose -f ./packages/${{ matrix.package }}/docker-compose.yml up -d; sleep 5
       - name: Restore dependencies from cache
         uses: actions/cache/restore@v3

jest.config.ts

@@ -6,6 +6,7 @@ const config: Config.InitialOptions = {
   collectCoverageFrom: [
     '<rootDir>/packages/*/src/**/*.ts',
     '!<rootDir>/packages/_example/src/**/*.ts',
+    '!<rootDir>/packages/datasource-customizer/src/decorators/search/generated-parser/*.ts',
   ],
   testMatch: ['<rootDir>/packages/*/test/**/*.test.ts'],
   setupFilesAfterEnv: ['jest-extended/all'],

packages/datasource-customizer/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ignorePatterns: ['src/decorators/search/generated-parser/**/*'],
+};

packages/datasource-customizer/package.json

@@ -16,6 +16,7 @@
     "dist/**/*.d.ts"
   ],
   "scripts": {
+    "build:parser": "antlr4 -Xexact-output-dir -o src/decorators/search/generated-parser -Dlanguage=TypeScript  src/decorators/search/Query.g4",
     "build": "tsc",
     "build:watch": "tsc --watch",
     "clean": "rm -rf coverage dist",
@@ -29,6 +30,7 @@
   },
   "dependencies": {
     "@forestadmin/datasource-toolkit": "1.29.2",
+    "antlr4": "^4.13.1-patch-1",
     "file-type": "^16.5.4",
     "luxon": "^3.2.1",
     "object-hash": "^3.0.0",

packages/datasource-customizer/src/decorators/search/Query.g4

@@ -0,0 +1,45 @@
+//
+// This file contains the description of the supported syntax for search queries.
+// It is used by the ANTLR parser generator to generate the parser.
+// To support additional syntax, this file must be updated and the parser regenerated.
+// It requires antlr4-tools to be installed on your machine
+// 
+grammar Query;
+
+query: (and | or | queryToken | parenthesized) EOF;
+
+parenthesized: PARENS_OPEN (or | and) PARENS_CLOSE;
+PARENS_OPEN: '(' ' '*;
+PARENS_CLOSE:  ' '* ')';
+
+// OR is a bit different because of operator precedence
+or: (and | queryToken | parenthesized) (SEPARATOR OR SEPARATOR (and | queryToken | parenthesized))+;
+OR: 'OR';
+
+and: (queryToken | parenthesized) (SEPARATOR (AND SEPARATOR)? (queryToken | parenthesized))+;
+AND: 'AND';
+
+queryToken: (quoted | negated | propertyMatching | word);
+
+
+quoted: SINGLE_QUOTED | DOUBLE_QUOTED;
+SINGLE_QUOTED: '\'' SINGLE_QUOTED_CONTENT '\'' | '\'' '\'';
+fragment SINGLE_QUOTED_CONTENT:~[']*; 
+DOUBLE_QUOTED: '"' DOUBLE_QUOTED_CONTENT '"' | '"' '"';
+fragment DOUBLE_QUOTED_CONTENT: ~["]*; 
+    
+negated: NEGATION (word | quoted | propertyMatching);
+NEGATION: '-';
+
+propertyMatching: name ':' value;
+name: TOKEN;
+value: word | quoted;
+
+word: TOKEN;
+TOKEN: ONE_CHAR_TOKEN | TWO_CHARS_TOKEN | MULTIPLE_CHARS_TOKEN;
+fragment ONE_CHAR_TOKEN: ~[\r\n :\-()];
+fragment TWO_CHARS_TOKEN: ~[\r\n :\-(]~[\r\n :)];
+fragment MULTIPLE_CHARS_TOKEN:~[\r\n :\-(]~[\r\n :]+~[\r\n :)];
+    
+SEPARATOR: SPACING+ | EOF;
+SPACING: [\r\n ];

packages/datasource-customizer/src/decorators/search/collection-search-context.ts

@@ -0,0 +1,42 @@
+import { Caller, Collection, ConditionTree } from '@forestadmin/datasource-toolkit';
+
+import CollectionCustomizationContext from '../../context/collection-context';
+import { TCollectionName, TColumnName, TSchema } from '../../templates';
+
+export type SearchOptions<
+  S extends TSchema = TSchema,
+  N extends TCollectionName<S> = TCollectionName<S>,
+> = {
+  /**
+   * Include fields from the first level of relations
+   */
+  extended?: boolean;
+  /**
+   * Remove these fields from the default search fields
+   */
+  excludeFields?: Array<TColumnName<S, N>>;
+  /**
+   * Add these fields to the default search fields
+   */
+  includeFields?: Array<TColumnName<S, N>>;
+  /**
+   * Replace the list of default searched field by these fields
+   */
+  onlyFields?: Array<TColumnName<S, N>>;
+};
+
+export default class CollectionSearchContext<
+  S extends TSchema = TSchema,
+  N extends TCollectionName<S> = TCollectionName<S>,
+> extends CollectionCustomizationContext<S, N> {
+  constructor(
+    collection: Collection,
+    caller: Caller,
+    public readonly generateSearchFilter: (
+      searchText: string,
+      options?: SearchOptions<S, N>,
+    ) => ConditionTree,
+  ) {
+    super(collection, caller);
+  }
+}

packages/datasource-customizer/src/decorators/search/collection.ts

@@ -6,43 +6,45 @@ import {
   ColumnSchema,
   ConditionTree,
   ConditionTreeFactory,
-  ConditionTreeLeaf,
-  Operator,
+  DataSourceDecorator,
   PaginatedFilter,
 } from '@forestadmin/datasource-toolkit';
-import { validate as uuidValidate } from 'uuid';
 
+import CollectionSearchContext, { SearchOptions } from './collection-search-context';
+import normalizeName from './normalize-name';
+import { extractSpecifiedFields, generateConditionTree, parseQuery } from './parse-query';
 import { SearchDefinition } from './types';
-import CollectionCustomizationContext from '../../context/collection-context';
 
 export default class SearchCollectionDecorator extends CollectionDecorator {
+  override dataSource: DataSourceDecorator<SearchCollectionDecorator>;
   replacer: SearchDefinition = null;
 
   replaceSearch(replacer: SearchDefinition): void {
     this.replacer = replacer;
   }
 
-  public override refineSchema(subSchema: CollectionSchema): CollectionSchema {
+  override refineSchema(subSchema: CollectionSchema): CollectionSchema {
     return { ...subSchema, searchable: true };
   }
 
-  public override async refineFilter(
-    caller: Caller,
-    filter?: PaginatedFilter,
-  ): Promise<PaginatedFilter> {
+  override async refineFilter(caller: Caller, filter?: PaginatedFilter): Promise<PaginatedFilter> {
     // Search string is not significant
     if (!filter?.search?.trim().length) {
       return filter?.override({ search: null });
     }
 
     // Implement search ourselves
     if (this.replacer || !this.childCollection.schema.searchable) {
-      const ctx = new CollectionCustomizationContext(this, caller);
-      let tree = this.defaultReplacer(filter.search, filter.searchExtended);
+      const ctx = new CollectionSearchContext(this, caller, this.generateSearchFilter.bind(this));
+      let tree: ConditionTree;
 
       if (this.replacer) {
         const plainTree = await this.replacer(filter.search, filter.searchExtended, ctx);
         tree = ConditionTreeFactory.fromPlainObject(plainTree);
+      } else {
+        tree = this.generateSearchFilter(caller, filter.search, {
+          extended: filter.searchExtended,
+        });
       }
 
       // Note that if no fields are searchable with the provided searchString, the conditions
@@ -58,57 +60,41 @@ export default class SearchCollectionDecorator extends CollectionDecorator {
     return filter;
   }
 
-  private defaultReplacer(search: string, extended: boolean): ConditionTree {
-    const searchableFields = SearchCollectionDecorator.getFields(this.childCollection, extended);
-    const conditions = searchableFields
-      .map(([field, schema]) => SearchCollectionDecorator.buildCondition(field, schema, search))
-      .filter(Boolean);
-
-    return ConditionTreeFactory.union(...conditions);
-  }
-
-  private static buildCondition(
-    field: string,
-    schema: ColumnSchema,
-    searchString: string,
+  private generateSearchFilter(
+    caller: Caller,
+    searchText: string,
+    options?: SearchOptions,
   ): ConditionTree {
-    const { columnType, enumValues, filterOperators } = schema;
-    const isNumber = Number(searchString).toString() === searchString;
-    const isUuid = uuidValidate(searchString);
-
-    if (columnType === 'Number' && isNumber && filterOperators?.has('Equal')) {
-      return new ConditionTreeLeaf(field, 'Equal', Number(searchString));
-    }
-
-    if (columnType === 'Enum' && filterOperators?.has('Equal')) {
-      const searchValue = SearchCollectionDecorator.lenientFind(enumValues, searchString);
-
-      if (searchValue) return new ConditionTreeLeaf(field, 'Equal', searchValue);
-    }
-
-    if (columnType === 'String') {
-      const isCaseSensitive = searchString.toLocaleLowerCase() !== searchString.toLocaleUpperCase();
-      const supportsIContains = filterOperators?.has('IContains');
-      const supportsContains = filterOperators?.has('Contains');
-      const supportsEqual = filterOperators?.has('Equal');
-
-      // Perf: don't use case-insensitive operator when the search string is indifferent to case
-      let operator: Operator;
-      if (supportsIContains && (isCaseSensitive || !supportsContains)) operator = 'IContains';
-      else if (supportsContains) operator = 'Contains';
-      else if (supportsEqual) operator = 'Equal';
+    const parsedQuery = parseQuery(searchText);
+
+    const specifiedFields = options?.onlyFields ? [] : extractSpecifiedFields(parsedQuery);
+
+    const defaultFields = options?.onlyFields
+      ? []
+      : this.getFields(this.childCollection, Boolean(options?.extended));
+
+    const searchableFields = new Map(
+      [
+        ...defaultFields,
+        ...[...specifiedFields, ...(options?.onlyFields ?? []), ...(options?.includeFields ?? [])]
+          .map(name => this.lenientGetSchema(name))
+          .filter(Boolean)
+          .map(schema => [schema.field, schema.schema] as [string, ColumnSchema]),
+      ]
+        .filter(Boolean)
+        .filter(([field]) => !options?.excludeFields?.includes(field)),
+    );
 
-      if (operator) return new ConditionTreeLeaf(field, operator, searchString);
-    }
+    const conditionTree = generateConditionTree(caller, parsedQuery, [...searchableFields]);
 
-    if (columnType === 'Uuid' && isUuid && filterOperators?.has('Equal')) {
-      return new ConditionTreeLeaf(field, 'Equal', searchString);
+    if (!conditionTree && searchText.trim().length) {
+      return ConditionTreeFactory.MatchNone;
     }
 
-    return null;
+    return conditionTree;
   }
 
-  private static getFields(collection: Collection, extended: boolean): [string, ColumnSchema][] {
+  private getFields(collection: Collection, extended: boolean): [string, ColumnSchema][] {
     const fields: [string, ColumnSchema][] = [];
 
     for (const [name, field] of Object.entries(collection.schema.fields)) {
@@ -125,10 +111,30 @@ export default class SearchCollectionDecorator extends CollectionDecorator {
     return fields;
   }
 
-  private static lenientFind(haystack: string[], needle: string): string {
-    return (
-      haystack?.find(v => v === needle.trim()) ??
-      haystack?.find(v => v.toLocaleLowerCase() === needle.toLocaleLowerCase().trim())
-    );
+  private lenientGetSchema(path: string): { field: string; schema: ColumnSchema } | null {
+    const [prefix, suffix] = path.split(/:(.*)/);
+    const fuzzyPrefix = normalizeName(prefix);
+
+    for (const [field, schema] of Object.entries(this.schema.fields)) {
+      const fuzzyFieldName = normalizeName(field);
+
+      if (fuzzyPrefix === fuzzyFieldName) {
+        if (!suffix && schema.type === 'Column') {
+          return { field, schema };
+        }
+
+        if (
+          suffix &&
+          (schema.type === 'OneToMany' || schema.type === 'ManyToOne' || schema.type === 'OneToOne')
+        ) {
+          const related = this.dataSource.getCollection(schema.foreignCollection);
+          const fuzzy = related.lenientGetSchema(suffix);
+
+          if (fuzzy) return { field: `${field}:${fuzzy.field}`, schema: fuzzy.schema };
+        }
+      }
+    }
+
+    return null;
   }
 }