[Backport 2.x] Refactor repository reading for recursive directory se…

…arching (#1246)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

server/adaptors/integrations/__test__/local_repository.test.ts

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { RepositoryReader } from '../repository/repository';
+import { TemplateManager } from '../repository/repository';
 import { IntegrationReader } from '../repository/integration';
 import path from 'path';
 import * as fs from 'fs/promises';
@@ -27,7 +27,7 @@ describe('The local repository', () => {
   });
 
   it('Should pass deep validation for all local integrations.', async () => {
-    const repository: RepositoryReader = new RepositoryReader(
+    const repository: TemplateManager = new TemplateManager(
       path.join(__dirname, '../__data__/repository')
     );
     const integrations: IntegrationReader[] = await repository.getIntegrationList();

server/adaptors/integrations/__test__/manager.test.ts

@@ -5,14 +5,14 @@
 
 import { IntegrationsManager } from '../integrations_manager';
 import { SavedObject, SavedObjectsClientContract } from '../../../../../../src/core/server/types';
-import { RepositoryReader } from '../repository/repository';
+import { TemplateManager } from '../repository/repository';
 import { IntegrationInstanceBuilder } from '../integrations_builder';
 import { IntegrationReader } from '../repository/integration';
 import { SavedObjectsFindResponse } from '../../../../../../src/core/server';
 
 describe('IntegrationsKibanaBackend', () => {
   let mockSavedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
-  let mockRepository: jest.Mocked<RepositoryReader>;
+  let mockRepository: jest.Mocked<TemplateManager>;
   let backend: IntegrationsManager;
 
   beforeEach(() => {

server/adaptors/integrations/integrations_manager.ts

@@ -8,17 +8,17 @@ import { addRequestToMetric } from '../../common/metrics/metrics_helper';
 import { IntegrationsAdaptor } from './integrations_adaptor';
 import { SavedObject, SavedObjectsClientContract } from '../../../../../src/core/server/types';
 import { IntegrationInstanceBuilder } from './integrations_builder';
-import { RepositoryReader } from './repository/repository';
+import { TemplateManager } from './repository/repository';
 
 export class IntegrationsManager implements IntegrationsAdaptor {
   client: SavedObjectsClientContract;
   instanceBuilder: IntegrationInstanceBuilder;
-  repository: RepositoryReader;
+  repository: TemplateManager;
 
-  constructor(client: SavedObjectsClientContract, repository?: RepositoryReader) {
+  constructor(client: SavedObjectsClientContract, repository?: TemplateManager) {
     this.client = client;
     this.repository =
-      repository ?? new RepositoryReader(path.join(__dirname, '__data__/repository'));
+      repository ?? new TemplateManager(path.join(__dirname, '__data__/repository'));
     this.instanceBuilder = new IntegrationInstanceBuilder(this.client);
   }
 

server/adaptors/integrations/repository/__test__/repository.test.ts

@@ -4,18 +4,22 @@
  */
 
 import * as fs from 'fs/promises';
-import { RepositoryReader } from '../repository';
+import { TemplateManager } from '../repository';
 import { IntegrationReader } from '../integration';
 import { Dirent, Stats } from 'fs';
 import path from 'path';
 
 jest.mock('fs/promises');
 
 describe('Repository', () => {
-  let repository: RepositoryReader;
+  let repository: TemplateManager;
 
   beforeEach(() => {
-    repository = new RepositoryReader('path/to/directory');
+    repository = new TemplateManager('path/to/directory');
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
   });
 
   describe('getIntegrationList', () => {

server/adaptors/integrations/repository/fs_data_adaptor.ts

@@ -109,13 +109,32 @@ export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor {
 
   async findIntegrations(dirname: string = '.'): Promise<Result<string[]>> {
     try {
-      const files = await fs.readdir(path.join(this.directory, dirname));
-      return { ok: true, value: files };
+      const integrations: string[] = [];
+      await this.collectIntegrationsRecursive(dirname, integrations);
+      return { ok: true, value: integrations };
     } catch (err: any) {
       return { ok: false, error: err };
     }
   }
 
+  private async collectIntegrationsRecursive(
+    dirname: string,
+    integrations: string[]
+  ): Promise<void> {
+    const entries = await fs.readdir(path.join(this.directory, dirname));
+
+    for (const entry of entries) {
+      const fullPath = path.join(dirname, entry);
+      const isDirectory = (await this.getDirectoryType(fullPath)) === 'integration';
+
+      if (isDirectory) {
+        integrations.push(fullPath);
+      } else if ((await this.getDirectoryType(fullPath)) === 'repository') {
+        await this.collectIntegrationsRecursive(fullPath, integrations);
+      }
+    }
+  }
+
   async findIntegrationVersions(dirname: string = '.'): Promise<Result<string[]>> {
     let files;
     const integPath = path.join(this.directory, dirname);

server/adaptors/integrations/repository/repository.ts

@@ -7,7 +7,7 @@ import * as path from 'path';
 import { IntegrationReader } from './integration';
 import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor';
 
-export class RepositoryReader {
+export class TemplateManager {
   reader: CatalogDataAdaptor;
   directory: string;
 
@@ -24,20 +24,22 @@ export class RepositoryReader {
       return [];
     }
     const integrations = await Promise.all(
-      folders.value.map((i) => this.getIntegration(path.basename(i)))
+      folders.value.map((i) =>
+        this.getIntegration(path.relative(this.directory, path.join(this.directory, i)))
+      )
     );
     return integrations.filter((x) => x !== null) as IntegrationReader[];
   }
 
-  async getIntegration(name: string): Promise<IntegrationReader | null> {
-    if ((await this.reader.getDirectoryType(name)) !== 'integration') {
-      console.error(`Requested integration '${name}' does not exist`);
+  async getIntegration(integPath: string): Promise<IntegrationReader | null> {
+    if ((await this.reader.getDirectoryType(integPath)) !== 'integration') {
+      console.error(`Requested integration '${integPath}' does not exist`);
       return null;
     }
-    const integ = new IntegrationReader(name, this.reader.join(name));
+    const integ = new IntegrationReader(integPath, this.reader.join(integPath));
     const checkResult = await integ.getConfig();
     if (!checkResult.ok) {
-      console.error(`Integration '${name}' is invalid:`, checkResult.error);
+      console.error(`Integration '${integPath}' is invalid:`, checkResult.error);
       return null;
     }
     return integ;