[Workspace] Add optional workspaces parameter to all saved objects A…

…PI (#185) * [Workspace] Add workspaces parameters to all saved objects API Signed-off-by: gaobinlong <gbinlong@amazon.com> * feat: update snapshot Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: optimize logic when checkConflict and bulkCreate (#189) * feat: optimize logic when checkConflict and bulkCreate Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: add options.workspace check Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: throw error when workspace check error in repository create Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: modify judgement Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: always get objects from DB when create-with-override Signed-off-by: SuZhou-Joe <suzhou@amazon.com> --------- Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: call get when create with override Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: update test according to count Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: add integration test Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * fix: unit test Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: regenerate ids when import Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: add more unit test Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: minor changes logic on repository Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: update unit test Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: update test Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: optimization according to comments Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: update test Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: optimize code Signed-off-by: SuZhou-Joe <suzhou@amazon.com> --------- Signed-off-by: gaobinlong <gbinlong@amazon.com> Signed-off-by: SuZhou-Joe <suzhou@amazon.com> Co-authored-by: SuZhou-Joe <suzhou@amazon.com>
ruanyl · Mar 18, 2024 · 1d8088e · 1d8088e
1 parent fa68b39
commit 1d8088e
Show file tree

Hide file tree

Showing 17 changed files with 701 additions and 22 deletions.
diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts
@@ -109,7 +109,9 @@ async function fetchObjectsToExport({
     if (typeof search === 'string') {
       throw Boom.badRequest(`Can't specify both "search" and "objects" properties when exporting`);
     }
-    const bulkGetResult = await savedObjectsClient.bulkGet(objects, { namespace });
+    const bulkGetResult = await savedObjectsClient.bulkGet(objects, {
+      namespace,
+    });
     const erroredObjects = bulkGetResult.saved_objects.filter((obj) => !!obj.error);
     if (erroredObjects.length) {
       const err = Boom.badRequest();

diff --git a/src/core/server/saved_objects/import/import_saved_objects.test.ts b/src/core/server/saved_objects/import/import_saved_objects.test.ts
@@ -42,7 +42,7 @@ import { typeRegistryMock } from '../saved_objects_type_registry.mock';
 import { importSavedObjectsFromStream } from './import_saved_objects';
 
 import { collectSavedObjects } from './collect_saved_objects';
-import { regenerateIds } from './regenerate_ids';
+import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
 import { validateReferences } from './validate_references';
 import { checkConflicts } from './check_conflicts';
 import { checkOriginConflicts } from './check_origin_conflicts';
@@ -70,6 +70,7 @@ describe('#importSavedObjectsFromStream', () => {
       importIdMap: new Map(),
     });
     getMockFn(regenerateIds).mockReturnValue(new Map());
+    getMockFn(regenerateIdsWithReference).mockReturnValue(Promise.resolve(new Map()));
     getMockFn(validateReferences).mockResolvedValue([]);
     getMockFn(checkConflicts).mockResolvedValue({
       errors: [],
@@ -278,6 +279,15 @@ describe('#importSavedObjectsFromStream', () => {
           ]),
         });
         getMockFn(validateReferences).mockResolvedValue([errors[1]]);
+        getMockFn(regenerateIdsWithReference).mockResolvedValue(
+          Promise.resolve(
+            new Map([
+              ['foo', {}],
+              ['bar', {}],
+              ['baz', {}],
+            ])
+          )
+        );
         getMockFn(checkConflicts).mockResolvedValue({
           errors: [errors[2]],
           filteredObjects,

diff --git a/src/core/server/saved_objects/import/import_saved_objects.ts b/src/core/server/saved_objects/import/import_saved_objects.ts
@@ -38,7 +38,7 @@ import { validateReferences } from './validate_references';
 import { checkOriginConflicts } from './check_origin_conflicts';
 import { createSavedObjects } from './create_saved_objects';
 import { checkConflicts } from './check_conflicts';
-import { regenerateIds } from './regenerate_ids';
+import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
 import { checkConflictsForDataSource } from './check_conflict_for_data_source';
 
 /**
@@ -87,6 +87,16 @@ export async function importSavedObjectsFromStream({
     importIdMap = regenerateIds(collectSavedObjectsResult.collectedObjects, dataSourceId);
   } else {
     // in check conclict and override mode
+
+    if (workspaces) {
+      importIdMap = await regenerateIdsWithReference({
+        savedObjects: collectSavedObjectsResult.collectedObjects,
+        savedObjectsClient,
+        workspaces,
+        objectLimit,
+        importIdMap,
+      });
+    }
     // Check single-namespace objects for conflicts in this namespace, and check multi-namespace objects for conflicts across all namespaces
     const checkConflictsParams = {
       objects: collectSavedObjectsResult.collectedObjects,

diff --git a/src/core/server/saved_objects/import/regenerate_ids.test.ts b/src/core/server/saved_objects/import/regenerate_ids.test.ts
@@ -29,8 +29,10 @@
  */
 
 import { mockUuidv4 } from './__mocks__';
-import { regenerateIds } from './regenerate_ids';
+import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
 import { SavedObject } from '../types';
+import { savedObjectsClientMock } from '../service/saved_objects_client.mock';
+import { SavedObjectsBulkResponse } from '../service';
 
 describe('#regenerateIds', () => {
   const objects = ([
@@ -68,3 +70,70 @@ describe('#regenerateIds', () => {
     `);
   });
 });
+
+describe('#regenerateIdsWithReference', () => {
+  const objects = ([
+    { type: 'foo', id: '1' },
+    { type: 'bar', id: '2' },
+    { type: 'baz', id: '3' },
+  ] as any) as SavedObject[];
+
+  test('returns expected values', async () => {
+    const mockedSavedObjectsClient = savedObjectsClientMock.create();
+    mockUuidv4.mockReturnValueOnce('uuidv4 #1');
+    const result: SavedObjectsBulkResponse<unknown> = {
+      saved_objects: [
+        {
+          error: {
+            statusCode: 404,
+            error: '',
+            message: '',
+          },
+          id: '1',
+          type: 'foo',
+          attributes: {},
+          references: [],
+        },
+        {
+          id: '2',
+          type: 'bar',
+          attributes: {},
+          references: [],
+          workspaces: ['bar'],
+        },
+        {
+          id: '3',
+          type: 'baz',
+          attributes: {},
+          references: [],
+          workspaces: ['foo'],
+        },
+      ],
+    };
+    mockedSavedObjectsClient.bulkGet.mockResolvedValue(result);
+    expect(
+      await regenerateIdsWithReference({
+        savedObjects: objects,
+        savedObjectsClient: mockedSavedObjectsClient,
+        workspaces: ['bar'],
+        objectLimit: 1000,
+        importIdMap: new Map(),
+      })
+    ).toMatchInlineSnapshot(`
+      Map {
+        "foo:1" => Object {
+          "id": "1",
+          "omitOriginId": true,
+        },
+        "bar:2" => Object {
+          "id": "2",
+          "omitOriginId": false,
+        },
+        "baz:3" => Object {
+          "id": "uuidv4 #1",
+          "omitOriginId": true,
+        },
+      }
+    `);
+  });
+});
diff --git a/src/core/server/saved_objects/import/regenerate_ids.ts b/src/core/server/saved_objects/import/regenerate_ids.ts
@@ -29,7 +29,8 @@
  */
 
 import { v4 as uuidv4 } from 'uuid';
-import { SavedObject } from '../types';
+import { SavedObject, SavedObjectsClientContract } from '../types';
+import { SavedObjectsUtils } from '../service';
 
 /**
  * Takes an array of saved objects and returns an importIdMap of randomly-generated new IDs.
@@ -47,3 +48,35 @@ export const regenerateIds = (objects: SavedObject[], dataSourceId: string | und
   }, new Map<string, { id: string; omitOriginId?: boolean }>());
   return importIdMap;
 };
+
+export const regenerateIdsWithReference = async (props: {
+  savedObjects: SavedObject[];
+  savedObjectsClient: SavedObjectsClientContract;
+  workspaces: string[];
+  objectLimit: number;
+  importIdMap: Map<string, { id?: string; omitOriginId?: boolean }>;
+}): Promise<Map<string, { id?: string; omitOriginId?: boolean }>> => {
+  const { savedObjects, savedObjectsClient, workspaces, importIdMap } = props;
+
+  const bulkGetResult = await savedObjectsClient.bulkGet(
+    savedObjects.map((item) => ({ type: item.type, id: item.id }))
+  );
+
+  return bulkGetResult.saved_objects.reduce((acc, object) => {
+    if (object.error?.statusCode === 404) {
+      acc.set(`${object.type}:${object.id}`, { id: object.id, omitOriginId: true });
+      return acc;
+    }
+
+    const filteredWorkspaces = SavedObjectsUtils.filterWorkspacesAccordingToBaseWorkspaces(
+      workspaces,
+      object.workspaces
+    );
+    if (filteredWorkspaces.length) {
+      acc.set(`${object.type}:${object.id}`, { id: uuidv4(), omitOriginId: true });
+    } else {
+      acc.set(`${object.type}:${object.id}`, { id: object.id, omitOriginId: false });
+    }
+    return acc;
+  }, importIdMap);
+};
diff --git a/...ore/server/saved_objects/migrations/core/__snapshots__/build_active_mappings.test.ts.snap b/...ore/server/saved_objects/migrations/core/__snapshots__/build_active_mappings.test.ts.snap
diff --git a/src/core/server/saved_objects/migrations/core/build_active_mappings.ts b/src/core/server/saved_objects/migrations/core/build_active_mappings.ts
@@ -219,6 +219,9 @@ function defaultMapping(): IndexMapping {
           },
         },
       },
+      workspaces: {
+        type: 'keyword',
+      },
       permissions: {
         properties: {
           read: principals,

diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts
@@ -231,6 +231,7 @@ describe('IndexMigrator', () => {
               references: '7997cf5a56cc02bdc9c93361bde732b0',
               type: '2f4316de49999235636386fe51dc06c1',
               updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+              workspaces: '2f4316de49999235636386fe51dc06c1',
             },
           },
           properties: {
@@ -241,6 +242,9 @@ describe('IndexMigrator', () => {
             originId: { type: 'keyword' },
             type: { type: 'keyword' },
             updated_at: { type: 'date' },
+            workspaces: {
+              type: 'keyword',
+            },
             permissions: {
               properties: {
                 library_read: {
@@ -383,6 +387,7 @@ describe('IndexMigrator', () => {
               references: '7997cf5a56cc02bdc9c93361bde732b0',
               type: '2f4316de49999235636386fe51dc06c1',
               updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+              workspaces: '2f4316de49999235636386fe51dc06c1',
             },
           },
           properties: {
@@ -394,6 +399,9 @@ describe('IndexMigrator', () => {
             originId: { type: 'keyword' },
             type: { type: 'keyword' },
             updated_at: { type: 'date' },
+            workspaces: {
+              type: 'keyword',
+            },
             permissions: {
               properties: {
                 library_read: {
@@ -479,6 +487,7 @@ describe('IndexMigrator', () => {
               references: '7997cf5a56cc02bdc9c93361bde732b0',
               type: '2f4316de49999235636386fe51dc06c1',
               updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+              workspaces: '2f4316de49999235636386fe51dc06c1',
             },
           },
           properties: {
@@ -490,6 +499,9 @@ describe('IndexMigrator', () => {
             originId: { type: 'keyword' },
             type: { type: 'keyword' },
             updated_at: { type: 'date' },
+            workspaces: {
+              type: 'keyword',
+            },
             permissions: {
               properties: {
                 library_read: {

diff --git a/...igrations/opensearch_dashboards/__snapshots__/opensearch_dashboards_migrator.test.ts.snap b/...igrations/opensearch_dashboards/__snapshots__/opensearch_dashboards_migrator.test.ts.snap
diff --git a/src/core/server/saved_objects/routes/integration_tests/find.test.ts b/src/core/server/saved_objects/routes/integration_tests/find.test.ts
@@ -288,4 +288,38 @@ describe('GET /api/saved_objects/_find', () => {
       defaultSearchOperator: 'OR',
     });
   });
+
+  it('accepts the query parameter workspaces as a string', async () => {
+    await supertest(httpSetup.server.listener)
+      .get('/api/saved_objects/_find?type=index-pattern&workspaces=foo')
+      .expect(200);
+
+    expect(savedObjectsClient.find).toHaveBeenCalledTimes(1);
+
+    const options = savedObjectsClient.find.mock.calls[0][0];
+    expect(options).toEqual({
+      defaultSearchOperator: 'OR',
+      perPage: 20,
+      page: 1,
+      type: ['index-pattern'],
+      workspaces: ['foo'],
+    });
+  });
+
+  it('accepts the query parameter workspaces as an array', async () => {
+    await supertest(httpSetup.server.listener)
+      .get('/api/saved_objects/_find?type=index-pattern&workspaces=default&workspaces=foo')
+      .expect(200);
+
+    expect(savedObjectsClient.find).toHaveBeenCalledTimes(1);
+
+    const options = savedObjectsClient.find.mock.calls[0][0];
+    expect(options).toEqual({
+      perPage: 20,
+      page: 1,
+      type: ['index-pattern'],
+      workspaces: ['default', 'foo'],
+      defaultSearchOperator: 'OR',
+    });
+  });
 });
diff --git a/src/core/server/saved_objects/routes/resolve_import_errors.ts b/src/core/server/saved_objects/routes/resolve_import_errors.ts
@@ -135,6 +135,7 @@ export const registerResolveImportErrorsRoute = (router: IRouter, config: SavedO
         workspaces,
         dataSourceId,
         dataSourceTitle,
+        workspaces,
       });
 
       return res.ok({ body: result });