Skip to content

Commit

Permalink
[Workspace] Add optional workspaces parameter to all saved objects A…
Browse files Browse the repository at this point in the history
…PI (#185)

* [Workspace] Add workspaces parameters to all saved objects API

Signed-off-by: gaobinlong <gbinlong@amazon.com>

* feat: update snapshot

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: optimize logic when checkConflict and bulkCreate (#189)

* feat: optimize logic when checkConflict and bulkCreate

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: add options.workspace check

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: throw error when workspace check error in repository create

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: modify judgement

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: always get objects from DB when create-with-override

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

---------

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: call get when create with override

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update test according to count

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: add integration test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* fix: unit test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: regenerate ids when import

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: add more unit test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: minor changes logic on repository

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update unit test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: optimization according to comments

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: optimize code

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

---------

Signed-off-by: gaobinlong <gbinlong@amazon.com>
Signed-off-by: SuZhou-Joe <suzhou@amazon.com>
Co-authored-by: SuZhou-Joe <suzhou@amazon.com>
  • Loading branch information
gaobinlong and SuZhou-Joe authored Sep 28, 2023
1 parent 5bc2ddf commit c05b434
Show file tree
Hide file tree
Showing 33 changed files with 939 additions and 29 deletions.
1 change: 1 addition & 0 deletions src/core/public/saved_objects/saved_objects_client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,7 @@ export class SavedObjectsClient {
filter: 'filter',
namespaces: 'namespaces',
preference: 'preference',
workspaces: 'workspaces',
};

const renamedQuery = renameKeys<SavedObjectsFindOptions, any>(renameMap, options);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ export interface SavedObjectsExportOptions {
excludeExportDetails?: boolean;
/** optional namespace to override the namespace used by the savedObjectsClient. */
namespace?: string;
/** optional workspaces to override the workspaces used by the savedObjectsClient. */
workspaces?: string[];
}

/**
Expand Down Expand Up @@ -87,13 +89,15 @@ async function fetchObjectsToExport({
exportSizeLimit,
savedObjectsClient,
namespace,
workspaces,
}: {
objects?: SavedObjectsExportOptions['objects'];
types?: string[];
search?: string;
exportSizeLimit: number;
savedObjectsClient: SavedObjectsClientContract;
namespace?: string;
workspaces?: string[];
}) {
if ((types?.length ?? 0) > 0 && (objects?.length ?? 0) > 0) {
throw Boom.badRequest(`Can't specify both "types" and "objects" properties when exporting`);
Expand All @@ -105,7 +109,9 @@ async function fetchObjectsToExport({
if (typeof search === 'string') {
throw Boom.badRequest(`Can't specify both "search" and "objects" properties when exporting`);
}
const bulkGetResult = await savedObjectsClient.bulkGet(objects, { namespace });
const bulkGetResult = await savedObjectsClient.bulkGet(objects, {
namespace,
});
const erroredObjects = bulkGetResult.saved_objects.filter((obj) => !!obj.error);
if (erroredObjects.length) {
const err = Boom.badRequest();
Expand All @@ -121,6 +127,7 @@ async function fetchObjectsToExport({
search,
perPage: exportSizeLimit,
namespaces: namespace ? [namespace] : undefined,
...(workspaces ? { workspaces } : {}),
});
if (findResponse.total > exportSizeLimit) {
throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
Expand Down Expand Up @@ -153,6 +160,7 @@ export async function exportSavedObjectsToStream({
includeReferencesDeep = false,
excludeExportDetails = false,
namespace,
workspaces,
}: SavedObjectsExportOptions) {
const rootObjects = await fetchObjectsToExport({
types,
Expand All @@ -161,6 +169,7 @@ export async function exportSavedObjectsToStream({
savedObjectsClient,
exportSizeLimit,
namespace,
workspaces,
});
let exportedObjects: Array<SavedObject<unknown>> = [];
let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = [];
Expand Down
3 changes: 3 additions & 0 deletions src/core/server/saved_objects/import/check_conflicts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ interface CheckConflictsParams {
ignoreRegularConflicts?: boolean;
retries?: SavedObjectsImportRetry[];
createNewCopies?: boolean;
workspaces?: string[];
}

const isUnresolvableConflict = (error: SavedObjectError) =>
Expand All @@ -56,6 +57,7 @@ export async function checkConflicts({
ignoreRegularConflicts,
retries = [],
createNewCopies,
workspaces,
}: CheckConflictsParams) {
const filteredObjects: Array<SavedObject<{ title?: string }>> = [];
const errors: SavedObjectsImportError[] = [];
Expand All @@ -77,6 +79,7 @@ export async function checkConflicts({
});
const checkConflictsResult = await savedObjectsClient.checkConflicts(objectsToCheck, {
namespace,
workspaces,
});
const errorMap = checkConflictsResult.errors.reduce(
(acc, { type, id, error }) => acc.set(`${type}:${id}`, error),
Expand Down
3 changes: 3 additions & 0 deletions src/core/server/saved_objects/import/create_saved_objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ interface CreateSavedObjectsParams<T> {
importIdMap: Map<string, { id?: string; omitOriginId?: boolean }>;
namespace?: string;
overwrite?: boolean;
workspaces?: string[];
}
interface CreateSavedObjectsResult<T> {
createdObjects: Array<CreatedObject<T>>;
Expand All @@ -56,6 +57,7 @@ export const createSavedObjects = async <T>({
importIdMap,
namespace,
overwrite,
workspaces,
}: CreateSavedObjectsParams<T>): Promise<CreateSavedObjectsResult<T>> => {
// filter out any objects that resulted in errors
const errorSet = accumulatedErrors.reduce(
Expand Down Expand Up @@ -103,6 +105,7 @@ export const createSavedObjects = async <T>({
const bulkCreateResponse = await savedObjectsClient.bulkCreate(objectsToCreate, {
namespace,
overwrite,
workspaces,
});
expectedResults = bulkCreateResponse.saved_objects;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import { typeRegistryMock } from '../saved_objects_type_registry.mock';
import { importSavedObjectsFromStream } from './import_saved_objects';

import { collectSavedObjects } from './collect_saved_objects';
import { regenerateIds } from './regenerate_ids';
import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
import { validateReferences } from './validate_references';
import { checkConflicts } from './check_conflicts';
import { checkOriginConflicts } from './check_origin_conflicts';
Expand All @@ -68,6 +68,7 @@ describe('#importSavedObjectsFromStream', () => {
importIdMap: new Map(),
});
getMockFn(regenerateIds).mockReturnValue(new Map());
getMockFn(regenerateIdsWithReference).mockReturnValue(Promise.resolve(new Map()));
getMockFn(validateReferences).mockResolvedValue([]);
getMockFn(checkConflicts).mockResolvedValue({
errors: [],
Expand Down Expand Up @@ -240,6 +241,15 @@ describe('#importSavedObjectsFromStream', () => {
]),
});
getMockFn(validateReferences).mockResolvedValue([errors[1]]);
getMockFn(regenerateIdsWithReference).mockResolvedValue(
Promise.resolve(
new Map([
['foo', {}],
['bar', {}],
['baz', {}],
])
)
);
getMockFn(checkConflicts).mockResolvedValue({
errors: [errors[2]],
filteredObjects,
Expand Down
14 changes: 13 additions & 1 deletion src/core/server/saved_objects/import/import_saved_objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import { validateReferences } from './validate_references';
import { checkOriginConflicts } from './check_origin_conflicts';
import { createSavedObjects } from './create_saved_objects';
import { checkConflicts } from './check_conflicts';
import { regenerateIds } from './regenerate_ids';
import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';

/**
* Import saved objects from given stream. See the {@link SavedObjectsImportOptions | options} for more
Expand All @@ -54,6 +54,7 @@ export async function importSavedObjectsFromStream({
savedObjectsClient,
typeRegistry,
namespace,
workspaces,
}: SavedObjectsImportOptions): Promise<SavedObjectsImportResponse> {
let errorAccumulator: SavedObjectsImportError[] = [];
const supportedTypes = typeRegistry.getImportableAndExportableTypes().map((type) => type.name);
Expand All @@ -80,12 +81,22 @@ export async function importSavedObjectsFromStream({
if (createNewCopies) {
importIdMap = regenerateIds(collectSavedObjectsResult.collectedObjects);
} else {
if (workspaces) {
importIdMap = await regenerateIdsWithReference({
savedObjects: collectSavedObjectsResult.collectedObjects,
savedObjectsClient,
workspaces,
objectLimit,
importIdMap,
});
}
// Check single-namespace objects for conflicts in this namespace, and check multi-namespace objects for conflicts across all namespaces
const checkConflictsParams = {
objects: collectSavedObjectsResult.collectedObjects,
savedObjectsClient,
namespace,
ignoreRegularConflicts: overwrite,
workspaces,
};
const checkConflictsResult = await checkConflicts(checkConflictsParams);
errorAccumulator = [...errorAccumulator, ...checkConflictsResult.errors];
Expand Down Expand Up @@ -118,6 +129,7 @@ export async function importSavedObjectsFromStream({
importIdMap,
overwrite,
namespace,
...(workspaces ? { workspaces } : {}),
};
const createSavedObjectsResult = await createSavedObjects(createSavedObjectsParams);
errorAccumulator = [...errorAccumulator, ...createSavedObjectsResult.errors];
Expand Down
71 changes: 70 additions & 1 deletion src/core/server/saved_objects/import/regenerate_ids.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@
*/

import { mockUuidv4 } from './__mocks__';
import { regenerateIds } from './regenerate_ids';
import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
import { SavedObject } from '../types';
import { savedObjectsClientMock } from '../service/saved_objects_client.mock';
import { SavedObjectsBulkResponse } from '../service';

describe('#regenerateIds', () => {
const objects = ([
Expand Down Expand Up @@ -62,3 +64,70 @@ describe('#regenerateIds', () => {
`);
});
});

describe('#regenerateIdsWithReference', () => {
const objects = ([
{ type: 'foo', id: '1' },
{ type: 'bar', id: '2' },
{ type: 'baz', id: '3' },
] as any) as SavedObject[];

test('returns expected values', async () => {
const mockedSavedObjectsClient = savedObjectsClientMock.create();
mockUuidv4.mockReturnValueOnce('uuidv4 #1');
const result: SavedObjectsBulkResponse<unknown> = {
saved_objects: [
{
error: {
statusCode: 404,
error: '',
message: '',
},
id: '1',
type: 'foo',
attributes: {},
references: [],
},
{
id: '2',
type: 'bar',
attributes: {},
references: [],
workspaces: ['bar'],
},
{
id: '3',
type: 'baz',
attributes: {},
references: [],
workspaces: ['foo'],
},
],
};
mockedSavedObjectsClient.bulkGet.mockResolvedValue(result);
expect(
await regenerateIdsWithReference({
savedObjects: objects,
savedObjectsClient: mockedSavedObjectsClient,
workspaces: ['bar'],
objectLimit: 1000,
importIdMap: new Map(),
})
).toMatchInlineSnapshot(`
Map {
"foo:1" => Object {
"id": "1",
"omitOriginId": true,
},
"bar:2" => Object {
"id": "2",
"omitOriginId": false,
},
"baz:3" => Object {
"id": "uuidv4 #1",
"omitOriginId": true,
},
}
`);
});
});
35 changes: 34 additions & 1 deletion src/core/server/saved_objects/import/regenerate_ids.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@
*/

import { v4 as uuidv4 } from 'uuid';
import { SavedObject } from '../types';
import { SavedObject, SavedObjectsClientContract } from '../types';
import { SavedObjectsUtils } from '../service';

/**
* Takes an array of saved objects and returns an importIdMap of randomly-generated new IDs.
Expand All @@ -42,3 +43,35 @@ export const regenerateIds = (objects: SavedObject[]) => {
}, new Map<string, { id: string; omitOriginId?: boolean }>());
return importIdMap;
};

export const regenerateIdsWithReference = async (props: {
savedObjects: SavedObject[];
savedObjectsClient: SavedObjectsClientContract;
workspaces: string[];
objectLimit: number;
importIdMap: Map<string, { id?: string; omitOriginId?: boolean }>;
}): Promise<Map<string, { id?: string; omitOriginId?: boolean }>> => {
const { savedObjects, savedObjectsClient, workspaces, importIdMap } = props;

const bulkGetResult = await savedObjectsClient.bulkGet(
savedObjects.map((item) => ({ type: item.type, id: item.id }))
);

return bulkGetResult.saved_objects.reduce((acc, object) => {
if (object.error?.statusCode === 404) {
acc.set(`${object.type}:${object.id}`, { id: object.id, omitOriginId: true });
return acc;
}

const filteredWorkspaces = SavedObjectsUtils.filterWorkspacesAccordingToBaseWorkspaces(
workspaces,
object.workspaces
);
if (filteredWorkspaces.length) {
acc.set(`${object.type}:${object.id}`, { id: uuidv4(), omitOriginId: true });
} else {
acc.set(`${object.type}:${object.id}`, { id: object.id, omitOriginId: false });
}
return acc;
}, importIdMap);
};
2 changes: 2 additions & 0 deletions src/core/server/saved_objects/import/resolve_import_errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export async function resolveSavedObjectsImportErrors({
typeRegistry,
namespace,
createNewCopies,
workspaces,
}: SavedObjectsResolveImportErrorsOptions): Promise<SavedObjectsImportResponse> {
// throw a BadRequest error if we see invalid retries
validateRetries(retries);
Expand Down Expand Up @@ -157,6 +158,7 @@ export async function resolveSavedObjectsImportErrors({
importIdMap,
namespace,
overwrite,
workspaces,
};
const { createdObjects, errors: bulkCreateErrors } = await createSavedObjects(
createSavedObjectsParams
Expand Down
4 changes: 4 additions & 0 deletions src/core/server/saved_objects/import/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ export interface SavedObjectsImportOptions {
namespace?: string;
/** If true, will create new copies of import objects, each with a random `id` and undefined `originId`. */
createNewCopies: boolean;
/** if specified, will import in given workspaces, else will import as global object */
workspaces?: string[];
}

/**
Expand All @@ -208,6 +210,8 @@ export interface SavedObjectsResolveImportErrorsOptions {
namespace?: string;
/** If true, will create new copies of import objects, each with a random `id` and undefined `originId`. */
createNewCopies: boolean;
/** if specified, will import in given workspaces, else will import as global object */
workspaces?: string[];
}

export type CreatedObject<T> = SavedObject<T> & { destinationId?: string };

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit c05b434

Please sign in to comment.