diff --git a/.changeset/grumpy-ligers-notice.md b/.changeset/grumpy-ligers-notice.md new file mode 100644 index 0000000..85ac7f9 --- /dev/null +++ b/.changeset/grumpy-ligers-notice.md @@ -0,0 +1,5 @@ +--- +'@dweber019/backstage-plugin-missing-entity-backend': patch +--- + +Improve batch size to 500, default kind filter remove users and batch cleanup of entities. diff --git a/plugins/missing-entity-backend/README.md b/plugins/missing-entity-backend/README.md index d1f2ee3..db2cb28 100644 --- a/plugins/missing-entity-backend/README.md +++ b/plugins/missing-entity-backend/README.md @@ -42,7 +42,7 @@ missingEntity: seconds: 15 age: days: 3 - batchSize: 2 # Default 20 + batchSize: 2 # Default 500 kindAndType: [ { kind: 'Component' }, { kind: 'Resource', type: 'db' }] # See config.d.ts for default excludeKindAndType: [] # See config.d.ts for default ``` @@ -52,15 +52,18 @@ missingEntity: ### Batch Size The missing entity backend is setup to process entities by acting as a queue where it will pull down all the applicable -entities from the Catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest +entities from the catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest entities that have not been processed to process them. +By default, 500 entities are pull and processed in 60 minutes (see config above). This means for every entity there is +around 7 seconds to get the relations and check them. + ### Refresh The default setup will only check missing entities once when processed. If you want this process to also refresh the data you can do so by adding the `age`. -It's recommended that if you choose to use this configuration to set it to 3 to update stale data. +It's recommended that if you choose to use this configuration to set it to 3 days to update stale data. ## Limitations diff --git a/plugins/missing-entity-backend/config.d.ts b/plugins/missing-entity-backend/config.d.ts index 5e36c67..42878be 100644 --- a/plugins/missing-entity-backend/config.d.ts +++ b/plugins/missing-entity-backend/config.d.ts @@ -6,7 +6,7 @@ export interface Config { missingEntity?: { schedule?: SchedulerServiceTaskScheduleDefinition; /** - * @default 20 + * @default 500 */ batchSize?: number; /** @@ -14,7 +14,7 @@ export interface Config { */ age?: HumanDuration; /** - * @default [{ kind: 'API' }, { kind: 'Component'}, { kind: 'Resource'}, { kind: 'User'}, { kind: 'Group'}, { kind: 'Domain'}, { kind: 'System'}] + * @default [{ kind: 'API' }, { kind: 'Component'}, { kind: 'Resource'}, { kind: 'Group'}, { kind: 'Domain'}, { kind: 'System'}] */ kindAndType?: { kind: string, type?: string }[]; /** diff --git a/plugins/missing-entity-backend/src/api/MissingEntityBackendClient.ts b/plugins/missing-entity-backend/src/api/MissingEntityBackendClient.ts index 0303f46..4439e55 100644 --- a/plugins/missing-entity-backend/src/api/MissingEntityBackendClient.ts +++ b/plugins/missing-entity-backend/src/api/MissingEntityBackendClient.ts @@ -11,7 +11,7 @@ import { Entity, GroupEntity, parseEntityRef, stringifyEntityRef } from '@backst import { assertError } from '@backstage/errors'; import { HumanDuration } from '@backstage/types'; import { type AuthService, LoggerService } from '@backstage/backend-plugin-api'; -import { isEqual } from 'lodash'; +import { isEqual, chunk } from 'lodash'; import { NotificationService } from '@backstage/plugin-notifications-node'; /** @public */ @@ -136,18 +136,30 @@ export class MissingEntityBackendClient implements MissingEntityBackendApi { this.logger?.info('Cleaning entities in missing entity queue'); const allEntities = (await this.store.getAllEntities(false)).map(item => item.entityRef); - for (const entityRef of allEntities) { + const allEntitiesChunks = chunk(allEntities, 500); + + for (const entitiesChunk of allEntitiesChunks) { const { token } = await this.auth.getPluginRequestToken({ onBehalfOf: await this.auth.getOwnServiceCredentials(), targetPluginId: 'catalog', }); - const result = await this.catalogApi.getEntityByRef(entityRef, { token }); + const result = await this.catalogApi.getEntitiesByRefs({ + entityRefs: entitiesChunk, + fields: ['kind', 'metadata', 'spec.type'], + }, { token }); - if (!result) { - this.logger?.info( - `Entity ${entityRef} was not found in the Catalog, it will be deleted`, - ); - await this.store.deleteEntity(entityRef); + for (const [index, item] of result.items.entries()) { + if (item === undefined) { + this.logger?.info( + `Entity ${entitiesChunk[index]} was not found in the Catalog, it will be deleted.`, + ); + await this.store.deleteEntity(entitiesChunk[index]); + } else if (this.isEntityExcluded(item) || !this.isEntityIncluded(item)) { + this.logger?.info( + `Entity ${entitiesChunk[index]} allowed by include or exclude filter, it will be deleted.`, + ); + await this.store.deleteEntity(entitiesChunk[index]); + } } } } @@ -161,7 +173,7 @@ export class MissingEntityBackendClient implements MissingEntityBackendApi { const entities = entitiesOverview.filteredEntities.slice( 0, - this.batchSize ?? 20, + this.batchSize ?? 500, ); for (const entityRef of entities) { @@ -303,7 +315,7 @@ export class MissingEntityBackendClient implements MissingEntityBackendApi { export function kindAndTypeOrDefault(kindAndType?: { kind: string, type?: string }[]): { kind: string, type?: string }[] { if (!kindAndType || kindAndType.length === 0) { - return [{ kind: 'API' }, { kind: 'Component' }, { kind: 'Resource' }, { kind: 'User' }, { kind: 'Group' }, + return [{ kind: 'API' }, { kind: 'Component' }, { kind: 'Resource' }, { kind: 'Group' }, { kind: 'System' }, { kind: 'Domain' }]; } return kindAndType;