feat: improve batch size, default kind filter and clean up of entities

.changeset/grumpy-ligers-notice.md

@@ -0,0 +1,5 @@
+---
+'@dweber019/backstage-plugin-missing-entity-backend': patch
+---
+
+Improve batch size to 500, default kind filter remove users and batch cleanup of entities.

plugins/missing-entity-backend/README.md

@@ -42,7 +42,7 @@ missingEntity:
       seconds: 15
   age:
     days: 3
-  batchSize: 2 # Default 20
+  batchSize: 2 # Default 500
   kindAndType: [ { kind: 'Component' }, { kind: 'Resource', type: 'db' }] # See config.d.ts for default
   excludeKindAndType: [] # See config.d.ts for default
 ```
@@ -52,15 +52,18 @@ missingEntity:
 ### Batch Size
 
 The missing entity backend is setup to process entities by acting as a queue where it will pull down all the applicable
-entities from the Catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest
+entities from the catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest
 entities that have not been processed to process them.
 
+By default, 500 entities are pull and processed in 60 minutes (see config above). This means for every entity there is
+around 7 seconds to get the relations and check them.
+
 ### Refresh
 
 The default setup will only check missing entities once when processed.
 If you want this process to also refresh the data you can do so by adding the `age`.
 
-It's recommended that if you choose to use this configuration to set it to 3 to update stale data.
+It's recommended that if you choose to use this configuration to set it to 3 days to update stale data.
 
 ## Limitations
 

plugins/missing-entity-backend/config.d.ts

@@ -6,15 +6,15 @@ export interface Config {
   missingEntity?: {
     schedule?: SchedulerServiceTaskScheduleDefinition;
     /**
-     * @default 20
+     * @default 500
      */
     batchSize?: number;
     /**
      * Refresh missing entity
      */
     age?: HumanDuration;
     /**
-     * @default [{ kind: 'API' }, { kind: 'Component'}, { kind: 'Resource'}, { kind: 'User'}, { kind: 'Group'}, { kind: 'Domain'}, { kind: 'System'}]
+     * @default [{ kind: 'API' }, { kind: 'Component'}, { kind: 'Resource'}, { kind: 'Group'}, { kind: 'Domain'}, { kind: 'System'}]
      */
     kindAndType?: { kind: string, type?: string }[];
     /**

plugins/missing-entity-backend/src/api/MissingEntityBackendClient.ts

@@ -11,7 +11,7 @@ import { Entity, GroupEntity, parseEntityRef, stringifyEntityRef } from '@backst
 import { assertError } from '@backstage/errors';
 import { HumanDuration } from '@backstage/types';
 import { type AuthService, LoggerService } from '@backstage/backend-plugin-api';
-import { isEqual } from 'lodash';
+import { isEqual, chunk } from 'lodash';
 import { NotificationService } from '@backstage/plugin-notifications-node';
 
 /** @public */
@@ -136,18 +136,30 @@ export class MissingEntityBackendClient implements MissingEntityBackendApi {
     this.logger?.info('Cleaning entities in missing entity queue');
     const allEntities = (await this.store.getAllEntities(false)).map(item => item.entityRef);
 
-    for (const entityRef of allEntities) {
+    const allEntitiesChunks = chunk(allEntities, 500);
+
+    for (const entitiesChunk of allEntitiesChunks) {
       const { token } = await this.auth.getPluginRequestToken({
         onBehalfOf: await this.auth.getOwnServiceCredentials(),
         targetPluginId: 'catalog',
       });
-      const result = await this.catalogApi.getEntityByRef(entityRef, { token });
+      const result = await this.catalogApi.getEntitiesByRefs({
+        entityRefs: entitiesChunk,
+        fields: ['kind', 'metadata', 'spec.type'],
+      }, { token });
 
-      if (!result) {
-        this.logger?.info(
-          `Entity ${entityRef} was not found in the Catalog, it will be deleted`,
-        );
-        await this.store.deleteEntity(entityRef);
+      for (const [index, item] of result.items.entries()) {
+        if (item === undefined) {
+          this.logger?.info(
+            `Entity ${entitiesChunk[index]} was not found in the Catalog, it will be deleted.`,
+          );
+          await this.store.deleteEntity(entitiesChunk[index]);
+        } else if (this.isEntityExcluded(item) || !this.isEntityIncluded(item)) {
+          this.logger?.info(
+            `Entity ${entitiesChunk[index]} allowed by include or exclude filter, it will be deleted.`,
+          );
+          await this.store.deleteEntity(entitiesChunk[index]);
+        }
       }
     }
   }
@@ -161,7 +173,7 @@ export class MissingEntityBackendClient implements MissingEntityBackendApi {
 
     const entities = entitiesOverview.filteredEntities.slice(
       0,
-      this.batchSize ?? 20,
+      this.batchSize ?? 500,
     );
 
     for (const entityRef of entities) {
@@ -303,7 +315,7 @@ export class MissingEntityBackendClient implements MissingEntityBackendApi {
 
 export function kindAndTypeOrDefault(kindAndType?: { kind: string, type?: string }[]): { kind: string, type?: string }[] {
   if (!kindAndType || kindAndType.length === 0) {
-    return [{ kind: 'API' }, { kind: 'Component' }, { kind: 'Resource' }, { kind: 'User' }, { kind: 'Group' },
+    return [{ kind: 'API' }, { kind: 'Component' }, { kind: 'Resource' }, { kind: 'Group' },
       { kind: 'System' }, { kind: 'Domain' }];
   }
   return kindAndType;