From 12e6c06b3f3dfc44f4d1f6bd315f2edab869927b Mon Sep 17 00:00:00 2001 From: jue-henry Date: Mon, 21 Oct 2024 14:12:00 -0700 Subject: [PATCH 1/6] Preliminary Implementation --- src/api/db/models/Image.ts | 20 ++++++++++++++++++++ src/api/db/schemas/Task.ts | 1 + src/task/handler.ts | 3 +++ src/task/image.ts | 9 +++++++++ 4 files changed, 33 insertions(+) create mode 100644 src/task/image.ts diff --git a/src/api/db/models/Image.ts b/src/api/db/models/Image.ts index 8f402177..2e98cb94 100644 --- a/src/api/db/models/Image.ts +++ b/src/api/db/models/Image.ts @@ -1051,6 +1051,26 @@ export class ImageModel { } } + static async deleteImagesTask( + input: gql.DeleteImagesInput, + context: Pick, + ): Promise> { + try { + return TaskModel.create( + { + type: 'DeleteImages', + projectId: context.user['curr_project'], + user: context.user['cognito:username'], + config: input, + }, + context, + ); + } catch (err) { + if (err instanceof GraphQLError) throw err; + throw new InternalServerError(err as string); + } + } + /** * A custom middleware-like method that is used to update the reviewed status of * images that should only be ran by operations that would affect the reviewed status. diff --git a/src/api/db/schemas/Task.ts b/src/api/db/schemas/Task.ts index a132d19f..ce15804f 100644 --- a/src/api/db/schemas/Task.ts +++ b/src/api/db/schemas/Task.ts @@ -18,6 +18,7 @@ const TaskSchema = new Schema({ 'UpdateDeployment', 'DeleteDeployment', 'UpdateSerialNumber', + 'DeleteImages', ], }, status: { diff --git a/src/task/handler.ts b/src/task/handler.ts index 42b34346..c1a4066c 100644 --- a/src/task/handler.ts +++ b/src/task/handler.ts @@ -11,6 +11,7 @@ import { parseMessage } from './utils.js'; import { type TaskInput } from '../api/db/models/Task.js'; import GraphQLError, { InternalServerError } from '../api/errors.js'; import { type User } from '../api/auth/authorization.js'; +import { DeleteImages } from './image.js'; async function handler(event: SQSEvent) { if (!event.Records || !event.Records.length) return; @@ -42,6 +43,8 @@ async function handler(event: SQSEvent) { output = await DeleteDeployment(task); } else if (task.type === 'UpdateSerialNumber') { output = await UpdateSerialNumber(task); + } else if (task.type === 'DeleteImages') { + output = await DeleteImages(task); } else { throw new Error(`Unknown Task: ${JSON.stringify(task)}`); } diff --git a/src/task/image.ts b/src/task/image.ts new file mode 100644 index 00000000..956b86cb --- /dev/null +++ b/src/task/image.ts @@ -0,0 +1,9 @@ +import { type User } from '../api/auth/authorization.js'; +import { ImageModel } from '../api/db/models/Image.js'; +import { TaskInput } from '../api/db/models/Task.js'; +import type * as gql from '../@types/graphql.js'; + +export async function DeleteImages(task: TaskInput) { + const context = { user: { is_superuser: true, curr_project: task.projectId } as User }; + return await ImageModel.deleteImages(task.config, context); +} From c5d258a3f52f6a354042f86352b78ef5ccfa1274 Mon Sep 17 00:00:00 2001 From: jue-henry Date: Wed, 23 Oct 2024 15:54:22 -0700 Subject: [PATCH 2/6] create delete images by filter task --- src/@types/graphql.ts | 4 +++ .../inputs/DeleteImagesByFilterTaskInput.ts | 5 ++++ src/task/handler.ts | 4 ++- src/task/image.ts | 27 ++++++++++++++++++- 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 src/api/type-defs/inputs/DeleteImagesByFilterTaskInput.ts diff --git a/src/@types/graphql.ts b/src/@types/graphql.ts index 51a06522..2235cd74 100644 --- a/src/@types/graphql.ts +++ b/src/@types/graphql.ts @@ -256,6 +256,10 @@ export type DeleteImageCommentInput = { imageId: Scalars['ID']['input']; }; +export type DeleteImagesByFilterTaskInput = { + filters: FiltersInput; +}; + export type DeleteImagesInput = { imageIds?: InputMaybe>; }; diff --git a/src/api/type-defs/inputs/DeleteImagesByFilterTaskInput.ts b/src/api/type-defs/inputs/DeleteImagesByFilterTaskInput.ts new file mode 100644 index 00000000..637c70ac --- /dev/null +++ b/src/api/type-defs/inputs/DeleteImagesByFilterTaskInput.ts @@ -0,0 +1,5 @@ +export default /* GraphQL */ ` + input DeleteImagesByFilterTaskInput { + filters: FiltersInput! + } +`; diff --git a/src/task/handler.ts b/src/task/handler.ts index c1a4066c..84ea05be 100644 --- a/src/task/handler.ts +++ b/src/task/handler.ts @@ -11,7 +11,7 @@ import { parseMessage } from './utils.js'; import { type TaskInput } from '../api/db/models/Task.js'; import GraphQLError, { InternalServerError } from '../api/errors.js'; import { type User } from '../api/auth/authorization.js'; -import { DeleteImages } from './image.js'; +import { DeleteImages, DeleteImagesByFilter } from './image.js'; async function handler(event: SQSEvent) { if (!event.Records || !event.Records.length) return; @@ -45,6 +45,8 @@ async function handler(event: SQSEvent) { output = await UpdateSerialNumber(task); } else if (task.type === 'DeleteImages') { output = await DeleteImages(task); + } else if (task.type === 'DeleteImagesByFilter') { + output = await DeleteImagesByFilter(task); } else { throw new Error(`Unknown Task: ${JSON.stringify(task)}`); } diff --git a/src/task/image.ts b/src/task/image.ts index 956b86cb..42ce9a9e 100644 --- a/src/task/image.ts +++ b/src/task/image.ts @@ -3,7 +3,32 @@ import { ImageModel } from '../api/db/models/Image.js'; import { TaskInput } from '../api/db/models/Task.js'; import type * as gql from '../@types/graphql.js'; +export async function DeleteImagesByFilter(task: TaskInput) { + const context = { user: { is_superuser: true, curr_project: task.projectId } as User }; + let images = await ImageModel.queryByFilter( + { filters: task.config.filters, limit: 100 }, + context, + ); + while (images.results.length > 0) { + await ImageModel.deleteImages({ imageIds: images.results.map((image) => image._id) }, context); + if (images.hasNext) { + images = await ImageModel.queryByFilter( + { filters: task.config.filters, limit: 100, next: images.next }, + context, + ); + } else { + break; + } + } + + return { isOk: true }; +} + export async function DeleteImages(task: TaskInput) { const context = { user: { is_superuser: true, curr_project: task.projectId } as User }; - return await ImageModel.deleteImages(task.config, context); + while (task.config.imageIds?.length && task.config.imageIds.length > 0) { + const batch = task.config.imageIds?.splice(0, 100); + await ImageModel.deleteImages({ imageIds: batch }, context); + } + return { isOk: true }; } From 740ee815aa6fa4ca2348b18062e7b79fffc9a1e8 Mon Sep 17 00:00:00 2001 From: jue-henry Date: Mon, 28 Oct 2024 15:59:13 -0700 Subject: [PATCH 3/6] adding deletion mutations --- src/@types/graphql.ts | 16 ++++++++++ src/api/db/models/Image.ts | 30 +++++++++++++++++++ src/api/db/schemas/Task.ts | 1 + src/api/resolvers/Mutation.ts | 16 ++++++++++ .../inputs/DeleteImagesByFilterInput.ts | 5 ++++ src/api/type-defs/root/Mutation.ts | 2 ++ src/task/image.ts | 3 +- 7 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 src/api/type-defs/inputs/DeleteImagesByFilterInput.ts diff --git a/src/@types/graphql.ts b/src/@types/graphql.ts index 2235cd74..2c8ab580 100644 --- a/src/@types/graphql.ts +++ b/src/@types/graphql.ts @@ -256,6 +256,10 @@ export type DeleteImageCommentInput = { imageId: Scalars['ID']['input']; }; +export type DeleteImagesByFilterInput = { + filters: FiltersInput; +}; + export type DeleteImagesByFilterTaskInput = { filters: FiltersInput; }; @@ -574,6 +578,8 @@ export type Mutation = { deleteDeployment?: Maybe; deleteImageComment?: Maybe; deleteImages?: Maybe; + deleteImagesByFilterTask?: Maybe; + deleteImagesTask?: Maybe; deleteLabels?: Maybe; deleteObjects?: Maybe; deleteProjectLabel?: Maybe; @@ -691,6 +697,16 @@ export type MutationDeleteImagesArgs = { }; +export type MutationDeleteImagesByFilterTaskArgs = { + input: DeleteImagesByFilterTaskInput; +}; + + +export type MutationDeleteImagesTaskArgs = { + input: DeleteImagesInput; +}; + + export type MutationDeleteLabelsArgs = { input: DeleteLabelsInput; }; diff --git a/src/api/db/models/Image.ts b/src/api/db/models/Image.ts index 2e98cb94..7c8ef55b 100644 --- a/src/api/db/models/Image.ts +++ b/src/api/db/models/Image.ts @@ -1071,6 +1071,26 @@ export class ImageModel { } } + static async deleteImagesByFilterTask( + input: gql.DeleteImagesByFilterTaskInput, + context: Pick, + ): Promise> { + try { + return TaskModel.create( + { + type: 'DeleteImagesByFilter', + projectId: context.user['curr_project'], + user: context.user['cognito:username'], + config: input, + }, + context, + ); + } catch (err) { + if (err instanceof GraphQLError) throw err; + throw new InternalServerError(err as string); + } + } + /** * A custom middleware-like method that is used to update the reviewed status of * images that should only be ran by operations that would affect the reviewed status. @@ -1158,6 +1178,16 @@ export default class AuthedImageModel extends BaseAuthedModel { return ImageModel.deleteImages(...args); } + @roleCheck(DELETE_IMAGES_ROLES) + deleteImagesTask(...args: MethodParams) { + return ImageModel.deleteImagesTask(...args); + } + + @roleCheck(DELETE_IMAGES_ROLES) + deleteImagesByFilterTask(...args: MethodParams) { + return ImageModel.deleteImagesByFilterTask(...args); + } + @roleCheck(WRITE_IMAGES_ROLES) createImage(...args: MethodParams) { return ImageModel.createImage(...args); diff --git a/src/api/db/schemas/Task.ts b/src/api/db/schemas/Task.ts index ce15804f..4a0bb6d6 100644 --- a/src/api/db/schemas/Task.ts +++ b/src/api/db/schemas/Task.ts @@ -19,6 +19,7 @@ const TaskSchema = new Schema({ 'DeleteDeployment', 'UpdateSerialNumber', 'DeleteImages', + 'DeleteImagesByFilter', ], }, status: { diff --git a/src/api/resolvers/Mutation.ts b/src/api/resolvers/Mutation.ts index b7d7f90a..94708d5b 100644 --- a/src/api/resolvers/Mutation.ts +++ b/src/api/resolvers/Mutation.ts @@ -132,6 +132,22 @@ export default { return context.models.Image.deleteImages(input, context); }, + deleteImagesTask: async ( + _: unknown, + { input }: gql.MutationDeleteImagesArgs, + context: Context, + ): Promise => { + return context.models.Image.deleteImagesTask(input, context); + }, + + deleteImagesByFilterTask: async ( + _: unknown, + { input }: gql.MutationDeleteImagesByFilterTaskArgs, + context: Context, + ): Promise => { + return context.models.Image.deleteImagesByFilterTask(input, context); + }, + registerCamera: async ( _: unknown, { input }: gql.MutationRegisterCameraArgs, diff --git a/src/api/type-defs/inputs/DeleteImagesByFilterInput.ts b/src/api/type-defs/inputs/DeleteImagesByFilterInput.ts new file mode 100644 index 00000000..6ab6e521 --- /dev/null +++ b/src/api/type-defs/inputs/DeleteImagesByFilterInput.ts @@ -0,0 +1,5 @@ +export default /* GraphQL */ ` + input DeleteImagesByFilterInput { + filters: FiltersInput! + } +`; diff --git a/src/api/type-defs/root/Mutation.ts b/src/api/type-defs/root/Mutation.ts index 2aa2c40f..201d76de 100644 --- a/src/api/type-defs/root/Mutation.ts +++ b/src/api/type-defs/root/Mutation.ts @@ -2,6 +2,8 @@ export default /* GraphQL */ ` type Mutation { createImage(input: CreateImageInput!): CreateImagePayload deleteImages(input: DeleteImagesInput!): StandardErrorPayload + deleteImagesTask(input: DeleteImagesInput!): Task + deleteImagesByFilterTask(input: DeleteImagesByFilterTaskInput!): Task createImageComment(input: CreateImageCommentInput!): ImageCommentsPayload updateImageComment(input: UpdateImageCommentInput!): ImageCommentsPayload diff --git a/src/task/image.ts b/src/task/image.ts index 42ce9a9e..ef85a172 100644 --- a/src/task/image.ts +++ b/src/task/image.ts @@ -10,7 +10,8 @@ export async function DeleteImagesByFilter(task: TaskInput 0) { - await ImageModel.deleteImages({ imageIds: images.results.map((image) => image._id) }, context); + const batch = images.results.map((image) => image._id); + await ImageModel.deleteImages({ imageIds: batch }, context); if (images.hasNext) { images = await ImageModel.queryByFilter( { filters: task.config.filters, limit: 100, next: images.next }, From a51696ce1dccd332a53e823dbcbd00a471424813 Mon Sep 17 00:00:00 2001 From: jue-henry Date: Thu, 31 Oct 2024 17:35:55 -0700 Subject: [PATCH 4/6] adding outputs --- src/task/image.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/task/image.ts b/src/task/image.ts index ef85a172..9a3611d8 100644 --- a/src/task/image.ts +++ b/src/task/image.ts @@ -22,14 +22,15 @@ export async function DeleteImagesByFilter(task: TaskInput) { const context = { user: { is_superuser: true, curr_project: task.projectId } as User }; + const output = task.config.imageIds?.slice(); while (task.config.imageIds?.length && task.config.imageIds.length > 0) { const batch = task.config.imageIds?.splice(0, 100); await ImageModel.deleteImages({ imageIds: batch }, context); } - return { isOk: true }; + return { imageIds: output }; } From ccb215094924248279a157cd0636642aae486f2c Mon Sep 17 00:00:00 2001 From: jue-henry Date: Mon, 4 Nov 2024 11:07:28 -0800 Subject: [PATCH 5/6] adding docs and cleaing up nullish logic --- src/task/image.ts | 20 ++++++++++++++++---- src/task/stats.ts | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/task/image.ts b/src/task/image.ts index 9a3611d8..dfabcc1d 100644 --- a/src/task/image.ts +++ b/src/task/image.ts @@ -4,6 +4,12 @@ import { TaskInput } from '../api/db/models/Task.js'; import type * as gql from '../@types/graphql.js'; export async function DeleteImagesByFilter(task: TaskInput) { + /** + * Deletes images that match the inputted filters in batches of 100. + * This is used by the frontend to delete all images currently shown. + * * @param {Object} input + * * @param {gql.FiltersInput} input.config.filters + */ const context = { user: { is_superuser: true, curr_project: task.projectId } as User }; let images = await ImageModel.queryByFilter( { filters: task.config.filters, limit: 100 }, @@ -26,11 +32,17 @@ export async function DeleteImagesByFilter(task: TaskInput) { + /** + * Deletes a list of images by their IDs in batches of 100. + * This is used by the frontend when the user is selecting more than 100 images to delete to delete at once. + * * @param {Object} input + * * @param {String[]} input.config.imageIds + */ const context = { user: { is_superuser: true, curr_project: task.projectId } as User }; - const output = task.config.imageIds?.slice(); - while (task.config.imageIds?.length && task.config.imageIds.length > 0) { - const batch = task.config.imageIds?.splice(0, 100); + const imagesToDelete = task.config.imageIds?.slice() ?? []; + while (imagesToDelete.length > 0) { + const batch = imagesToDelete.splice(0, 100); await ImageModel.deleteImages({ imageIds: batch }, context); } - return { imageIds: output }; + return { imageIds: task.config.imageIds }; } diff --git a/src/task/stats.ts b/src/task/stats.ts index 2a43100a..b16eabe7 100644 --- a/src/task/stats.ts +++ b/src/task/stats.ts @@ -1,5 +1,5 @@ import { ProjectModel } from '../api/db/models/Project.js'; -import { buildPipeline, isImageReviewed, idMatch } from '../api/db/models/utils.js'; +import { buildPipeline, idMatch } from '../api/db/models/utils.js'; import Image, { type ImageSchema } from '../api/db/schemas/Image.js'; import _ from 'lodash'; import { type TaskInput } from '../api/db/models/Task.js'; From ee5a9b42d8c50ea393f2a2ec93268740abc1f04e Mon Sep 17 00:00:00 2001 From: jue-henry Date: Mon, 4 Nov 2024 11:48:59 -0800 Subject: [PATCH 6/6] adding docs --- src/api/db/models/Image.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/api/db/models/Image.ts b/src/api/db/models/Image.ts index 7c8ef55b..140afe6b 100644 --- a/src/api/db/models/Image.ts +++ b/src/api/db/models/Image.ts @@ -1051,6 +1051,10 @@ export class ImageModel { } } + /** + * Used by the frontend when the user manually selects and deletes more than + * 100 images at once + */ static async deleteImagesTask( input: gql.DeleteImagesInput, context: Pick, @@ -1071,6 +1075,9 @@ export class ImageModel { } } + /** + * Used by the frontend to delete all currently filtered images + */ static async deleteImagesByFilterTask( input: gql.DeleteImagesByFilterTaskInput, context: Pick,