From 489f47dfb7e539e6bd044002f2cd7330dd41d9f8 Mon Sep 17 00:00:00 2001 From: Daniel Searle Date: Mon, 21 Oct 2024 11:14:44 +0100 Subject: [PATCH 1/4] feat(CB2-14255): snowball sqs --- src/functions/reportGen.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/functions/reportGen.ts b/src/functions/reportGen.ts index f7bd0f4..b2ac9bc 100644 --- a/src/functions/reportGen.ts +++ b/src/functions/reportGen.ts @@ -1,7 +1,7 @@ import { LambdaClient } from "@aws-sdk/client-lambda"; import { PutObjectRequest } from "@aws-sdk/client-s3"; import { unmarshall } from "@aws-sdk/util-dynamodb"; -import { Callback, Context, Handler } from "aws-lambda"; +import { Callback, Context, DynamoDBBatchItemFailure, DynamoDBBatchResponse, Handler } from "aws-lambda"; import { ERRORS } from "../assets/enum"; import { ActivitiesService } from "../services/ActivitiesService"; import { LambdaService } from "../services/LambdaService"; @@ -16,18 +16,20 @@ import { ActivitySchema } from "@dvsa/cvs-type-definitions/types/v1/activity"; * @param context - λ Context * @param callback - callback function */ -const reportGen: Handler = async (event: any, context?: Context, callback?: Callback): Promise => { +const reportGen: Handler = async (event: any, context?: Context, callback?: Callback): Promise => { if (!event || !event.Records || !Array.isArray(event.Records) || !event.Records.length) { console.error("ERROR: event is not defined."); throw new Error(ERRORS.EVENT_IS_EMPTY); } + const batchItemFailures: DynamoDBBatchItemFailure[] = []; + const lambdaService = new LambdaService(new LambdaClient({})); const reportService: ReportGenerationService = new ReportGenerationService(new TestResultsService(lambdaService), new ActivitiesService(lambdaService)); const sendATFReport: SendATFReport = new SendATFReport(); console.debug("Services injected, looping over sqs events"); - try { - for (const record of event.Records) { + for (const record of event.Records) { + try { const recordBody = JSON.parse(record?.body); const visit: ActivitySchema = unmarshall(recordBody?.dynamodb.NewImage) as ActivitySchema; @@ -39,11 +41,15 @@ const reportGen: Handler = async (event: any, context?: Context, callback?: Call await sendATFReport.sendATFReport(generationServiceResponse, visit); console.debug("All emails sent, terminating lambda"); } + + } catch (error) { + console.error(error); + batchItemFailures.push({ + itemIdentifier: record.dynamodb?.SequenceNumber ?? "", + }); } - } catch(error) { - console.error(error); - throw error; } + return { batchItemFailures }; }; export { reportGen }; From 32201254f9eeeb304a04cc7c9b649d3eed02c293 Mon Sep 17 00:00:00 2001 From: Daniel Searle Date: Mon, 21 Oct 2024 15:47:30 +0100 Subject: [PATCH 2/4] feat(CB2-14229): snowball sqs --- tests/unit/reportGenFunction.unitTest.ts | 116 +++++++++-------------- 1 file changed, 46 insertions(+), 70 deletions(-) diff --git a/tests/unit/reportGenFunction.unitTest.ts b/tests/unit/reportGenFunction.unitTest.ts index 6eac9b2..0c4a3ae 100644 --- a/tests/unit/reportGenFunction.unitTest.ts +++ b/tests/unit/reportGenFunction.unitTest.ts @@ -1,9 +1,9 @@ -const mockProcessRecord = jest.fn(); - import { reportGen } from "../../src/functions/reportGen"; import { ReportGenerationService } from "../../src/services/ReportGenerationService"; import { SendATFReport } from "../../src/services/SendATFReport"; -// import mockConfig from "../util/mockConfig"; +import { ERRORS } from "../../src/assets/enum"; + +const mockProcessRecord = jest.fn(); jest.mock("../../src/services/ReportGenerationService"); jest.mock("../../src/services/SendATFReport"); @@ -16,91 +16,67 @@ describe("Retro Gen Function", () => { jest.setTimeout(5000); return new Promise((r) => setTimeout(r, 0)); }); - // const ctx = mockContext(); - // mockConfig(); + const ctx = {}; - context("Receiving an empty event (of various types)", () => { - it("should throw errors (event = {})", async () => { - expect.assertions(1); - try { - await reportGen({}, ctx as any, () => { - return; - }); - } catch (e) { - expect(e.message).toEqual("Event is empty"); - } + + describe("When Receiving an invalid event", () => { + it("should throw an error when it is empty)", async () => { + await expect(reportGen({}, ctx as any, () => { + return; + })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); - it("should throw errors (event = null)", async () => { - expect.assertions(1); - try { - await reportGen(null, ctx as any, () => { - return; - }); - } catch (e) { - expect(e.message).toEqual("Event is empty"); - } + + it("should throw an error when the event is null)", async () => { + await expect(reportGen(null, ctx as any, () => { + return; + })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); - it("should throw errors (event has no records)", async () => { - expect.assertions(1); - try { - await reportGen({ something: true }, ctx as any, () => { - return; - }); - } catch (e) { - expect(e.message).toEqual("Event is empty"); - } + + it("should throw an error when the event has no records", async () => { + await expect(reportGen({something: true}, ctx as any, () => { + return; + })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); - it("should throw errors (event Records is not array)", async () => { - expect.assertions(1); - try { - await reportGen({ Records: true }, ctx as any, () => { - return; - }); - } catch (e) { - expect(e.message).toEqual("Event is empty"); - } + + it("should throw an error when the event records is not array", async () => { + await expect(reportGen({Records: true}, ctx as any, () => { + return; + })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); - it("should throw errors (event Records array is empty)", async () => { - expect.assertions(1); - try { - await reportGen({ Records: [] }, ctx as any, () => { - return; - }); - } catch (e) { - expect(e.message).toEqual("Event is empty"); - } + + it("should throw an error when the event records array is empty", async () => { + await expect(reportGen({Records: []}, ctx as any, () => { + return; + })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); }); - context("Inner services fail", () => { + describe("Inner services fail", () => { afterEach(() => { jest.restoreAllMocks(); }); - it("Should throw an error (generateATFReport fails)", async () => { + it("Should add to batchItemFailures (generateATFReport fails)", async () => { ReportGenerationService.prototype.generateATFReport = jest.fn().mockRejectedValue(new Error("Oh no!")); mockProcessRecord.mockReturnValueOnce("All good"); - expect.assertions(1); - try { - await reportGen({ Records: [{ body: mockPayload }] }, ctx as any, () => { - return; - }); - } catch (e) { - expect(e.message).toEqual("Oh no!"); - } + + const result = await reportGen({Records: [JSON.parse(mockPayload)]}, ctx as any, () => { + return; + }); + expect(result.batchItemFailures).toHaveLength(1); + expect(result.batchItemFailures[0].itemIdentifier).toBe("1234"); }); - it("Should throw an error (bucket upload fails)", async () => { + + it("Should add to batchItemFailures (bucket upload fails)", async () => { ReportGenerationService.prototype.generateATFReport = jest.fn().mockResolvedValue("Looking good"); SendATFReport.prototype.sendATFReport = jest.fn().mockRejectedValue(new Error("Oh dear")); mockProcessRecord.mockReturnValueOnce("All good"); - expect.assertions(1); - try { - await reportGen({ Records: [{ body: mockPayload }] }, ctx as any, () => { - return; - }); - } catch (e) { - expect(e.message).toEqual("Oh dear"); - } + + const result = await reportGen({Records: [JSON.parse(mockPayload)]}, ctx as any, () => { + return; + }); + expect(result.batchItemFailures).toHaveLength(1); + expect(result.batchItemFailures[0].itemIdentifier).toBe("1234"); }); }); }); From 12cc5e670c37d493de1f167cb2aea24a74e2a6f0 Mon Sep 17 00:00:00 2001 From: Daniel Searle Date: Tue, 22 Oct 2024 11:12:47 +0100 Subject: [PATCH 3/4] feat(CB2-14229): snowball sqs --- src/functions/reportGen.ts | 9 ++-- tests/unit/reportGenFunction.unitTest.ts | 61 ++++++++++++++++++------ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/functions/reportGen.ts b/src/functions/reportGen.ts index b2ac9bc..88ef13c 100644 --- a/src/functions/reportGen.ts +++ b/src/functions/reportGen.ts @@ -1,7 +1,6 @@ import { LambdaClient } from "@aws-sdk/client-lambda"; -import { PutObjectRequest } from "@aws-sdk/client-s3"; import { unmarshall } from "@aws-sdk/util-dynamodb"; -import { Callback, Context, DynamoDBBatchItemFailure, DynamoDBBatchResponse, Handler } from "aws-lambda"; +import { Callback, Context, Handler, SQSBatchItemFailure, SQSBatchResponse } from "aws-lambda"; import { ERRORS } from "../assets/enum"; import { ActivitiesService } from "../services/ActivitiesService"; import { LambdaService } from "../services/LambdaService"; @@ -16,12 +15,12 @@ import { ActivitySchema } from "@dvsa/cvs-type-definitions/types/v1/activity"; * @param context - λ Context * @param callback - callback function */ -const reportGen: Handler = async (event: any, context?: Context, callback?: Callback): Promise => { +const reportGen: Handler = async (event: any, context?: Context, callback?: Callback): Promise => { if (!event || !event.Records || !Array.isArray(event.Records) || !event.Records.length) { console.error("ERROR: event is not defined."); throw new Error(ERRORS.EVENT_IS_EMPTY); } - const batchItemFailures: DynamoDBBatchItemFailure[] = []; + const batchItemFailures: SQSBatchItemFailure[] = []; const lambdaService = new LambdaService(new LambdaClient({})); const reportService: ReportGenerationService = new ReportGenerationService(new TestResultsService(lambdaService), new ActivitiesService(lambdaService)); @@ -45,7 +44,7 @@ const reportGen: Handler = async (event: any, context?: Context, callback?: Call } catch (error) { console.error(error); batchItemFailures.push({ - itemIdentifier: record.dynamodb?.SequenceNumber ?? "", + itemIdentifier: record.messageId, }); } } diff --git a/tests/unit/reportGenFunction.unitTest.ts b/tests/unit/reportGenFunction.unitTest.ts index 0c4a3ae..d0823f8 100644 --- a/tests/unit/reportGenFunction.unitTest.ts +++ b/tests/unit/reportGenFunction.unitTest.ts @@ -1,6 +1,6 @@ -import { reportGen } from "../../src/functions/reportGen"; import { ReportGenerationService } from "../../src/services/ReportGenerationService"; import { SendATFReport } from "../../src/services/SendATFReport"; +import { reportGen } from "../../src/functions/reportGen"; import { ERRORS } from "../../src/assets/enum"; const mockProcessRecord = jest.fn(); @@ -8,9 +8,41 @@ const mockProcessRecord = jest.fn(); jest.mock("../../src/services/ReportGenerationService"); jest.mock("../../src/services/SendATFReport"); -const mockPayload = '{"eventID":"f9e63bf29bd6adf174e308201a97259f","eventName":"MODIFY","eventVersion":"1.1","eventSource":"aws:dynamodb","awsRegion":"eu-west-1","dynamodb":{"ApproximateCreationDateTime":1711549645,"Keys":{"id":{"S":"6e4bd304-446e-4678-8289-dasdasjkl"}},"NewImage":{"testerStaffId":{"S":"132"},"testStationPNumber":{"S":"87-1369564"},"testerEmail":{"S":"tester@dvsa.gov.uk1111"},"testStationType":{"S":"gvts"},"testStationEmail":{"S":"teststationname@dvsa.gov.uk"},"startTime":{"S":"2022-01-01T10:00:40.561Z"},"endTime":{"S":"2022-01-01T10:00:40.561Z"},"id":{"S":"6e4bd304-446e-4678-8289-dasdasjkl"},"testStationName":{"S":"Rowe, Wunsch and Wisoky"},"activityType":{"S":"visit"},"activityDay":{"S":"2022-01-01"},"testerName":{"S":"namey mcname"}},"OldImage":{"testerStaffId":{"S":"132"},"testStationPNumber":{"S":"87-1369564"},"testerEmail":{"S":"tester@dvsa.gov.uk1111"},"testStationType":{"S":"gvts"},"testStationEmail":{"S":"teststationname@dvsa.gov.uk"},"startTime":{"S":"2022-01-01T10:00:40.561Z"},"endTime":{"S":"2022-01-01T10:00:40.561Z"},"id":{"S":"6e4bd304-446e-4678-8289-dasdasjkl"},"testStationName":{"S":"Rowe, Wunsch and Wisoky"},"activityType":{"S":"visit"},"activityDay":{"S":"2022-01-01"},"testerName":{"S":"231232132"}},"SequenceNumber":"1234","SizeBytes":704,"StreamViewType":"NEW_AND_OLD_IMAGES"},"eventSourceARN":"arn:aws::eu--1::/cvs---//:32:37.491"}'; +const mockSQSPayload = { + messageId: "059f36b4-87a3-44ab-83d2-661975830a7d", + receiptHandle: "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a", + body: JSON.stringify({ + dynamodb: { + NewImage: { + testerStaffId: { S: "132" }, + testStationPNumber: { S: "87-1369564" }, + testerEmail: { S: "tester@dvsa.gov.uk1111" }, + testStationType: { S: "gvts" }, + testStationEmail: { S: "teststationname@dvsa.gov.uk" }, + startTime: { S: "2022-01-01T10:00:40.561Z" }, + endTime: { S: "2022-01-01T10:00:40.561Z" }, + id: { S: "6e4bd304-446e-4678-8289-dasdasjkl" }, + testStationName: { S: "Rowe, Wunsch and Wisoky" }, + activityType: { S: "visit" }, + activityDay: { S: "2022-01-01" }, + testerName: { S: "Jon Mcdonald" } + } + } + }), + attributes: { + ApproximateReceiveCount: "1", + SentTimestamp: "1545082649183", + SenderId: "AIDAIENQZJOLO23YVJ4VO", + ApproximateFirstReceiveTimestamp: "1545082649185" + }, + messageAttributes: {}, + md5OfBody: "e4e68fb7bd0e697a0ae8f1bb342846b3", + eventSource: "aws:sqs", + eventSourceARN: "arn:aws:sqs:region:123456789012:queue", + awsRegion: "eu-west-1" +}; -describe("Retro Gen Function", () => { +describe("Report Generation Lambda Function", () => { beforeAll(() => jest.setTimeout(60000)); afterAll(() => { jest.setTimeout(5000); @@ -20,32 +52,32 @@ describe("Retro Gen Function", () => { const ctx = {}; describe("When Receiving an invalid event", () => { - it("should throw an error when it is empty)", async () => { + it("should throw an error when it is empty", async () => { await expect(reportGen({}, ctx as any, () => { return; })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); - it("should throw an error when the event is null)", async () => { + it("should throw an error when the event is null", async () => { await expect(reportGen(null, ctx as any, () => { return; })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); it("should throw an error when the event has no records", async () => { - await expect(reportGen({something: true}, ctx as any, () => { + await expect(reportGen({ something: true }, ctx as any, () => { return; })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); it("should throw an error when the event records is not array", async () => { - await expect(reportGen({Records: true}, ctx as any, () => { + await expect(reportGen({ Records: true }, ctx as any, () => { return; })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); it("should throw an error when the event records array is empty", async () => { - await expect(reportGen({Records: []}, ctx as any, () => { + await expect(reportGen({ Records: [] }, ctx as any, () => { return; })).rejects.toThrow(ERRORS.EVENT_IS_EMPTY); }); @@ -56,27 +88,28 @@ describe("Retro Gen Function", () => { jest.restoreAllMocks(); }); - it("Should add to batchItemFailures (generateATFReport fails)", async () => { + it("Should add to batchItemFailures when generateATFReport fails", async () => { ReportGenerationService.prototype.generateATFReport = jest.fn().mockRejectedValue(new Error("Oh no!")); mockProcessRecord.mockReturnValueOnce("All good"); - const result = await reportGen({Records: [JSON.parse(mockPayload)]}, ctx as any, () => { + const result = await reportGen({ Records: [mockSQSPayload] }, ctx as any, () => { return; }); + expect(result.batchItemFailures).toHaveLength(1); - expect(result.batchItemFailures[0].itemIdentifier).toBe("1234"); + expect(result.batchItemFailures[0].itemIdentifier).toBe(mockSQSPayload.messageId); }); - it("Should add to batchItemFailures (bucket upload fails)", async () => { + it("Should add to batchItemFailures when bucket upload fails", async () => { ReportGenerationService.prototype.generateATFReport = jest.fn().mockResolvedValue("Looking good"); SendATFReport.prototype.sendATFReport = jest.fn().mockRejectedValue(new Error("Oh dear")); mockProcessRecord.mockReturnValueOnce("All good"); - const result = await reportGen({Records: [JSON.parse(mockPayload)]}, ctx as any, () => { + const result = await reportGen({ Records: [mockSQSPayload] }, ctx as any, () => { return; }); expect(result.batchItemFailures).toHaveLength(1); - expect(result.batchItemFailures[0].itemIdentifier).toBe("1234"); + expect(result.batchItemFailures[0].itemIdentifier).toBe(mockSQSPayload.messageId); }); }); }); From 131d75952b3a151288bde499350f946b67089892 Mon Sep 17 00:00:00 2001 From: Daniel Searle Date: Tue, 22 Oct 2024 15:46:36 +0100 Subject: [PATCH 4/4] feat(CB2-14229): pr comments --- src/functions/reportGen.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/functions/reportGen.ts b/src/functions/reportGen.ts index 88ef13c..c495672 100644 --- a/src/functions/reportGen.ts +++ b/src/functions/reportGen.ts @@ -1,6 +1,6 @@ import { LambdaClient } from "@aws-sdk/client-lambda"; import { unmarshall } from "@aws-sdk/util-dynamodb"; -import { Callback, Context, Handler, SQSBatchItemFailure, SQSBatchResponse } from "aws-lambda"; +import { Callback, Context, Handler, SQSBatchItemFailure, SQSBatchResponse, SQSEvent } from "aws-lambda"; import { ERRORS } from "../assets/enum"; import { ActivitiesService } from "../services/ActivitiesService"; import { LambdaService } from "../services/LambdaService"; @@ -15,7 +15,7 @@ import { ActivitySchema } from "@dvsa/cvs-type-definitions/types/v1/activity"; * @param context - λ Context * @param callback - callback function */ -const reportGen: Handler = async (event: any, context?: Context, callback?: Callback): Promise => { +const reportGen: Handler = async (event: SQSEvent, context?: Context, callback?: Callback): Promise => { if (!event || !event.Records || !Array.isArray(event.Records) || !event.Records.length) { console.error("ERROR: event is not defined."); throw new Error(ERRORS.EVENT_IS_EMPTY); @@ -38,14 +38,11 @@ const reportGen: Handler = async (event: any, context?: Context, callback?: Call const generationServiceResponse = await reportService.generateATFReport(visit); console.debug(`Report generated: ${JSON.stringify(generationServiceResponse)}`); await sendATFReport.sendATFReport(generationServiceResponse, visit); - console.debug("All emails sent, terminating lambda"); } } catch (error) { console.error(error); - batchItemFailures.push({ - itemIdentifier: record.messageId, - }); + batchItemFailures.push({ itemIdentifier: record.messageId }); } } return { batchItemFailures };