Batch Updates from feature branch to Stage branch (adobecom#32)

* Promote API Batching Implementation (adobecom#29)

* Promote API Batching Implementation

* Refactor Promote Batching

* Performing sequential Promote of Batches, instead of parallel promote actions & Removed spToken and validateAction

* Batching Updates (adobecom#31)

Batching updates
 - Deleting all image paths which exist only in .md and show up while processing
 - Don't execute async copy or promote operations, perform sequentially
 - Handle Promote and Copy batches in parallel
 - Adding in-progress statuses for not repeating the already initiated steps in schedulers and actions
 - Awaiting all copy and promote steps before updating the status to avoid impact of promote and copy in parallel
 - Cross referencing & checking existence of same bathes in Copy from promote worker and vice versa before updating batch statuses as promoted

* Removing DriveID from github secrets and .env, so it gets passed as input from API Payload from UI (adobecom#33)

actions/graybox/copy-sched.js

@@ -0,0 +1,132 @@
+/* ************************************************************************
+* ADOBE CONFIDENTIAL
+* ___________________
+*
+* Copyright 2024 Adobe
+* All Rights Reserved.
+*
+* NOTICE: All information contained herein is, and remains
+* the property of Adobe and its suppliers, if any. The intellectual
+* and technical concepts contained herein are proprietary to Adobe
+* and its suppliers and are protected by all applicable intellectual
+* property laws, including trade secret and copyright laws.
+* Dissemination of this information or reproduction of this material
+* is strictly forbidden unless prior written permission is obtained
+* from Adobe.
+************************************************************************* */
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+const openwhisk = require('openwhisk');
+const { getAioLogger } = require('../utils');
+const initFilesWrapper = require('./filesWrapper');
+
+async function main(params) {
+    const logger = getAioLogger();
+    const ow = openwhisk();
+    let responsePayload = 'Graybox Copy Scheduler invoked';
+    logger.info(responsePayload);
+
+    const filesWrapper = await initFilesWrapper(logger);
+
+    try {
+        let projectQueue = await filesWrapper.readFileIntoObject('graybox_promote/project_queue.json');
+        logger.info(`From Copy-sched Project Queue Json: ${JSON.stringify(projectQueue)}`);
+
+        // Sorting the Promote Projects based on the 'createdTime' property, pick the oldest project
+        projectQueue = projectQueue.sort((a, b) => a.createdTime - b.createdTime);
+
+        // Find the First Project where status is 'processed'
+        const projectEntry = projectQueue.find((project) => project.status === 'processed');
+        if (projectEntry && projectEntry.projectPath) {
+            const project = projectEntry.projectPath;
+            const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/status.json`);
+            logger.info(`In Copy-sched Project Status Json: ${JSON.stringify(projectStatusJson)}`);
+
+            // Read the Batch Status in the current project's "batch_status.json" file
+            const batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/batch_status.json`);
+            logger.info(`In Copy Sched, batchStatusJson: ${JSON.stringify(batchStatusJson)}`);
+
+            const copyBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/copy_batches.json`);
+            logger.info(`In Copy-sched Copy Batches Json: ${JSON.stringify(copyBatchesJson)}`);
+
+            // Find if any batch is in 'copy_in_progress' status, if yes then don't trigger another copy action for another "processed" batch
+            const copyOrPromoteInProgressBatch = Object.entries(batchStatusJson)
+                .find(([batchName, copyBatchJson]) => (copyBatchJson.status === 'copy_in_progress' || copyBatchJson.status === 'promote_in_progress'));
+
+            if (copyOrPromoteInProgressBatch && Array.isArray(copyOrPromoteInProgressBatch) && copyOrPromoteInProgressBatch.length > 0) {
+                responsePayload = `Promote or Copy Action already in progress for Batch: ${copyOrPromoteInProgressBatch[0]}, not triggering another action until it completes`;
+                return {
+                    code: 200,
+                    payload: responsePayload
+                };
+            }
+
+            // Find the First Batch where status is 'processed', to promote one batch at a time
+            const processedBatchName = Object.keys(copyBatchesJson)
+                .find((batchName) => copyBatchesJson[batchName].status === 'processed');
+            // If no batch is found with status 'processed then nothing to promote', return
+            if (!processedBatchName) {
+                responsePayload = 'No Copy Batches found with status "processed"';
+                return {
+                    code: 200,
+                    payload: responsePayload
+                };
+            }
+
+            if (copyBatchesJson[processedBatchName].status === 'processed') {
+                // copy all params from json into the params object
+                const inputParams = projectStatusJson?.params;
+                Object.keys(inputParams).forEach((key) => {
+                    params[key] = inputParams[key];
+                });
+                // Set the Project & Batch Name in params for the Copy Content Worker Action to read and process
+                params.project = project;
+                params.batchName = processedBatchName;
+
+                logger.info(`In Copy-sched, Invoking Copy Content Worker for Batch: ${processedBatchName} of Project: ${project}`);
+                try {
+                    return ow.actions.invoke({
+                        name: 'graybox/copy-worker',
+                        blocking: false,
+                        result: false,
+                        params
+                    }).then(async (result) => {
+                        logger.info(result);
+                        return {
+                            code: 200,
+                            payload: responsePayload
+                        };
+                    }).catch(async (err) => {
+                        responsePayload = 'Failed to invoke graybox copy action';
+                        logger.error(`${responsePayload}: ${err}`);
+                        return {
+                            code: 500,
+                            payload: responsePayload
+                        };
+                    });
+                } catch (err) {
+                    responsePayload = 'Unknown error occurred while invoking Copy Content Worker Action';
+                    logger.error(`${responsePayload}: ${err}`);
+                    responsePayload = err;
+                }
+            }
+            responsePayload = 'Triggered multiple Copy Content Worker Actions';
+            return {
+                code: 200,
+                payload: responsePayload,
+            };
+        }
+    } catch (err) {
+        responsePayload = 'Unknown error occurred while processing the projects for Copy';
+        logger.error(`${responsePayload}: ${err}`);
+        responsePayload = err;
+    }
+
+    // No errors while initiating all the Copy Content Worker Action for all the projects
+    return {
+        code: 200,
+        payload: responsePayload
+    };
+}
+
+exports.main = main;

actions/graybox/copy-worker.js

@@ -0,0 +1,176 @@
+/* ************************************************************************
+* ADOBE CONFIDENTIAL
+* ___________________
+*
+* Copyright 2024 Adobe
+* All Rights Reserved.
+*
+* NOTICE: All information contained herein is, and remains
+* the property of Adobe and its suppliers, if any. The intellectual
+* and technical concepts contained herein are proprietary to Adobe
+* and its suppliers and are protected by all applicable intellectual
+* property laws, including trade secret and copyright laws.
+* Dissemination of this information or reproduction of this material
+* is strictly forbidden unless prior written permission is obtained
+* from Adobe.
+************************************************************************* */
+
+const { getAioLogger, toUTCStr } = require('../utils');
+const AppConfig = require('../appConfig');
+const Sharepoint = require('../sharepoint');
+const initFilesWrapper = require('./filesWrapper');
+
+const logger = getAioLogger();
+
+async function main(params) {
+    logger.info('Graybox Copy Content Action triggered');
+
+    const appConfig = new AppConfig(params);
+    const { gbRootFolder, experienceName, projectExcelPath } = appConfig.getPayload();
+
+    const sharepoint = new Sharepoint(appConfig);
+
+    // process data in batches
+    const filesWrapper = await initFilesWrapper(logger);
+    let responsePayload;
+    let promotes = [];
+    const failedPromotes = [];
+
+    logger.info('In Copy Worker, Processing Copy Content');
+
+    const project = params.project || '';
+    const batchName = params.batchName || '';
+
+    // Read the Batch Status in the current project's "batch_status.json" file
+    let batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/batch_status.json`);
+
+    const promoteErrorsJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/promote_errors.json`);
+
+    let copyBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/copy_batches.json`);
+
+    const copyBatchJson = copyBatchesJson[batchName] || {};
+
+    logger.info(`In Copy Worker, Copy File Paths for batchname ${batchName}:  ${JSON.stringify(copyBatchJson)}`);
+
+    // Update & Write the Batch Status to in progress "batch_status.json" file
+    // So that the scheduler doesn't pick the same batch again
+    batchStatusJson[batchName] = 'copy_in_progress';
+    await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJson);
+    // Write the copy batches JSON file
+    copyBatchesJson[batchName].status = 'promote_in_progress';
+    await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/copy_batches.json`, copyBatchesJson);
+
+    // Process the Copy Content
+    const copyFilePathsJson = copyBatchJson.files || [];
+    for (let i = 0; i < copyFilePathsJson.length; i += 1) {
+        const copyPathsEntry = copyFilePathsJson[i];
+        // Download the grayboxed file and save it to default content location
+        // eslint-disable-next-line no-await-in-loop
+        const { fileDownloadUrl } = await sharepoint.getFileData(copyPathsEntry.copySourceFilePath, true);
+        // eslint-disable-next-line no-await-in-loop
+        const file = await sharepoint.getFileUsingDownloadUrl(fileDownloadUrl);
+        // eslint-disable-next-line no-await-in-loop
+        const saveStatus = await sharepoint.saveFileSimple(file, copyPathsEntry.copyDestFilePath);
+
+        if (saveStatus?.success) {
+            promotes.push(copyPathsEntry.copyDestFilePath);
+        } else if (saveStatus?.errorMsg?.includes('File is locked')) {
+            failedPromotes.push(`${copyPathsEntry.copyDestFilePath} (locked file)`);
+        } else {
+            failedPromotes.push(copyPathsEntry.copyDestFilePath);
+        }
+    }
+
+    logger.info(`In Copy Worker, Promotes for batchname ${batchName} no.of files ${promotes.length}, files list: ${JSON.stringify(promotes)}`);
+    // Update the Promoted Paths in the current project's "promoted_paths.json" file
+    if (promotes.length > 0) {
+        const promotedPathsJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/promoted_paths.json`) || {};
+        // Combined existing If any promotes already exist in promoted_paths.json for the current batch either from Copy action or Promote Action
+        if (promotedPathsJson[batchName]) {
+            promotes = promotes.concat(promotedPathsJson[batchName]);
+        }
+        promotedPathsJson[batchName] = promotes;
+        await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promoted_paths.json`, promotedPathsJson);
+    }
+
+    if (failedPromotes.length > 0) {
+        await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promote_errors.json`, promoteErrorsJson.concat(failedPromotes));
+    }
+
+    // Update the Copy Batch Status in the current project's "copy_batches.json" file
+    copyBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/copy_batches.json`);
+    copyBatchesJson[batchName].status = 'promoted';
+    // Write the copy batches JSON file
+    await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/copy_batches.json`, copyBatchesJson);
+
+    // Check in parallel if the Same Batch Name Exists & is Promoted in the Promote Batches JSON
+    const promoteBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/promote_batches.json`);
+    const promoteBatchJson = promoteBatchesJson[batchName];
+    let markBatchAsPromoted = true;
+    if (promoteBatchJson) {
+        markBatchAsPromoted = promoteBatchJson.status === 'promoted';
+    }
+
+    batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/batch_status.json`);
+    if (markBatchAsPromoted) {
+        // Update the Batch Status in the current project's "batch_status.json" file
+        if (batchStatusJson && batchStatusJson[batchName] && (promotes.length > 0 || failedPromotes.length > 0)) {
+            batchStatusJson[batchName] = 'promoted';
+            // Write the updated batch_status.json file
+            await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJson);
+        }
+
+        // If all batches are promoted, then mark the project as 'promoted'
+        const allBatchesPromoted = Object.keys(batchStatusJson).every((key) => batchStatusJson[key] === 'promoted');
+        if (allBatchesPromoted) {
+            // Update the Project Status in JSON files
+            updateProjectStatus(gbRootFolder, experienceName, filesWrapper);
+        }
+    }
+
+    // Update the Project Excel with the Promote Status
+    try {
+        const sFailedPromoteStatuses = failedPromotes.length > 0 ? `Failed Promotes: \n${failedPromotes.join('\n')}` : '';
+        const promoteExcelValues = [[`Step 4 of 5: Promote Copy completed for Batch ${batchName}`, toUTCStr(new Date()), sFailedPromoteStatuses]];
+        await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', promoteExcelValues);
+    } catch (err) {
+        logger.error(`Error Occured while updating Excel during Graybox Promote Copy: ${err}`);
+    }
+
+    responsePayload = `Copy Worker finished promoting content for batch ${batchName}`;
+    logger.info(responsePayload);
+    return exitAction({
+        body: responsePayload,
+        statusCode: 200
+    });
+}
+
+/**
+ * Update the Project Status in the current project's "status.json" file & the parent "project_queue.json" file
+ * @param {*} gbRootFolder graybox root folder
+ * @param {*} experienceName graybox experience name
+ * @param {*} filesWrapper filesWrapper object
+ * @returns updated project status
+ */
+async function updateProjectStatus(gbRootFolder, experienceName, filesWrapper) {
+    const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/status.json`);
+
+    // Update the Project Status in the current project's "status.json" file
+    projectStatusJson.status = 'promoted';
+    await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/status.json`, projectStatusJson);
+
+    // Update the Project Status in the parent "project_queue.json" file
+    const projectQueue = await filesWrapper.readFileIntoObject('graybox_promote/project_queue.json');
+    const index = projectQueue.findIndex((obj) => obj.projectPath === `${gbRootFolder}/${experienceName}`);
+    if (index !== -1) {
+        // Replace the object at the found index
+        projectQueue[index].status = 'promoted';
+        await filesWrapper.writeFile('graybox_promote/project_queue.json', projectQueue);
+    }
+}
+
+function exitAction(resp) {
+    return resp;
+}
+
+exports.main = main;