Merge pull request #233 from bcgov/feature/createBucketChild

Add support for children bucket creation
bcgov · Dec 12, 2023 · 4904784 · 4904784
2 parents a07a0c8 + 496d385
commit 4904784
Show file tree

Hide file tree

Showing 29 changed files with 929 additions and 219 deletions.
diff --git a/app/src/components/log.js b/app/src/components/log.js
@@ -58,7 +58,8 @@ if (config.has('server.logFile')) {
 /**
  * Returns a Winston Logger or Child Winston Logger
  * @param {string} [filename] Optional module filename path to annotate logs with
- * @returns {object} A child logger with appropriate metadata if `filename` is defined. Otherwise returns a standard logger.
+ * @returns {object} A child logger with appropriate metadata if `filename` is defined.
+ * Otherwise returns a standard logger.
  */
 const getLogger = (filename) => {
   return filename ? log.child({ component: parse(filename).name }) : log;

diff --git a/app/src/components/queueManager.js b/app/src/components/queueManager.js
@@ -97,18 +97,30 @@ class QueueManager {
 
         const objectId = await syncService.syncJob(job.path, job.bucketId, job.full, job.createdBy);
 
-        log.verbose(`Finished processing job id ${job.id}`, { function: 'processNextJob', job: job, objectId: objectId });
+        log.verbose(`Finished processing job id ${job.id}`, {
+          function: 'processNextJob',
+          job: job,
+          objectId: objectId
+        });
 
         this.isBusy = false;
         // If job is completed, check if there are more jobs
         if (!this.toClose) this.checkQueue();
       }
     } catch (err) {
-      log.error(`Error encountered on job id ${job.id}: ${err.message}`, { function: 'processNextJob', job: job, error: err });
+      log.error(`Error encountered on job id ${job.id}: ${err.message}`, {
+        function: 'processNextJob',
+        job: job,
+        error: err
+      });
 
       const maxRetries = parseInt(config.get('server.maxRetries'));
       if (job.retries + 1 > maxRetries) {
-        log.warn(`Job has exceeded the ${maxRetries} maximum retries permitted`, { function: 'processNextJob', job: job, maxRetries: maxRetries });
+        log.warn(`Job has exceeded the ${maxRetries} maximum retries permitted`, {
+          function: 'processNextJob',
+          job: job,
+          maxRetries: maxRetries
+        });
       } else {
         objectQueueService.enqueue({
           jobs: [{ bucketId: job.bucketId, path: job.path }],

diff --git a/app/src/controllers/bucket.js b/app/src/controllers/bucket.js
@@ -133,6 +133,69 @@ const controller = {
     }
   },
 
+  /**
+   * @function createBucketChild
+   * Creates a child bucket
+   * @param {object} req Express request object
+   * @param {object} res Express response object
+   * @param {function} next The next callback function
+   * @returns {function} Express middleware function
+   * @throws The error encountered upon failure
+   */
+  async createBucketChild(req, res, next) {
+    try {
+      // Get Parent bucket data
+      const parentBucketId = addDashesToUuid(req.params.bucketId);
+      const parentBucket = await bucketService.read(parentBucketId);
+
+      // Check new child key length
+      const childKey = joinPath(stripDelimit(parentBucket.key), stripDelimit(req.body.subKey));
+      if (childKey.length > 255) {
+        throw new Problem(422, {
+          detail: 'New derived key exceeds maximum length of 255',
+          instance: req.originalUrl,
+          key: childKey
+        });
+      }
+
+      // Check for existing bucket collision
+      const bucketCollision = await bucketService.readUnique({
+        bucket: parentBucket.bucket,
+        endpoint: parentBucket.endpoint,
+        key: childKey
+      }).catch(() => undefined);
+
+      if (bucketCollision) {
+        throw new Problem(409, {
+          bucketId: bucketCollision.bucketId,
+          detail: 'Requested bucket already exists',
+          instance: req.originalUrl,
+          key: childKey
+        });
+      }
+
+      // Check for credential accessibility/validity
+      const childBucket = {
+        bucketName: req.body.bucketName,
+        accessKeyId: parentBucket.accessKeyId,
+        bucket: parentBucket.bucket,
+        endpoint: parentBucket.endpoint,
+        key: childKey,
+        secretAccessKey: parentBucket.secretAccessKey,
+        region: parentBucket.region ?? undefined,
+        active: parentBucket.active
+      };
+      await controller._validateCredentials(childBucket);
+      childBucket.userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER));
+
+      // Create child bucket
+      const response = await bucketService.create(childBucket);
+      res.status(201).json(redactSecrets(response, secretFields));
+    } catch (e) {
+      next(errorToProblem(SERVICE, e));
+    }
+  },
+
   /**
    * @function deleteBucket
    * Deletes the bucket

diff --git a/app/src/controllers/bucketPermission.js b/app/src/controllers/bucketPermission.js
@@ -90,7 +90,9 @@ const controller = {
   async addPermissions(req, res, next) {
     try {
       const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER));
-      const response = await bucketPermissionService.addPermissions(addDashesToUuid(req.params.bucketId), req.body, userId);
+      const response = await bucketPermissionService.addPermissions(
+        addDashesToUuid(req.params.bucketId),req.body, userId
+      );
       res.status(201).json(response);
     } catch (e) {
       next(errorToProblem(SERVICE, e));

diff --git a/app/src/controllers/object.js b/app/src/controllers/object.js
@@ -149,8 +149,15 @@ const controller = {
       await utils.trxWrapper(async (trx) => {
         // create or update version in DB (if a non-versioned object)
         const version = s3Response.VersionId ?
-          await versionService.copy(sourceS3VersionId, s3Response.VersionId, objId, s3Response.CopyObjectResult?.ETag, userId, trx) :
-          await versionService.update({ ...data, id: objId, etag: s3Response.CopyObjectResult?.ETag, isLatest: true }, userId, trx);
+          await versionService.copy(
+            sourceS3VersionId, s3Response.VersionId, objId, s3Response.CopyObjectResult?.ETag, userId, trx
+          ) :
+          await versionService.update({
+            ...data,
+            id: objId,
+            etag: s3Response.CopyObjectResult?.ETag,
+            isLatest: true
+          }, userId, trx);
 
         // add metadata for version in DB
         await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
@@ -183,9 +190,15 @@ const controller = {
       // format new tags to array of objects
       const newTags = Object.entries({ ...req.query.tagset }).map(([k, v]) => ({ Key: k, Value: v }));
       // get source version that we are adding tags to
-      const sourceS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const sourceS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
       // get existing tags on source version
-      const { TagSet: existingTags } = await storageService.getObjectTagging({ filePath: objPath, s3VersionId: sourceS3VersionId, bucketId: bucketId });
+      const { TagSet: existingTags } = await storageService.getObjectTagging({
+        filePath: objPath,
+        s3VersionId: sourceS3VersionId,
+        bucketId: bucketId
+      });
 
       const newSet = newTags
         // Join new tags and existing tags
@@ -243,7 +256,11 @@ const controller = {
       // Preflight CREATE permission check if bucket scoped and OIDC authenticated
       const bucketId = req.query.bucketId ? addDashesToUuid(req.query.bucketId) : undefined;
       if (bucketId && userId) {
-        const permission = await bucketPermissionService.searchPermissions({ userId: userId, bucketId: bucketId, permCode: 'CREATE' });
+        const permission = await bucketPermissionService.searchPermissions({
+          userId: userId,
+          bucketId: bucketId,
+          permCode: 'CREATE'
+        });
         if (!permission.length) {
           throw new Problem(403, {
             detail: 'User lacks permission to complete this action',
@@ -381,7 +398,9 @@ const controller = {
       const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER));
 
       // Source S3 Version to copy from
-      const sourceS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const sourceS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
 
       const source = await storageService.headObject({ filePath: objPath, s3VersionId: sourceS3VersionId, bucketId });
       if (source.ContentLength > MAXCOPYOBJECTLENGTH) {
@@ -428,8 +447,15 @@ const controller = {
       await utils.trxWrapper(async (trx) => {
         // create or update version in DB(if a non-versioned object)
         const version = s3Response.VersionId ?
-          await versionService.copy(sourceS3VersionId, s3Response.VersionId, objId, s3Response.CopyObjectResult?.ETag, userId, trx) :
-          await versionService.update({ ...data, id: objId, etag: s3Response.CopyObjectResult?.ETag, isLatest: true }, userId, trx);
+          await versionService.copy(
+            sourceS3VersionId, s3Response.VersionId, objId, s3Response.CopyObjectResult?.ETag, userId, trx
+          ) :
+          await versionService.update({
+            ...data,
+            id: objId,
+            etag: s3Response.CopyObjectResult?.ETag,
+            isLatest: true
+          }, userId, trx);
         // add metadata to version in DB
         await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
 
@@ -457,7 +483,9 @@ const controller = {
       const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER));
 
       // target S3 version to delete
-      const targetS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const targetS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
 
       const data = {
         bucketId: req.currentObject?.bucketId,
@@ -519,9 +547,15 @@ const controller = {
       const objPath = req.currentObject?.path;
 
       // Target S3 version
-      const targetS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const targetS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
 
-      const sourceObject = await storageService.getObjectTagging({ filePath: objPath, s3VersionId: targetS3VersionId, bucketId: bucketId });
+      const sourceObject = await storageService.getObjectTagging({
+        filePath: objPath,
+        s3VersionId: targetS3VersionId,
+        bucketId: bucketId
+      });
 
       // Generate object subset by subtracting/omitting defined keys via filter/inclusion
       const keysToRemove = req.query.tagset ? Object.keys(req.query.tagset) : [];
@@ -633,7 +667,9 @@ const controller = {
       const objId = addDashesToUuid(req.params.objectId);
 
       // target S3 version
-      const targetS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const targetS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
 
       const data = {
         bucketId: req.currentObject?.bucketId,
@@ -692,7 +728,9 @@ const controller = {
       const objId = addDashesToUuid(req.params.objectId);
 
       // target S3 version
-      const targetS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const targetS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
 
       const data = {
         // TODO: use req.currentObject.bucketId
@@ -703,7 +741,8 @@ const controller = {
 
       // Download via service proxy
       if (req.query.download && req.query.download === DownloadMode.PROXY) {
-        // TODO: Consider if we need a HEAD operation first before doing the actual read on large files for pre-flight caching behavior?
+        // TODO: Consider if we need a HEAD operation first before doing the actual read on large files
+        // for pre-flight caching behavior?
         const response = await storageService.readObject(data);
 
         // Set Headers via CORS library
@@ -756,7 +795,9 @@ const controller = {
       const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER));
 
       // source S3 version
-      const sourceS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const sourceS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
 
       // get metadata for source version
       const source = await storageService.headObject({ filePath: objPath, s3VersionId: sourceS3VersionId, bucketId });
@@ -795,9 +836,15 @@ const controller = {
       await utils.trxWrapper(async (trx) => {
         // create or update version (if a non-versioned object)
         const version = s3Response.VersionId ?
-          await versionService.copy(sourceS3VersionId, s3Response.VersionId, objId, s3Response.CopyObjectResult?.ETag, userId, trx) :
-
-          await versionService.update({ ...data, id: objId, etag: s3Response.CopyObjectResult?.ETag, isLatest: true }, userId, trx);
+          await versionService.copy(
+            sourceS3VersionId, s3Response.VersionId, objId, s3Response.CopyObjectResult?.ETag, userId, trx
+          ) :
+          await versionService.update({
+            ...data,
+            id: objId,
+            etag: s3Response.CopyObjectResult?.ETag,
+            isLatest: true
+          }, userId, trx);
 
         // add metadata
         await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
@@ -829,7 +876,9 @@ const controller = {
       const newTags = Object.entries({ ...req.query.tagset }).map(([k, v]) => ({ Key: k, Value: v }));
 
       // source S3 version
-      const sourceS3VersionId = await getS3VersionId(req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId);
+      const sourceS3VersionId = await getS3VersionId(
+        req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
+      );
 
       const data = {
         bucketId: req.currentObject?.bucketId,
@@ -1040,10 +1089,14 @@ const controller = {
         object.versionId = version.id;
 
         // Update Metadata
-        if (data.metadata && Object.keys(data.metadata).length) await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
+        if (data.metadata && Object.keys(data.metadata).length) {
+          await metadataService.associateMetadata(version.id, getKeyValue(data.metadata), userId, trx);
+        }
 
         // Update Tags
-        if (data.tags && Object.keys(data.tags).length) await tagService.replaceTags(version.id, getKeyValue(data.tags), userId, trx);
+        if (data.tags && Object.keys(data.tags).length) {
+          await tagService.replaceTags(version.id, getKeyValue(data.tags), userId, trx);
+        }
 
         return object;
       });

diff --git a/app/src/controllers/objectPermission.js b/app/src/controllers/objectPermission.js
@@ -88,7 +88,9 @@ const controller = {
   async addPermissions(req, res, next) {
     try {
       const userId = await userService.getCurrentUserId(utils.getCurrentIdentity(req.currentUser, SYSTEM_USER));
-      const response = await objectPermissionService.addPermissions(utils.addDashesToUuid(req.params.objectId), req.body, userId);
+      const response = await objectPermissionService.addPermissions(
+        utils.addDashesToUuid(req.params.objectId), req.body, userId
+      );
       res.status(201).json(response);
     } catch (e) {
       next(errorToProblem(SERVICE, e));

diff --git a/app/src/db/dataConnection.js b/app/src/db/dataConnection.js
@@ -62,10 +62,14 @@ class DataConnection {
     log.verbose(`Connect OK: ${connectOk}, Schema OK: ${schemaOk}, Models OK: ${modelsOk}`, { function: 'checkAll' });
 
     if (!connectOk) {
-      log.error('Could not connect to the database, check configuration and ensure database server is running', { function: 'checkAll' });
+      log.error('Could not connect to the database, check configuration and ensure database server is running', {
+        function: 'checkAll'
+      });
     }
     if (!schemaOk) {
-      log.error('Connected to the database, could not verify the schema. Ensure proper migrations have been run.', { function: 'checkAll' });
+      log.error('Connected to the database, could not verify the schema. Ensure proper migrations have been run.', {
+        function: 'checkAll'
+      });
     }
     if (!modelsOk) {
       log.error('Connected to the database, schema is ok, could not initialize Knex Models.', { function: 'checkAll' });

diff --git a/app/src/db/migrations/20220130133615_000-init.js b/app/src/db/migrations/20220130133615_000-init.js
@@ -1,3 +1,4 @@
+/* eslint-disable max-len */
 const stamps = require('../stamps');
 const { NIL: SYSTEM_USER } = require('uuid');
 

diff --git a/app/src/db/migrations/20220627000000_002-metadata-tags.js b/app/src/db/migrations/20220627000000_002-metadata-tags.js
@@ -18,7 +18,8 @@ exports.up = function (knex) {
     .then(() => knex.schema.createTable('version_metadata', table => {
       table.primary(['versionId', 'metadataId']);
       table.uuid('versionId').notNullable().references('id').inTable('version').onDelete('CASCADE').onUpdate('CASCADE');
-      table.integer('metadataId').notNullable().references('id').inTable('metadata').onDelete('CASCADE').onUpdate('CASCADE');
+      table.integer('metadataId').notNullable().references('id').inTable('metadata').onDelete('CASCADE')
+        .onUpdate('CASCADE');
       stamps(knex, table);
     }))
 

diff --git a/app/src/db/migrations/20221014000000_003-multi-bucket.js b/app/src/db/migrations/20221014000000_003-multi-bucket.js
@@ -18,9 +18,11 @@ exports.up = function (knex) {
     }))
     .then(() => knex.schema.createTable('bucket_permission', table => {
       table.uuid('id').primary();
-      table.uuid('bucketId').references('bucketId').inTable('bucket').notNullable().onUpdate('CASCADE').onDelete('CASCADE');
+      table.uuid('bucketId').references('bucketId').inTable('bucket').notNullable().onUpdate('CASCADE')
+        .onDelete('CASCADE');
       table.uuid('userId').references('userId').inTable('user').notNullable().onUpdate('CASCADE').onDelete('CASCADE');
-      table.string('permCode').references('permCode').inTable('permission').notNullable().onUpdate('CASCADE').onDelete('CASCADE');
+      table.string('permCode').references('permCode').inTable('permission').notNullable().onUpdate('CASCADE')
+        .onDelete('CASCADE');
       stamps(knex, table);
     }))
 

diff --git a/app/src/db/models/utils.js b/app/src/db/models/utils.js
@@ -59,7 +59,9 @@ const utils = {
 
   toArray(values) {
     if (values) {
-      return Array.isArray(values) ? values.filter(p => p && p.trim().length > 0) : [values].filter(p => p && p.trim().length > 0);
+      return Array.isArray(values)
+        ? values.filter(p => p && p.trim().length > 0)
+        : [values].filter(p => p && p.trim().length > 0);
     }
     return [];
   },