Merge pull request #285 from 4dn-dcic/drr_glacier_copy_kms

KMS encryption for glacier utils
4dn-dcic · Sep 19, 2023 · 0066b66 · 0066b66
2 parents 989593d + c3c4efd
commit 0066b66
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 5 deletions.
diff --git a/.gitignore b/.gitignore
@@ -121,3 +121,7 @@ ENV/
 
 # PyCharm metadata
 .idea/
+
+# Vi
+*.swp
+*.swo
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -7,6 +7,14 @@ Change Log
 ----------
 
 
+7.12.0
+======
+
+* In ``glacier_utils``:
+
+  * Add functionality for KMS key encrypted accounts
+
+
 7.11.0
 ======
 
@@ -16,6 +24,7 @@ Change Log
   * Fix in ``get_schema`` and ``get_schemas`` for the ``portal_vapp`` returning webtest.response.TestResponse
     which has a ``json`` object property rather than a function.
 
+
 7.10.0
 ======
 

diff --git a/dcicutils/glacier_utils.py b/dcicutils/glacier_utils.py
@@ -58,6 +58,10 @@ def __init__(self, env_name: str):
         self.env_key = self.key_manager.get_keydict_for_env(env_name)
         self.health_page = get_health_page(key=self.env_key, ff_env=env_name)
 
+    @property
+    def kms_key_id(self) -> str:
+        return self.health_page.get("s3_encrypt_key_id", "")
+
     @classmethod
     def is_glacier_storage_class(cls, storage_class: S3StorageClass):
         return storage_class in S3_GLACIER_CLASSES
@@ -295,6 +299,9 @@ def _do_multipart_upload(self, bucket: str, key: str, total_size: int, part_size
             }
             if tags:
                 cmu['Tagging'] = tags
+            if self.kms_key_id:
+                cmu['ServerSideEncryption'] = 'aws:kms'
+                cmu['SSEKMSKeyId'] = self.kms_key_id
             mpu = self.s3.create_multipart_upload(**cmu)
             mpu_upload_id = mpu['UploadId']
         except Exception as e:
@@ -381,16 +388,21 @@ def copy_object_back_to_original_location(self, bucket: str, key: str, storage_c
             else:
                 # Force copy the object into standard in a single operation
                 copy_source = {'Bucket': bucket, 'Key': key}
-                copy_target = {
+                copy_args = {
                     'Bucket': bucket, 'Key': key,
                     'StorageClass': storage_class,
                 }
                 if version_id:
                     copy_source['VersionId'] = version_id
-                    copy_target['CopySourceVersionId'] = version_id
+                    copy_args['CopySourceVersionId'] = version_id
                 if tags:
-                    copy_target['Tagging'] = tags
-                response = self.s3.copy_object(CopySource=copy_source, **copy_target)
+                    copy_args['Tagging'] = tags
+                if self.kms_key_id:
+                    copy_args['ServerSideEncryption'] = 'aws:kms'
+                    copy_args['SSEKMSKeyId'] = self.kms_key_id
+                response = self.s3.copy_object(
+                    **copy_args, CopySource=copy_source
+                )
                 PRINT(f'Response from boto3 copy:\n{response}')
                 PRINT(f'Object {bucket}/{key} copied back to its original location in S3')
                 return response

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "7.11.0"
+version = "7.12.0"
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

diff --git a/test/test_glacier_utils.py b/test/test_glacier_utils.py
@@ -27,6 +27,7 @@ def mock_health_page() -> dict:
         'file_upload_bucket': 'cgap-dummy-main-application-cgap-dummy-files',
         'namespace': 'cgap-dummy',
         'processed_file_bucket': 'cgap-dummy-main-application-cgap-dummy-wfoutput',
+        's3_encrypt_key_id': 'dummy_kms_key',
     }