Safe tag retrieval for rds, cluster and docdb (#151)

deploy-sam-template.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-SHELVERY_VERSION=0.9.10
+SHELVERY_VERSION=0.9.11
 
 # set DOCKERUSERID to current user. could be changed with -u uid
 DOCKERUSERID="-u $(id -u)"

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-setup(name='shelvery', version='0.9.10', author='Base2Services R&D',
+setup(name='shelvery', version='0.9.11', author='Base2Services R&D',
       author_email='itsupport@base2services.com',
       url='http://github.com/base2Services/shelvery-aws-backups',
       classifiers=[

shelvery/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.9.10'
+__version__ = '0.9.11'
 LAMBDA_WAIT_ITERATION = 'lambda_wait_iteration'
 S3_DATA_PREFIX = 'backups'
 SHELVERY_DO_BACKUP_TAGS = ['True', 'true', '1', 'TRUE']

shelvery/documentdb_backup.py

@@ -167,7 +167,7 @@ def get_backup_resource(self, backup_region: str, backup_id: str) -> BackupResou
                                               external_id=self.role_external_id)
         snapshots = docdb_client.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
         snapshot = snapshots['DBClusterSnapshots'][0]
-        tags = docdb_client.list_tags_for_resource(ResourceName=snapshot['DBClusterSnapshotArn'])['TagList']
+        tags = docdb_client.list_tags_for_resource(ResourceName=snapshot['DBClusterSnapshotArn']).get('TagList', [])
         d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
         resource = BackupResource.construct(d_tags['shelvery:tag_name'], backup_id, d_tags)
         resource.resource_properties = snapshot
@@ -189,7 +189,7 @@ def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
         # collect tags in check if instance tagged with marker tag
 
         for instance in db_clusters:
-            tags = docdb_client.list_tags_for_resource(ResourceName=instance['DBClusterArn'])['TagList']
+            tags = docdb_client.list_tags_for_resource(ResourceName=instance['DBClusterArn']).get('TagList', [])
 
             # convert api response to dictionary
             d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
@@ -239,7 +239,7 @@ def get_shelvery_backups_only(self, all_snapshots, backup_tag_prefix, docdb_clie
         all_backups = []
         marker_tag = f"{backup_tag_prefix}:{BackupResource.BACKUP_MARKER_TAG}"
         for snap in all_snapshots:
-            tags = docdb_client.list_tags_for_resource(ResourceName=snap['DBClusterSnapshotArn'])['TagList']
+            tags = docdb_client.list_tags_for_resource(ResourceName=snap['DBClusterSnapshotArn']).get('TagList', [])
             self.logger.info(f"Checking DocumentDb Snapshot {snap['DBClusterSnapshotIdentifier']}")
             d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
             if marker_tag in d_tags:
@@ -291,7 +291,7 @@ def populate_snap_entity_resource(self, all_snapshots):
             try:
                 self.logger.info(f"Collecting tags from DB cluster {cluster_id} ...")
                 docdb_instance = docdb_client.describe_db_clusters(DBClusterIdentifier=cluster_id)['DBClusters'][0]
-                tags = docdb_client.list_tags_for_resource(ResourceName=docdb_instance['DBClusterArn'])['TagList']
+                tags = docdb_client.list_tags_for_resource(ResourceName=docdb_instance['DBClusterArn']).get('TagList', [])
                 d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
                 docdb_entity = EntityResource(cluster_id,
                                               local_region,

shelvery/rds_backup.py

@@ -153,7 +153,7 @@ def get_backup_resource(self, backup_region: str, backup_id: str) -> BackupResou
         rds_client = AwsHelper.boto3_client('rds', region_name=backup_region, arn=self.role_arn, external_id=self.role_external_id)
         snapshots = rds_client.describe_db_snapshots(DBSnapshotIdentifier=backup_id)
         snapshot = snapshots['DBSnapshots'][0]
-        tags = snapshot['TagList']
+        tags = snapshot.get('TagList', [])
         d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
         resource = BackupResource.construct(d_tags['shelvery:tag_name'], backup_id, d_tags)
         resource.resource_properties = snapshot
@@ -174,7 +174,7 @@ def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
 
         for instance in db_instances:
             # collect tags in check if instance tagged with marker tag
-            tags = instance['TagList']
+            tags = instance.get('TagList', [])
             # convert api response to dictionary
             d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
             if 'DBClusterIdentifier' in instance:
@@ -222,7 +222,7 @@ def get_shelvery_backups_only(self, all_snapshots, backup_tag_prefix, rds_client
 
         for snap in all_snapshots:
             #collect tags
-            tags = snap['TagList']
+            tags = snap.get('TagList', [])
             d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
             self.logger.info(f"Checking RDS Snap {snap['DBSnapshotIdentifier']}")
 
@@ -300,7 +300,7 @@ def populate_snap_entity_resource(self, all_snapshots):
         for instance_id in instance_ids:
             try:
                 rds_instance = rds_client.describe_db_instances(DBInstanceIdentifier=instance_id)['DBInstances'][0]
-                tags = rds_client.list_tags_for_resource(ResourceName=rds_instance['DBInstanceArn'])['TagList']
+                tags = rds_client.list_tags_for_resource(ResourceName=rds_instance['DBInstanceArn']).get('TagList', [])
                 d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
                 d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
                 rds_entity = EntityResource(instance_id,

shelvery/rds_cluster_backup.py

@@ -190,7 +190,7 @@ def get_backup_resource(self, backup_region: str, backup_id: str) -> BackupResou
         rds_client = AwsHelper.boto3_client('rds', region_name=backup_region, arn=self.role_arn, external_id=self.role_external_id)
         snapshots = rds_client.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
         snapshot = snapshots['DBClusterSnapshots'][0]
-        tags = snapshot['TagList']
+        tags = snapshot.get('TagList', [])
         d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
         resource = BackupResource.construct(d_tags['shelvery:tag_name'], backup_id, d_tags)
         resource.resource_properties = snapshot
@@ -212,7 +212,7 @@ def get_entities_to_backup(self, tag_name: str) -> List[EntityResource]:
         # collect tags in check if instance tagged with marker tag
 
         for instance in db_clusters:
-            tags = instance['TagList']
+            tags = instance.get('TagList', [])
 
             # convert api response to dictionary
             d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
@@ -256,7 +256,7 @@ def get_shelvery_backups_only(self, all_snapshots, backup_tag_prefix, rds_client
         all_backups = []
         marker_tag = f"{backup_tag_prefix}:{BackupResource.BACKUP_MARKER_TAG}"
         for snap in all_snapshots:
-            tags = snap['TagList']
+            tags = snap.get('TagList', [])
             self.logger.info(f"Checking RDS Snap {snap['DBClusterSnapshotIdentifier']}")
             d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
             if marker_tag in d_tags:
@@ -307,7 +307,7 @@ def populate_snap_entity_resource(self, all_snapshots):
             try:
                 self.logger.info(f"Collecting tags from DB cluster {cluster_id} ...")
                 rds_instance = rds_client.describe_db_clusters(DBClusterIdentifier=cluster_id)['DBClusters'][0]
-                tags = rds_instance['TagList']
+                tags = rds_instance.get('TagList', []) 
                 d_tags = dict(map(lambda t: (t['Key'], t['Value']), tags))
                 rds_entity = EntityResource(cluster_id,
                                             local_region,

template.yaml

@@ -122,7 +122,7 @@ Resources:
       Tags:
         Name: Shelvery
         CreatedBy: Shelvery
-        ShelveryVersion: 0.9.10
+        ShelveryVersion: 0.9.11
 
       Environment:
         Variables: