Merge pull request #308 from aws-quickstart/release/v0.8

Release taskcat [0.8.40]

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.36
+current_version = 0.8.40
 commit = True
 tag = False
 

README_v0.9_MIGRATION.md

@@ -0,0 +1 @@
+placeholder

bin/amiupdater

@@ -2,7 +2,7 @@
 import sys
 import os
 import argparse
-from taskcat.amiupdater import AMIUpdater, AMIUpdaterFatalException, AMIUpdaterCommitNeededException
+from taskcat.amiupdater import AMIUpdater, AMIUpdaterFatalException, AMIUpdaterCommitNeededException, AMIUpdaterNoFiltersException
 from taskcat.colored_console import PrintMsg
 
 def exit(code):
@@ -48,3 +48,5 @@ if __name__ == "__main__":
             exit(100)
         except AMIUpdaterFatalException:
             exit(1)
+        except AMIUpdaterNoFiltersException:
+            exit(0)

setup.py

@@ -20,7 +20,7 @@ def test_suite():
     url='https://aws-quickstart.github.io/taskcat/',
     license='Apache License 2.0',
     download_url='https://github.com/aws-quickstart/taskcat/tarball/master',
-    version = '0.8.36',
+    version = '0.8.40',
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',

taskcat/amiupdater.py

@@ -17,7 +17,14 @@
 
 class AMIUpdaterFatalException(TaskCatException):
     """Raised when AMIUpdater experiences a fatal error"""
-    pass
+    def __init__(self, message=None):
+        if message:
+            print("{} {}".format(PrintMsg.ERROR, message))
+
+class AMIUpdaterNoFiltersException(TaskCatException):
+    def __init__(self, message=None):
+        if message:
+            print("{} {}".format(PrintMsg.ERROR, message))
 
 class AMIUpdaterCommitNeededException(TaskCatException):
     pass
@@ -131,7 +138,7 @@ def fetch_latest_amis(cls):
         # Create a ThreadPool, size is the number of regions.
 
         if len(RegionalCodename.objects()) == 0:
-            raise AMIUpdaterFatalException("No AMI filters were found. Nothing to fetch from the EC2 API.")
+            raise AMIUpdaterNoFiltersException("No AMI filters were found. Nothing to fetch from the EC2 API.")
 
         pool = ThreadPool(len(TemplateClass.regions()))
         # For reach RegionalCodename that we've generated....
@@ -330,7 +337,7 @@ def _determine_regions(self):
                 self._regions.add(region)
 
     def set_region_ami(self, cn, region, ami_id):
-        currvalue = self._contents['Mappings']['AWSAMIRegionMap'].get(region, None).get(cn, None)
+        currvalue = self._mapping_root.get(region, {}).get(cn, None)
         if currvalue:
             if currvalue != ami_id:
                 self.set_updates(True)
@@ -411,7 +418,7 @@ def list_unknown_mappings(self):
             for unknown_map in unknown_mappings:
                 print(unknown_map)
 
-    def update_amis(self):
+    def update_amis(self, write=True):
         for template_file in self._fetch_template_files():
             # Loads each template as an object.
             TemplateObject(template_file)
@@ -425,16 +432,16 @@ def update_amis(self):
         Codenames.parse_api_results()
         print("{} API results parsed".format(PrintMsg.INFO))
 
-        updates = False
         for template_object in TemplateObject.objects():
             for result in APIResultsData.results:
                 template_object.set_region_ami(result.codename, result.region, result.ami_id)
 
         # For each template, write it to disk.
-        for template_object in TemplateObject.objects():
-            template_object.write()
-        print("{} Templates updated as necessary".format(PrintMsg.INFO))
-        print("{} Complete!".format(PrintMsg.INFO))
+        if write:
+            for template_object in TemplateObject.objects():
+                template_object.write()
+            print("{} Templates updated as necessary".format(PrintMsg.INFO))
+            print("{} Complete!".format(PrintMsg.INFO))
 
         if TemplateObject.updates:
             raise AMIUpdaterCommitNeededException

taskcat/common_utils.py

@@ -34,7 +34,7 @@ def parse_stack_info(self):
         """
         stack_info = dict()
         region_re = re.compile('(?<=:)(.\w-.+(\w*)-\d)(?=:)')
-        stack_name_re = re.compile('(?<=:stack/)(tCaT.*.)(?=/)')
+        stack_name_re = re.compile('(?<=:stack/)(tCaT.*.)(?=/)', re.IGNORECASE)
         stack_info['region'] = self.regxfind(region_re, self.stack_name)
         stack_info['stack_name'] = self.regxfind(stack_name_re, self.stack_name)
         return stack_info

taskcat/stacker.py

@@ -68,7 +68,7 @@
     _run_mode = 0
 
 version = __version__
-sig = base64.b64decode("dENhVA==").decode()
+sig = base64.b64decode("dGNhdA==").decode()
 jobid = str(uuid.uuid4())
 
 '''
@@ -429,33 +429,31 @@ def stage_in_s3(self, taskcat_cfg):
                 auto_bucket = auto_bucket[:self._max_bucket_name_length]
             if self.get_default_region():
                 print('{0}Creating bucket {1} in {2}'.format(PrintMsg.INFO, auto_bucket, self.get_default_region()))
-                if self.get_default_region() == 'us-east-1':
-                    response = s3_client.create_bucket(ACL=bucket_or_object_acl,
-                                                       Bucket=auto_bucket)
-                else:
-                    response = s3_client.create_bucket(ACL=bucket_or_object_acl,
-                                                       Bucket=auto_bucket,
-                                                       CreateBucketConfiguration={
-                                                           'LocationConstraint': self.get_default_region()
-                                                       })
 
-                self.set_s3bucket_type('auto')
-            else:
-                raise TaskCatException("Default_region = " + self.get_default_region())
+                try:
+                    if self.get_default_region() == 'us-east-1':
+                        s3_client.create_bucket(ACL=bucket_or_object_acl,
+                                                           Bucket=auto_bucket)
+                    else:
+                        s3_client.create_bucket(ACL=bucket_or_object_acl,
+                                                           Bucket=auto_bucket,
+                                                           CreateBucketConfiguration={
+                                                               'LocationConstraint': self.get_default_region()
+                                                           })
 
-            if response['ResponseMetadata']['HTTPStatusCode'] is 200:
-                print(PrintMsg.INFO + "Staging Bucket => [%s]" % auto_bucket)
-                self.set_s3bucket(auto_bucket)
-            else:
-                print('{0}Creating bucket {1} in {2}'.format(PrintMsg.INFO, auto_bucket, self.get_default_region()))
-                response = s3_client.create_bucket(ACL=bucket_or_object_acl,
-                                                   Bucket=auto_bucket,
-                                                   CreateBucketConfiguration={
-                                                       'LocationConstraint': self.get_default_region()})
+                    s3_client.get_waiter('bucket_exists').wait(Bucket=auto_bucket)
 
-                if response['ResponseMetadata']['HTTPStatusCode'] is 200:
+                    self.set_s3bucket_type('auto')
                     print(PrintMsg.INFO + "Staging Bucket => [%s]" % auto_bucket)
                     self.set_s3bucket(auto_bucket)
+                except Exception as e:
+                    print(PrintMsg.ERROR + "Bucket (%s) creation failed" % auto_bucket)
+                    if self.verbose:
+                        print(PrintMsg.DEBUG + str(e))
+                    raise
+            else:
+                raise TaskCatException("Default_region = " + self.get_default_region())
+
             if self.tags:
                 s3_client.put_bucket_tagging(
                     Bucket=auto_bucket,
@@ -1664,7 +1662,14 @@ def interface(self):
     def checkforupdate(silent=False):
         update_needed = False
 
-        def _print_upgrade_msg(newversion):
+        def _print_upgrade_msg(newversion, nine=False):
+            if nine:
+                print("{} !!!!!".format(PrintMsg.INFO))
+                print("{} Breaking changes have been introduced to the taskcat CLI arguments in the v0.9.x branch.".format(PrintMsg.INFO))
+                print("{} Please review the migration document *BEFORE* upgrading to v0.9.x!".format(PrintMsg.INFO))
+                print("{} - https://github.com/aws-quickstart/taskcat/blob/master/README_v0.9_MIGRATION.md".format(PrintMsg.INFO))
+                print("{} !!!!!".format(PrintMsg.INFO))
+                print('\n')
             print("version %s" % version)
             print('\n')
             print("{} A newer version of {} is available ({})".format(
@@ -1676,16 +1681,19 @@ def _print_upgrade_msg(newversion):
             print('\n')
 
         if _run_mode > 0:
+            nine=False
             if 'dev' not in version:
                 current_version = get_pip_version(
                     'https://pypi.org/pypi/taskcat/json')
                 if version not in current_version:
                     update_needed = True
+                if '0.9' in current_version:
+                    nine=True
             if not silent:
                 if not update_needed:
                     print("version %s" % version)
                 else:
-                    _print_upgrade_msg(current_version)
+                    _print_upgrade_msg(current_version, nine)
         else:
             print(PrintMsg.INFO + "Using local source (development mode) \n")
 

tests/unittest/test_amiupdater.py

@@ -794,15 +794,17 @@ def describe_images(self, Filters):
 
 class TestAMIUpdater(unittest.TestCase):
 
-    def _module_loader(self, return_module=False, commit_needed=False):
+    def _module_loader(self, return_module=False, commit_needed=False, no_filters=False):
         try:
             del sys.modules['taskcat.amiupdater']
         except KeyError:
             pass
-        from taskcat.amiupdater import AMIUpdater, AMIUpdaterFatalException, AMIUpdaterCommitNeededException
+        from taskcat.amiupdater import AMIUpdater, AMIUpdaterFatalException, AMIUpdaterCommitNeededException, AMIUpdaterNoFiltersException
         module_tuple = (AMIUpdater, AMIUpdaterFatalException)
         if commit_needed:
             module_tuple += (AMIUpdaterCommitNeededException,)
+        if no_filters:
+            module_tuple += (AMIUpdaterNoFiltersException,)
         if return_module:
             import taskcat.amiupdater
             module_tuple += (taskcat.amiupdater,)
@@ -871,6 +873,37 @@ def _module_loader(self, return_module=False, commit_needed=False):
             }
         }
     }
+    nonstandard_mapping_and_filter_skeleton_template = {
+        "Filters": {
+            "NON_STANDARD_TEST":{
+                "name": "amzn-ami-hvm-????.??.?.*-x86_64-gp2",
+                "owner-alias": "amazon"
+            }
+        },
+        "Images":{
+            "us-east-1":{
+                "AMZNLINUXHVM": "FOOBAR",
+                "AMZNLINUXHVM_CUSTOM_CONFIG":"FOOBAR",
+                "NON_STANDARD_TEST":"FOOBAR"
+            },
+            "us-east-2":{
+                "AMZNLINUXHVM": "FOOBAR",
+                "AMZNLINUXHVM_CUSTOM_CONFIG":"FOOBAR",
+                "NON_STANDARD_TEST":"FOOBAR"
+            },
+            "us-west-1":{
+                    "AMZNLINUXHVM": "FOOBAR",
+                    "AMZNLINUXHVM_CUSTOM_CONFIG": "FOOBAR",
+                    "NON_STANDARD_TEST": "FOOBAR"
+            },
+            "us-west-2":{
+                    "AMZNLINUXHVM": "FOOBAR",
+                    "AMZNLINUXHVM_CUSTOM_CONFIG": "FOOBAR",
+                    "NON_STANDARD_TEST": "FOOBAR"
+            }
+        }
+    }
+
 
     no_mapping_skeleton_template = {
         "Mappings":{
@@ -933,6 +966,8 @@ def create_ephemeral_template(self, template_type="generic"):
             data = self.generic_skeleton_template
         elif template_type == "inline":
             data = self.inline_skeleton_template
+        elif template_type == "nonstandard_inline":
+            data = self.nonstandard_mapping_and_filter_skeleton_template
         elif template_type == "no_mapping":
             data = self.no_mapping_skeleton_template
         elif template_type == "invalid_region":
@@ -1010,6 +1045,29 @@ def test_local_config_ALAMI(self):
                     with self.subTest(i="Verifying Updated AMI: [{}] / [{}]".format(mapping_name, region)):
                         self.assertRegex(ami_id, self.ami_regex_pattern)
 
+    def test_unconventional_roots(self):
+        au, AMIUpdaterFatalException, AMIUpdaterCommitNeededException, tcau  = self._module_loader(commit_needed=True, return_module=True)
+        cf = self.client_factory_handler()
+        mapping_name = "NON_STANDARD_TEST"
+        template_file = self.create_ephemeral_template(template_type="nonstandard_inline")
+        amiupdater_args = {
+            "path_to_templates": template_file,
+            "use_upstream_mappings": False,
+            "client_factory": cf
+        }
+        with self.assertRaises(AMIUpdaterCommitNeededException):
+            tcau.TemplateClass.mapping_path = "Images"
+            tcau.TemplateClass.metadata_path = "Filters"
+            a = au(**amiupdater_args)
+            a.update_amis()
+
+        template_result = self.load_modified_template(template_file)
+        for region, mapping_data in template_result["Images"].items():
+            for codename, ami_id in mapping_data.items():
+                if codename == mapping_name:
+                    with self.subTest(i="Verifying Updated AMI: [{}] / [{}]".format(mapping_name, region)):
+                        self.assertRegex(ami_id, self.ami_regex_pattern)
+
     def test_in_template_ALAMI(self):
         au, AMIUpdaterFatalException, AMIUpdaterCommitNeededException  = self._module_loader(commit_needed=True)
         cf = self.client_factory_handler()
@@ -1045,7 +1103,7 @@ def test_invalid_region_exception(self):
         self.assertRaises(AMIUpdaterFatalException, a.update_amis)
 
     def test_no_filters_exception(self):
-        au, AMIUpdaterFatalException = self._module_loader()
+        au, _, AMIUpdaterNoFiltersException = self._module_loader(no_filters=True)
         cf = self.client_factory_handler()
         template_file = self.create_ephemeral_template()
         amiupdater_args = {
@@ -1054,7 +1112,7 @@ def test_no_filters_exception(self):
             "client_factory": cf
         }
         a = au(**amiupdater_args)
-        self.assertRaises(AMIUpdaterFatalException, a.update_amis)
+        self.assertRaises(AMIUpdaterNoFiltersException, a.update_amis)
 
     def test_APIResults_lessthan_comparison_standard(self):
         from taskcat.amiupdater import APIResultsData