Merge pull request #769 from aws-ia/gitignore_syntax

Migrating to pathspec (gitignore syntax) method for excludes

THIRD_PARTY

@@ -0,0 +1,8 @@
+  Copyright 2022, Caleb P. Burns  <2126043+cpburnz@users.noreply.github.com>
+
+  SPDX-License-Identifier: MPL-2.0
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+  https://github.com/cpburnz/python-pathspec/releases/tag/v0.10.3

requirements.txt

@@ -1,3 +1,4 @@
+pathspec==0.10.3
 reprint
 tabulate>=0.8.2,<1.0
 cfn_lint>=0.13.0,<1.0

taskcat/_s3_sync.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import time
+import pathspec
 from functools import partial
 from multiprocessing.dummy import Pool as ThreadPool
 from typing import List
@@ -47,10 +48,14 @@ def __init__(self, s3_client, bucket, prefix, path, acl="private", dry_run=False
             prefix = prefix + "/"
         self.s3_client = s3_client
         self.dry_run = dry_run
+        self.exclude_patterns = pathspec.PathSpec.from_lines('gitwildmatch', self.exclude_files + self.exclude_path_prefixes + self.exclude_remote_path_prefixes)
         file_list = self._get_local_file_list(path)
         s3_file_list = self._get_s3_file_list(bucket, prefix)
         self._sync(file_list, s3_file_list, bucket, prefix, acl=acl)
 
+    def _exclude_via_gitignore_syntax(self, file_path):
+        return self.exclude_patterns.match_file(file_path)
+
     @staticmethod
     def _hash_file(file_path, chunk_size=8 * 1024 * 1024):
         # This is a bit funky because of the way multipart upload etags are done, they
@@ -87,10 +92,8 @@ def _get_local_file_list(
             if relpath == "./":
                 relpath = ""
             # exclude defined paths
-            for prefix in S3Sync.exclude_path_prefixes:
-                if relpath.startswith(prefix):
-                    exclude_path = True
-                    break
+            if self._exclude_via_gitignore_syntax(relpath):
+                exclude_path = True
             if not exclude_path:
                 file_list.update(
                     self._iterate_files(files, root, include_checksums, relpath)
@@ -143,14 +146,9 @@ def _get_s3_file_list(self, bucket, prefix):
                 is_paginated = False
         return objects
 
-    @staticmethod
-    def _exclude_remote(path):
-        keep = False
-        for exclude in S3Sync.exclude_remote_path_prefixes:
-            if path.startswith(exclude):
-                keep = True
-                break
-        return keep
+
+    def _exclude_remote(self, path):
+        return self._exclude_via_gitignore_syntax(path)
 
     # TODO: refactor
     def _sync(  # noqa: C901