Merge pull request #2358 from aws/release/v1.43.0

Release/v1.43.0

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,10 +6,11 @@
 
 *Checklist:*
 
-- [ ] Add/update tests using:
+- [ ] Add/update [unit tests](https://github.com/aws/serverless-application-model/blob/develop/DEVELOPMENT_GUIDE.md#unit-testing-with-multiple-python-versions) using:
     - [ ] Correct values
     - [ ] Bad/wrong values (None, empty, wrong type, length, etc.)
     - [ ] [Intrinsic Functions](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference.html)
+- [ ] Add/update [integration tests](https://github.com/aws/serverless-application-model/blob/develop/INTEGRATION_TESTS.md)
 - [ ] `make pr` passes
 - [ ] Update documentation
 - [ ] Verify transformed template deploys and application functions as expected

.github/workflows/README.md

@@ -0,0 +1,7 @@
+This folder has Github Actions for this repo. 
+
+** pr-labler **
+
+This is responsible for tagging our prs automattically. The primary thing it does is tags internal vs external (to the team) PRs.
+This is run on `pull_request_target` which only runs what is in the repo not what is in the Pull Request. This is done to help guard against
+a PR running and changing. For this, the Action should NEVER download or checkout the PR. It is purely for tagging/labeling not CI.

.github/workflows/pr-labeler.yml

@@ -0,0 +1,31 @@
+name: "Pull Request Labeler"
+on:
+  pull_request_target:
+    types: [opened]
+
+jobs:
+  apply-internal-external-label:
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v5
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const maintainers = ['jfuss', 'c2tarun', 'hoffa', 'awood45', 'CoshUS', 'aahung', 'hawflau', 'mndeveci', 'ssenchenko', 'wchengru', 'mingkun2020', 'qingchm', 'moelasmar', 'xazhao', 'mildaniel', 'marekaiv', 'torresxb1']
+            if (maintainers.includes(context.payload.sender.login)) {
+              github.rest.issues.addLabels({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                labels: ['pr/internal']
+              })
+            } else {
+              github.rest.issues.addLabels({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                labels: ['pr/external']
+              })
+            }

.pylintrc

@@ -9,16 +9,21 @@
 
 # Add files or directories to the ignore list. They should be base names, not
 # paths.
-ignore=compat.py
+ignore=compat.py, __main__.py
 
 # Pickle collected data for later comparisons.
 persistent=yes
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
 load-plugins=
+    pylint.extensions.check_elif, # Else If Used checker
+    pylint.extensions.emptystring, # compare to empty string
+    pylint.extensions.comparetozero, # compare to 0
+    pylint.extensions.docparams, # Parameter documentation checker
 
 # Use multiple processes to speed up Pylint.
+# Pylint has a bug on multitread, turn it off.
 jobs=1
 
 # Allow loading of arbitrary C extensions. Extensions are imported into the
@@ -59,7 +64,103 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=R0201,W0613,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904
+disable=
+    W0613, # Unused argument %r
+    W0640, # Cell variable %s defined in loop A variable used in a closure is defined in a loop
+    R0902, # Too many instance attributes (%s/%s)
+    R0903, # Too few public methods (%s/%s)
+    R0913, # Too many arguments (%s/%s)
+    W0703, # Catching too general exception %s
+    R0904, # Too many public methods (%s/%s)
+    R0914, # Too many local variables (%s/%s)
+    R0915, # Too many statements
+    C0415, # Import outside toplevel (%s) Used when an import statement is used anywhere other than the module toplevel. Move this import to the top of the file.
+    C0115, # missing-class-docstring
+    # below are docstring lint rules, in order to incrementally improve our docstring while
+    # without introduce too much effort at a time, we exclude most of the docstring lint rules.
+    # We will remove them one by one.
+    # W9005, # "%s" has constructor parameters documented in class and __init__
+    W9006, # "%s" not documented as being raised
+    # W9008, # Redundant returns documentation
+    # W9010, # Redundant yields documentation
+    W9011, # Missing return documentation
+    W9012, # Missing return type documentation
+    # W9013, # Missing yield documentation
+    # W9014, # Missing yield type documentation
+    # W9015, # "%s" missing in parameter documentation
+    W9016, # "%s" missing in parameter type documentation
+    # W9017, # "%s" differing in parameter documentation
+    # W9018, # "%s" differing in parameter type documentation
+    # W9019, # "%s" useless ignored parameter documentation
+    # W9020, # "%s" useless ignored parameter type documentation
+    # Constant name style warnings. We will remove them one by one.
+    C0103, # Class constant name "%s" doesn't conform to UPPER_CASE naming style ('([^\\W\\da-z][^\\Wa-z]*|__.*__)$' pattern) (invalid-name)
+    # New recommendations, disable for now to avoid introducing behaviour changes.
+    R1729, # Use a generator instead '%s' (use-a-generator)
+    R1732, # Consider using 'with' for resource-allocating operations (consider-using-with)
+    # This applies to CPython only.
+    I1101, # Module 'math' has no 'pow' member, but source is unavailable. Consider adding this module to extension-pkg-allow-list if you want to perform analysis based on run-time introspection of living objects. (c-extension-no-member)    
+    # Added to allow linting. Need to work on removing over time
+    R0801, # dupilcated code
+    R0401, # Cyclic import
+    C0411, # import ordering
+    W9015, # missing parameter in doc strings
+    W0612, # unused variable
+    R0205, # useless-object-inheritanc
+    C0301, # line to long
+    R0201, # no self use, method could be a function
+    C0114, # missing-module-docstring
+    W1202, # Use lazy % formatting in logging functions (logging-format-interpolation)
+    E1101, # No member 
+    W0622, # Redefining built-in 'property' (redefined-builtin)
+    W0611, # unused imports
+    W0231, # super not called
+    W0212, # protected-access
+    W0201, # attribute-defined-outside-init
+    R1725, # Consider using Python 3 style super() without arguments (super-with-arguments)
+    C2001, # Avoid comparisons to zero (compare-to-zero)
+    R0912, # too many branches
+    W0235, # Useless super delegation in method '__init__' (useless-super-delegation)
+    C0412, # Imports from package samtranslator are not grouped (ungrouped-imports)
+    W0223, # abstract-method
+    W0107, # unnecessary-pass
+    W0707, # raise-missing-from
+    R1720, # no-else-raise
+    W0621, # redefined-outer-name
+    E0203, # access-member-before-definition
+    W0221, # arguments-differ
+    R1710, # inconsistent-return-statements
+    R1702, # too-many-nested-blocks
+    C0123, # Use isinstance() rather than type() for a typecheck. (unidiomatic-typecheck)
+    W0105, # String statement has no effect (pointless-string-statement)
+    C0206, # Consider iterating with .items() (consider-using-dict-items)
+    W9008, # Redundant returns documentation (redundant-returns-doc)
+    E0602, # undefined-variable
+    C0112, # empty-docstring
+    C0116, # missing-function-docstring
+    C0200, # consider-using-enumerate
+    R5501, # Consider using "elif" instead of "else if" (else-if-used)
+    W9017, # differing-param-doc
+    W9018, # differing-type-doc
+    W0511, # fixme
+    C0325, # superfluous-parens
+    R1701, # consider-merging-isinstance
+    C0302, # too-many-lines
+    C0303, # trailing-whitespace
+    R1721, # unnecessary-comprehension
+    C0121, # singleton-comparison
+    E1305, # too-many-format-args
+    R1724, # no-else-continue
+    E0611, # no-name-in-module
+    W9013, # missing-yield-doc
+    W9014, # missing-yield-type-doc
+    C0201, # consider-iterating-dictionary
+    W0237, # arguments-renamed
+    R1718, # consider-using-set-comprehension
+    R1723, # no-else-break
+    E1133, # not-an-iterable
+    E1135, # unsupported-membership-test
+    R1705, # no-else-return
 
 
 [REPORTS]
@@ -162,18 +263,20 @@ module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 
 # Regular expression matching correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
+method-rgx=[a-z_][a-z0-9_]{2,50}$
 
 # Naming hint for method names
 method-name-hint=[a-z_][a-z0-9_]{2,30}$
 
 # Regular expression which should only match function or class names that do
 # not require a docstring.
-no-docstring-rgx=.*
+no-docstring-rgx=^_
 
 # Minimum line length for functions/classes that require docstrings, shorter
 # ones are exempt.
-docstring-min-length=-1
+# To improve our docstring without spending too much effort at a time,
+# here set it to 30 and decrease it gradually in the future.
+docstring-min-length=30
 
 
 [FORMAT]
@@ -221,16 +324,16 @@ notes=FIXME,XXX
 [SIMILARITIES]
 
 # Minimum lines number of a similarity.
-min-similarity-lines=5
+min-similarity-lines=12
 
 # Ignore comments when computing similarities.
-ignore-comments=no
+ignore-comments=yes
 
 # Ignore docstrings when computing similarities.
-ignore-docstrings=no
+ignore-docstrings=yes
 
 # Ignore imports when computing similarities.
-ignore-imports=no
+ignore-imports=yes
 
 
 [SPELLING]
@@ -259,7 +362,7 @@ ignore-mixin-members=yes
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis
-ignored-modules=six.moves
+ignored-modules=
 
 # List of classes names for which member attributes should not be checked
 # (useful for classes with attributes dynamically set).
@@ -315,7 +418,7 @@ max-args=5
 ignored-argument-names=_.*
 
 # Maximum number of locals for function / method body
-max-locals=15
+max-locals=17
 
 # Maximum number of return / yield for function / method body
 max-returns=6

DEVELOPMENT_GUIDE.md

@@ -15,7 +15,7 @@ Windows users, consider using [pipenv](https://docs.pipenv.org/).
 -------------------------
 For setting up a local development environment, we recommend using Gitpod - a service that allows you to spin up an in-browser Visual Studio Code-compatible editor, with everything set up and ready to go for development on this project. Just click the button below to create your private workspace:
 
-[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/awslabs/aws-sam-cli)
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/aws/serverless-application-model.git)
 
 This will start a new Gitpod workspace, and immediately kick off a build of the code. Once it's done, you can start working.
 
@@ -26,9 +26,8 @@ Environment Setup
 -----------------
 ### 1. Install Python Versions
 
-Our officially supported Python versions are 2.7, 3.6, 3.7 and 3.8. Follow the idioms from this [excellent cheatsheet](http://python-future.org/compatible_idioms.html) to
-make sure your code is compatible with both Python 2.7 and 3 (>=3.6) versions.
-Our CI/CD pipeline is setup to run unit tests against both Python 2.7 and 3 versions. So make sure you test it with both versions before sending a Pull Request.
+Our officially supported Python versions are 3.6, 3.7 and 3.8. 
+Our CI/CD pipeline is setup to run unit tests against Python 3 versions. Make sure you test it before sending a Pull Request.
 See [Unit testing with multiple Python versions](#unit-testing-with-multiple-python-versions).
 
 [pyenv](https://github.com/pyenv/pyenv) is a great tool to
@@ -41,12 +40,11 @@ easily setup multiple Python versions. For
 1.  Install PyEnv -
     `curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash`
 1. Restart shell so the path changes take effect - `exec $SHELL`
-1.  `pyenv install 2.7.17`
 1.  `pyenv install 3.6.12`
 1.  `pyenv install 3.7.9`
 1.  `pyenv install 3.8.6`
 1.  Make Python versions available in the project:
-    `pyenv local 2.7.17 3.6.12 3.7.9 3.8.6`
+    `pyenv local 3.6.12 3.7.9 3.8.6`
 
 Note: also make sure the following lines were written into your `.bashrc` (or `.zshrc`, depending on which shell you are using):
 ```
@@ -117,11 +115,10 @@ Running Tests
 ### Unit testing with one Python version
 
 If you're trying to do a quick run, it's ok to use the current python version.  Run `make pr`.
-If you're using Python2.7, you can run `make pr2.7` instead.
 
 ### Unit testing with multiple Python versions
 
-Currently, our officially supported Python versions are 2.7, 3.6, 3.7 and 3.8. For the most
+Currently, our officially supported Python versions are 3.6, 3.7 and 3.8. For the most
 part, code that works in Python3.6 will work in Python3.7 and Python3.8. You only run into problems if you are
 trying to use features released in a higher version (for example features introduced into Python3.7
 will not work in Python3.6). If you want to test in many versions, you can create a virtualenv for

INTEGRATION_TESTS.md

@@ -2,7 +2,7 @@
 
 These tests run SAM against AWS services by translating SAM templates, deploying them to Cloud Formation and verifying the resulting objects.
 
-They must run successfully under Python 2 and 3.
+They must run successfully under Python 3.
 
 ## Run the tests
 

MANIFEST.in

@@ -5,5 +5,6 @@ recursive-include samtranslator/validator/sam_schema *.json
 include samtranslator/policy_templates_data/policy_templates.json
 include samtranslator/policy_templates_data/schema.json
 include README.md
+include THIRD_PARTY_LICENSES
 
 prune tests

Makefile

@@ -6,7 +6,7 @@ init:
 	pip install -e '.[dev]'
 
 test:
-	pytest --cov samtranslator --cov-report term-missing --cov-fail-under 95 tests/*
+	pytest --cov samtranslator --cov-report term-missing --cov-fail-under 95 -n auto tests/*
 
 test-cov-report:
 	pytest --cov samtranslator --cov-report term-missing --cov-report html --cov-fail-under 95 tests/*
@@ -20,14 +20,15 @@ black:
 black-check:
 	black --check setup.py samtranslator/* tests/* integration/* bin/*.py
 
+lint:
+	# Linter performs static analysis to catch latent bugs
+	pylint --rcfile .pylintrc samtranslator
+
 # Command to run everytime you make changes to verify everything works
 dev: test
 
 # Verifications to run before sending a pull request
-pr: black-check init dev
-
-# Verifications to run before sending a pull request, skipping black check because black requires Python 3.6+
-pr2.7: init dev
+pr: black-check lint init dev
 
 define HELP_MESSAGE
 

THIRD_PARTY_LICENSES

@@ -0,0 +1,23 @@
+The AWS Serverless Application Model includes the following third-party software/licensing:
+
+** py27hash; version 1.0.2 -- https://pypi.org/project/py27hash/
+Copyright (c) 2020 NeuML LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+----------------

appveyor-integration-test.yml

@@ -3,14 +3,14 @@ image: Ubuntu
 
 environment:
   matrix:
-  - TOXENV: py27
-    PYTHON_VERSION: '2.7'
-  - TOXENV: py36
-    PYTHON_VERSION: '3.6'
   - TOXENV: py37
     PYTHON_VERSION: '3.7'
   - TOXENV: py38
     PYTHON_VERSION: '3.8'
+  - TOXENV: py39
+    PYTHON_VERSION: '3.9'
+  - TOXENV: py310
+    PYTHON_VERSION: '3.10'
 
 build: off
 

appveyor.yml

@@ -3,14 +3,14 @@ image: Ubuntu
 
 environment:
   matrix:
-  - TOXENV: py27
-    PYTHON_VERSION: '2.7'
-  - TOXENV: py36
-    PYTHON_VERSION: '3.6'
   - TOXENV: py37
     PYTHON_VERSION: '3.7'
   - TOXENV: py38
     PYTHON_VERSION: '3.8'
+  - TOXENV: py39
+    PYTHON_VERSION: '3.9'
+  - TOXENV: py310
+    PYTHON_VERSION: '3.10'
 
 build: off