From 6a64a512e40ba6a3cc0e6788fc67207f3e77ebd2 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 3 May 2019 16:49:35 -0400 Subject: [PATCH 001/417] Add a configuration file for Bandit, and configure pre-commit to use it. --- .bandit.yml | 12 ++++++++++++ .pre-commit-config.yaml | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 .bandit.yml diff --git a/.bandit.yml b/.bandit.yml new file mode 100644 index 0000000..f122c97 --- /dev/null +++ b/.bandit.yml @@ -0,0 +1,12 @@ +# Configuration file for the Bandit python security scanner +# https://bandit.readthedocs.io/en/latest/config.html + +# Test are first included by `tests`, and then excluded by `skips`. +# If `tests` is empty, all tests are are considered included. + +tests: + #- B101 + #- B102 + +skips: + #- B101 # skip "assert used" check since assertions are required in pytests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb12c99..2a03327 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,6 +52,8 @@ repos: rev: 2a1dbab hooks: - id: bandit + args: + - --config=.bandit.yml - repo: https://github.com/ambv/black rev: 19.3b0 hooks: From 1a98e0ed062cc55719893bbbeafc3df40510b42d Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 3 May 2019 16:51:21 -0400 Subject: [PATCH 002/417] Improve english --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index f122c97..46bab94 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,7 +1,7 @@ # Configuration file for the Bandit python security scanner # https://bandit.readthedocs.io/en/latest/config.html -# Test are first included by `tests`, and then excluded by `skips`. +# Tests are first included by `tests`, and then excluded by `skips`. # If `tests` is empty, all tests are are considered included. tests: From 19afa70f67a10f14d76b9132f508849146b35656 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 6 May 2019 10:44:34 -0400 Subject: [PATCH 003/417] Run pre-commit autoupdate Also update the URL for black since it moved GitHub orgs from ambv to python. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a03327..0bac8c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.1.0 + rev: v2.2.1 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.14.1 + rev: v0.15.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -45,7 +45,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.14.0 + rev: v1.16.3 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -54,7 +54,7 @@ repos: - id: bandit args: - --config=.bandit.yml - - repo: https://github.com/ambv/black + - repo: https://github.com/python/black rev: 19.3b0 hooks: - id: black @@ -74,6 +74,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.16.2 + rev: 1.17.0 hooks: - id: prettier From eb4fbd751b57db9c728e7b3065edbfde50633d97 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 6 May 2019 11:16:05 -0400 Subject: [PATCH 004/417] Make .bandit.yml pass yamllint --- .bandit.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 46bab94..8ba42d1 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,3 +1,4 @@ +--- # Configuration file for the Bandit python security scanner # https://bandit.readthedocs.io/en/latest/config.html @@ -5,8 +6,8 @@ # If `tests` is empty, all tests are are considered included. tests: - #- B101 - #- B102 +# - B101 +# - B102 skips: - #- B101 # skip "assert used" check since assertions are required in pytests +# - B101 # skip "assert used" check since assertions are required in pytests From a9ff3a336c0e9c9ee153b22a0de3bf5d07f5e462 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 26 May 2019 11:49:13 -0400 Subject: [PATCH 005/417] Enable TravisCI caching for pip and pre-commit plugins This should speed up TravisCI builds. --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index a4493e4..2c70e21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,15 @@ dist: xenial language: python python: 3.7 +# pre-commit hooks can use Docker, so we should go ahead and enable it services: docker +# Cache pip packages and pre-commit plugins to speed up builds +cache: + pip: true + directories: + - $HOME/.cache/pre-commit + install: - pip install --upgrade -r requirements-test.txt script: From 4e9280eb2baf8a72fa7abdf03898efb8ca82e2a1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 26 May 2019 11:57:27 -0400 Subject: [PATCH 006/417] Update pre-commit plugins to latest versions Via pre-commit autoupdate. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bac8c8..b6a0939 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.1 + rev: v2.2.3 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.15.0 + rev: v0.16.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -35,7 +35,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.4 + rev: 1.0.5 hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 @@ -45,11 +45,11 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.16.3 + rev: v1.17.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 2a1dbab + rev: 1.6.0 hooks: - id: bandit args: @@ -74,6 +74,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.17.0 + rev: 1.17.1 hooks: - id: prettier From 1a6b6b4af4c468668e2e2b1ce7eb5aa0df38560b Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 28 May 2019 14:39:17 -0400 Subject: [PATCH 007/417] update pre-commit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6a0939..69ebc03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.11.0 + rev: v1.12.0 hooks: - id: terraform_fmt - id: terraform_validate_no_variables From 82236155ac2cc6d7235d8d0a40b04c2f34996e32 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 28 May 2019 18:04:22 -0400 Subject: [PATCH 008/417] Add initial postfix templates and configurations --- .pre-commit-config.yaml | 1 + Dockerfile | 26 ++++++++++++++++++++ docker-compose.yml | 35 +++++++++++++++++++++++++++ secrets/fullchain.pem | 31 ++++++++++++++++++++++++ secrets/privkey.pem | 52 ++++++++++++++++++++++++++++++++++++++++ src/docker-entrypoint.sh | 16 +++++++++++++ templates/main.cf | 32 +++++++++++++++++++++++++ templates/master.cf | 9 +++++++ 8 files changed, 202 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 secrets/fullchain.pem create mode 100644 secrets/privkey.pem create mode 100755 src/docker-entrypoint.sh create mode 100644 templates/main.cf create mode 100644 templates/master.cf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69ebc03..2ac7afe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,7 @@ repos: args: - --allow-missing-credentials - id: detect-private-key + exclude: secrets/privkey.pem - id: end-of-file-fixer exclude: files/(issue|motd) - id: mixed-line-ending diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7064af7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM debian:buster-slim +MAINTAINER Mark Feldhousen + +RUN apt-get update && \ +apt-get install --no-install-recommends -y \ +ca-certificates \ +gettext-base \ +opendkim \ +opendkim-tools \ +postfix \ +sasl2-bin \ +&& apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +USER root +WORKDIR /root + +RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig + +COPY ./templates ./templates/ +COPY ./src/docker-entrypoint.sh . + +VOLUME ["/var/log", "/var/spool/postfix"] +EXPOSE 25/TCP 587/TCP + +ENTRYPOINT ["./docker-entrypoint.sh"] +CMD ["postfix", "-v", "start-fg"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..525e957 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +--- +version: "3.7" + +secrets: + fullchain_pem: + file: ./secrets/fullchain.pem + privkey_pem: + file: ./secrets/privkey.pem + + +services: + postfix: + build: + context: . + dockerfile: Dockerfile + image: postfix + init: true + restart: always + environment: + - PRIMARY_DOMAIN=example.com + - RELAY_IP= + ports: + - target: "25" + published: "1025" + protocol: tcp + mode: host + - target: "587" + published: "1587" + protocol: tcp + mode: host + secrets: + - source: fullchain_pem + target: fullchain.pem + - source: privkey_pem + target: privkey.pem diff --git a/secrets/fullchain.pem b/secrets/fullchain.pem new file mode 100644 index 0000000..ddc24ba --- /dev/null +++ b/secrets/fullchain.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFXTCCA0WgAwIBAgIJAPWv/2ssPwHVMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTkwNTI4MjAxNDM0WhcNMjAwNTI3MjAxNDM0WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAvclFvQ6WAkQXpwNksjjojlvAKkqTnHJ8vHaM0C3yrSm+aMPH0/lzLTuT +pv5eaSBUUzi5f/VjBFslH7kAGct4m1MJUfxRYdP4uZXqnfkiMyT8x7z+k6SbD22U +6Xxa7yV+hwkbhNDFOmcCWawgrERvfkSdyp/l94u+TWg5v/LvmkmsFRixT+U5dl/g +vSdXbAvjdrn+x/IRVMFrEDTm5QNCHrx1lTQf4giFl7VU820HQiNT3Y3JD7v+JHLO +DUyYZAA6bpg5vYYMbhxfegZu/C7DkT3/ZtpRXEPv1mR+koS7nPAtZmJ8t76GR/eU +A5rYLv1P6a4KvUf0/uRPGHQZsj57lVRqyR3TDi37aDywvcdTBQZHe9fNyYYe7g6s +ToKzY/Z+KwVuI+KcamVH6QLLhHOZ7IFIdB5PrSCMLHlDRP22GWV5lbFjxP+9H/2P +B5QkoBuPS4vV/GNxMlGlnbCoVDdUluypnsl0pdpae36PiFPlkA3dzJ1OcOt803jR +E3HHBg4Mq9rO51NQfxx3LsnKPVvajVChcEie1UEH+DQLEVciRBIp9Jho4SMh9xS9 +VXFRXXrnEKkfLOXwkd3Vy4upvyQrxpEMmj1SHSsFqJ4xz1uo5akpgwzB5I14oe+k +OK/sLP+B4GryWp78cASQQ/0ldbvOWxBxm7OVSPQB6F6vHVKNExsCAwEAAaNQME4w +HQYDVR0OBBYEFOAh81Tup7BSdYBjAPGyd1GW5GJhMB8GA1UdIwQYMBaAFOAh81Tu +p7BSdYBjAPGyd1GW5GJhMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +ALBpfqJWpTf58mv1n6HuSLI7n/FGA0wxoVPmvG9qBV0uBx0kbsuJVBUXfO+FE/3x +aiTiCwKweqdRhA2H7TnGs12D1Dweh3jd1gs+inClN7w4Ge9hIMiDcc9K45wgP/Sk +IIJYgLfctGfzc5EetqWQFq1GEPpNPNpfBMC7Z/KcgiZ8+RmoIACDlJ+EwSkDuXgf +045n45cF2xqfpeq/qKZDfWwg5+js/LN7abDRxX+JjH1i678Wx+SOlxsRK2plmmIL +c46vSErWoRddwxyZyP4TqBCTgV6ZcoZSWKFvHrn4c2YZHMuagsBaDIBg05jfxv72 +ewuKeWIC0/2PkbJ+W15X+/Ltgru7gIcidt8Xm+JnBjvz0bCiS2qVGxu0DxBmbmjv +CW3pvaNhc7NWxnTbJdpC/G2wH1RV06CS2WIWuBXu/AlkVlI/HHqYD4fWdETx9VjM +821dJY0oGBVqK+2/2d9Q1J1bfzs/J2kLaNocERog2RMapygHoNI0qtfsQIyEecGw +LoDcYltzKM+tPbWsYc1lI1rNo0v0/Y2TjQ6Jq+P4eZQ9gf6XgmdLg5nIDPkO4til +epYT2sgG6TAMyzf475BX92ect9KWD1efAavL+aSxwpngWBc7uqYvvbX8w1EePX7G +EoxN6uctuyBtDpzhdKxEEVBZ7NfU6X/91ZgVdlOR/rN4 +-----END CERTIFICATE----- diff --git a/secrets/privkey.pem b/secrets/privkey.pem new file mode 100644 index 0000000..52e5024 --- /dev/null +++ b/secrets/privkey.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC9yUW9DpYCRBen +A2SyOOiOW8AqSpOccny8dozQLfKtKb5ow8fT+XMtO5Om/l5pIFRTOLl/9WMEWyUf +uQAZy3ibUwlR/FFh0/i5leqd+SIzJPzHvP6TpJsPbZTpfFrvJX6HCRuE0MU6ZwJZ +rCCsRG9+RJ3Kn+X3i75NaDm/8u+aSawVGLFP5Tl2X+C9J1dsC+N2uf7H8hFUwWsQ +NOblA0IevHWVNB/iCIWXtVTzbQdCI1PdjckPu/4kcs4NTJhkADpumDm9hgxuHF96 +Bm78LsORPf9m2lFcQ+/WZH6ShLuc8C1mYny3voZH95QDmtgu/U/prgq9R/T+5E8Y +dBmyPnuVVGrJHdMOLftoPLC9x1MFBkd7183Jhh7uDqxOgrNj9n4rBW4j4pxqZUfp +AsuEc5nsgUh0Hk+tIIwseUNE/bYZZXmVsWPE/70f/Y8HlCSgG49Li9X8Y3EyUaWd +sKhUN1SW7KmeyXSl2lp7fo+IU+WQDd3MnU5w63zTeNETcccGDgyr2s7nU1B/HHcu +yco9W9qNUKFwSJ7VQQf4NAsRVyJEEin0mGjhIyH3FL1VcVFdeucQqR8s5fCR3dXL +i6m/JCvGkQyaPVIdKwWonjHPW6jlqSmDDMHkjXih76Q4r+ws/4HgavJanvxwBJBD +/SV1u85bEHGbs5VI9AHoXq8dUo0TGwIDAQABAoICAQCRaDhKVXaRXeJRT8RC2F81 +Uw60WFcoMn9nVd0lU07vZWBBnF7qBeE88rx54cIsAV0aNgfKBhRLLhoPaAqvuLk7 +KC+n5Q3lSiby6e3MAyk0zk3uKttR+3fiJi9FhMWXHL8Ibu3qoJm72Vhvo/WUhwp1 +T9UlfcUQGL1BSW2Vp2f0aiWyNC0F7bZM/8CMrCvK2ID6Yh7WypyEt3xz+lQ9enWa +XwInwrv6zlSsm33u08YP4klLImq952ccPempPtozJAmg2njCwIWdh5ePQoaeKKYm +Db4062gSrOqA9JYVZCTqZQoju6majhsL4KBC8sxXlDU58OLBivQmpn4DWlClxEGi +IbY/FIE6WEhOrdoGPzIjAcC3OYYTasIMBDLdA0tODmtv9Nvst30IGZc4Pm/QIJOk +EGJo4hqWbxiy4gisWxHwYeQ9/EEwrrc3FP94VscVkT8x0i22w5WMLtcrnCGpwzMg +E10+9v4ZUZ7cu9V+IeWQUkeuP3xhumI7RIDVRHpGC6TfEk/Q2gNdsPL2E8ng2Ytx +KMI3Pj5FuYi7enIR9AWdBVmVc2u7nzJMF/ODAwY6GmqHxni7PD97cnYwCy7Gxp/S +DZqiiD32RHwUwBm0AgdLhftkgqyTN/qo/Bhmj9ieO2CkuAvTYoXG0VMzxCb9wBG/ +7BJSGcbwtTJOJGK7LvrDAQKCAQEA6Q45teOKcmOSw5ne2cXzXuaXZ0OOCkjJ2ens +M89YmKXDVEZRbGoHVtftInUpr0H2UJ/N268Ogfzw62enZ40WIGwNALvp9PkLvdT0 +6LD/4MhcgZGQ5WDwqfqwkOanHdw9HJb752yEJ+3OG+fojmKkOs6OoQk1Ypxv5+5K +OuG/qtiKKpSLbG/nKAbPsPObArBxyfH9pV5F2E6vy38lYoDTURlA2BXHPoXu9M4c +/K2BMmO5zvGu5VOpAtnag5CWUwVvnX9DKDYs+k+exErluEj+U8GbKNQUTE+1p6fT +j4KKNVZBgnavOST3Xm/i4qVbccF/CwUc387HPdK5FU6kn3evewKCAQEA0HiEAytq +jzlBBHm892tojRzvpQa65fT7khsxETLhABvqeWZ2h9lE8TJTLC46N4cG1MC/hnWB +Q7XzKd7jAeht41Lp0mlDWv6eqKN4VyXSpAYzATcEO739eja7WNTgkYB91eDSyT+K +DVaElaXMjw/uX9tBnqaVyEe8JDqHw9E3Gl0MLWi89ztYptaWvKjt0+QqENBc6o+G +K/qzO+B4o9AyjyYkUYVA87tRrDk746LA5DbkpLQKPmQ3lb1hvVysJOnEdRabu5ly +mC0HR9n2UwcU98Op/EX3D4MuCUoFB/HQNMXq7oRMg+AcfsG0/ENcbiY6o0yRhxHu +ACgcjTi/QKAI4QKCAQBbgzB6EZ0diafpkpQFI0uLKjStYcN2mlpYbRhIx9RcLErk +3q++SGwVV7hP3X2+ycH0qqtk5fpmZHIdnZgIe0gC9yqr7R3TCa/onKSGcmonU8Wv +Qv+IcmZN+Jg4bbmVahO9FDRaDSxfmWtjXc7dijI+vTkYVstVq2PtyI3xTQ+8AEdQ +rP+KVu6HsxT+wMlPZwVnbNRSiRAX/d3dpFGDul4/7BCgSPzxuhm4mu6a8W5X4Pzn +G9O3TQCClBTPsIi2lN3dFEnEknFa4MTRAy/tCwyCyvUoNQ67YFlOOgJCydmHVBVp +Kz1mzPMta/XFVXTw2DAQnbNW1pU523K9wSG3VIHdAoIBACJTZbE76dzRWZJKFUJM +DjgGBrOOiyGoF/Azx/2D+iZRcmcw5t1xefeZCLbimbVg51AKuL6EBJfIktRXHdvH +kKh4k4WQzYVjHW65E+yNjsRxPN67V1ga7Wy9LFXxH1T16kJYNXzrmGif0U7usOLx +hZeE+6YK2ejTXvg8JvSoM0GFBqdHcq3muK8n8EP6MMbN79s648G/hiEhs3dte4/F +jT2i0yIVJd+7/TO1bNYLi2VIYJd6CaHCUKC4QSqz4qhlUXLSGSxnlMXXzDYZfoSn +St2M+yVNw+Nq/x6KcI+hUl4OJKPHZu3j7e01Kf7LfKGqa8dNqTyrSBwAfssGB/+1 +GiECggEAJD0KWTfJrSbgCkMfp1fNkwNExW2+neB+MI1eIR1sWsu8rz1a5d/NIdQq +pkoJp4FQUgRFEK+CzPWbKBDOxDVwpZ5o84JzxAEc78tL8/QIYwbtw5ZOiHNZ+wS6 +OYk6weY7rro7PwzqsTXcGdg/yxtphwguveSQM8y6McqBNZKqlN2fvXY8a4KZtt8O +RXBwpsqYulHpMGPh2MsMJBGEEII7Y2WKZG41oU1SGb5J2tBdGixW0buQnr6qwBgL +Ie8VV5kgbei97WK1lwvosn3HetBYSEE0GWMvjx93yoeozV8L/IF1rf7xss2BSqzF +UjgsHxWMDJWcER8NHXkE5DQORLtKCA== +-----END PRIVATE KEY----- diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh new file mode 100755 index 0000000..6436d6f --- /dev/null +++ b/src/docker-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# shellcheck disable=SC2016 + +set -e + +if [ "$1" = 'postfix' ]; then + + # generate confgurations using environment variables + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf + cp /etc/postfix/master.cf.orig /etc/postfix/master.cf + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf + + exec "$@" +fi + +exec "$@" diff --git a/templates/main.cf b/templates/main.cf new file mode 100644 index 0000000..d69a9f1 --- /dev/null +++ b/templates/main.cf @@ -0,0 +1,32 @@ +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no +append_dot_mydomain = no +readme_directory = no +smtpd_tls_cert_file=/run/secrets/fullchain.pem +smtpd_tls_key_file=/run/secrets/privkey.pem +smtpd_tls_security_level = may +smtp_tls_security_level = may +smtpd_tls_session_cache_database = btree:$data_directory/smtpd_scache +smtp_tls_session_cache_database = btree:$data_directory/smtp_scache +smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination +myhostname = ${PRIMARY_DOMAIN} +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = ${PRIMARY_DOMAIN} +mydestination = ${PRIMARY_DOMAIN}, localhost.com, , localhost +relayhost = +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 ${RELAY_IP} +mailbox_command = procmail -a "\$EXTENSION" +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all +inet_protocols = ipv4 +milter_default_action = accept +milter_protocol = 6 +smtpd_milters = inet:12301,inet:localhost:54321 +non_smtpd_milters = inet:12301,inet:localhost:54321 +disable_vrfy_command = yes +smtp_tls_note_starttls_offer = yes +always_bcc = mailarchive@${PRIMARY_DOMAIN} +maillog_file = /dev/stdout +compatibility_level = 2 diff --git a/templates/master.cf b/templates/master.cf new file mode 100644 index 0000000..92ed8a7 --- /dev/null +++ b/templates/master.cf @@ -0,0 +1,9 @@ +submission inet n - - - - smtpd + -o syslog_name=postfix/submission + -o smtpd_tls_wrappermode=no + -o smtpd_tls_security_level=may + -o smtpd_sasl_auth_enable=yes + -o smtpd_recipient_restrictions=permit_mynetworks,permit_sasl_authenticated,reject + -o milter_macro_daemon_name=ORIGINATING + -o smtpd_sasl_type=dovecot + -o smtpd_sasl_path=private/auth From 443de5b3e60df523d9c25dc011316e68f732e14e Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 17:35:41 -0400 Subject: [PATCH 009/417] Add password secret for mailarchive user --- secrets/mailarchive_passwd.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 secrets/mailarchive_passwd.txt diff --git a/secrets/mailarchive_passwd.txt b/secrets/mailarchive_passwd.txt new file mode 100644 index 0000000..323fae0 --- /dev/null +++ b/secrets/mailarchive_passwd.txt @@ -0,0 +1 @@ +foobar From 98498d729140e93c738495731c34585a201fa694 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 17:36:16 -0400 Subject: [PATCH 010/417] Sort postfix config and add proper environment variables --- templates/main.cf | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/templates/main.cf b/templates/main.cf index d69a9f1..09d9cc4 100644 --- a/templates/main.cf +++ b/templates/main.cf @@ -1,32 +1,32 @@ -smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) -biff = no -append_dot_mydomain = no -readme_directory = no -smtpd_tls_cert_file=/run/secrets/fullchain.pem -smtpd_tls_key_file=/run/secrets/privkey.pem -smtpd_tls_security_level = may -smtp_tls_security_level = may -smtpd_tls_session_cache_database = btree:$data_directory/smtpd_scache -smtp_tls_session_cache_database = btree:$data_directory/smtp_scache -smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -myhostname = ${PRIMARY_DOMAIN} -alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases -myorigin = ${PRIMARY_DOMAIN} -mydestination = ${PRIMARY_DOMAIN}, localhost.com, , localhost -relayhost = -mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 ${RELAY_IP} -mailbox_command = procmail -a "\$EXTENSION" -mailbox_size_limit = 0 -recipient_delimiter = + +alias_maps = hash:/etc/aliases +always_bcc = mailarchive@${PRIMARY_DOMAIN} +append_dot_mydomain = no +biff = no +compatibility_level = 2 +disable_vrfy_command = yes inet_interfaces = all inet_protocols = ipv4 +mailbox_command = procmail -a "\$EXTENSION" +mailbox_size_limit = 0 +maillog_file = /dev/stdout milter_default_action = accept milter_protocol = 6 -smtpd_milters = inet:12301,inet:localhost:54321 +mydestination = ${PRIMARY_DOMAIN}, localhost.com, , localhost +myhostname = ${PRIMARY_DOMAIN} +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 ${RELAY_IP} +myorigin = ${PRIMARY_DOMAIN} non_smtpd_milters = inet:12301,inet:localhost:54321 -disable_vrfy_command = yes +readme_directory = no +recipient_delimiter = + +relayhost = smtp_tls_note_starttls_offer = yes -always_bcc = mailarchive@${PRIMARY_DOMAIN} -maillog_file = /dev/stdout -compatibility_level = 2 +smtp_tls_security_level = may +smtp_tls_session_cache_database = btree:$data_directory/smtp_scache +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +smtpd_milters = inet:12301,inet:localhost:54321 +smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination +smtpd_tls_cert_file=/run/secrets/fullchain.pem +smtpd_tls_key_file=/run/secrets/privkey.pem +smtpd_tls_security_level = may +smtpd_tls_session_cache_database = btree:$data_directory/smtpd_scache From baf31426f3a57ac266169ff3c0b28af3c5257fe0 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 17:36:38 -0400 Subject: [PATCH 011/417] Add configurations for supporting services --- templates/TrustedHosts | 4 ++++ templates/dovecot.conf | 38 ++++++++++++++++++++++++++++++++++++++ templates/opendkim.conf | 17 +++++++++++++++++ templates/opendmarc.conf | 10 ++++++++++ 4 files changed, 69 insertions(+) create mode 100644 templates/TrustedHosts create mode 100644 templates/dovecot.conf create mode 100644 templates/opendkim.conf create mode 100644 templates/opendmarc.conf diff --git a/templates/TrustedHosts b/templates/TrustedHosts new file mode 100644 index 0000000..53aaa67 --- /dev/null +++ b/templates/TrustedHosts @@ -0,0 +1,4 @@ +127.0.0.1 +localhost +${PRIMARY_DOMAIN} +${RELAY_IP} diff --git a/templates/dovecot.conf b/templates/dovecot.conf new file mode 100644 index 0000000..167b8a6 --- /dev/null +++ b/templates/dovecot.conf @@ -0,0 +1,38 @@ +disable_plaintext_auth = no +mail_privileged_group = mail +mail_location = mbox:~/mail:INBOX=/var/mail/%u +userdb { + driver = passwd +} +passdb { + args = %s + driver = pam +} +protocols = " imap" +protocol imap { + mail_plugins = " autocreate" +} +plugin { + autocreate = Trash + autocreate2 = Sent + autosubscribe = Trash + autosubscribe2 = Sent +} +service imap-login { + inet_listener imap { + port = 0 + } + inet_listener imaps { + port = 993 + } +} +service auth { + unix_listener /var/spool/postfix/private/auth { + group = postfix + mode = 0660 + user = postfix + } +} +ssl=required +ssl_cert = Date: Thu, 30 May 2019 18:00:39 -0400 Subject: [PATCH 012/417] Update docker components --- Dockerfile | 15 +++++++++++++-- docker-compose.yml | 21 ++++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7064af7..6dd598e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,25 +2,36 @@ FROM debian:buster-slim MAINTAINER Mark Feldhousen RUN apt-get update && \ -apt-get install --no-install-recommends -y \ +DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ +dovecot-imapd \ +dovecot-lmtpd \ gettext-base \ +mailutils \ opendkim \ opendkim-tools \ +opendmarc \ postfix \ +procmail \ sasl2-bin \ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN adduser mailarchive --quiet --disabled-password \ +--shell /usr/sbin/nologin --gecos "Mail Archive" + USER root WORKDIR /root +# make backups of configurations. These are modified at startup. RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig +RUN mv /etc/default/opendkim /etc/default/opendkim.orig +RUN mv /etc/default/opendmarc /etc/default/opendmarc.orig COPY ./templates ./templates/ COPY ./src/docker-entrypoint.sh . VOLUME ["/var/log", "/var/spool/postfix"] -EXPOSE 25/TCP 587/TCP +EXPOSE 25/TCP 587/TCP 993/TCP ENTRYPOINT ["./docker-entrypoint.sh"] CMD ["postfix", "-v", "start-fg"] diff --git a/docker-compose.yml b/docker-compose.yml index 525e957..1a50d6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ secrets: file: ./secrets/fullchain.pem privkey_pem: file: ./secrets/privkey.pem + mailarchive_passwd_txt: + file: ./secrets/mailarchive_passwd.txt services: @@ -18,7 +20,10 @@ services: restart: always environment: - PRIMARY_DOMAIN=example.com - - RELAY_IP= + - RELAY_IP=172.16.202.1/32 + networks: + front: + ipv4_address: 172.16.202.2 ports: - target: "25" published: "1025" @@ -28,8 +33,22 @@ services: published: "1587" protocol: tcp mode: host + - target: "993" + published: "1993" + protocol: tcp + mode: host secrets: - source: fullchain_pem target: fullchain.pem - source: privkey_pem target: privkey.pem + - source: mailarchive_passwd_txt + target: mailarchive_passwd.txt + +networks: + front: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.16.202.0/24 From a031cf06b5a65ecf61bf16373d92b13c1109790b Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 18:01:55 -0400 Subject: [PATCH 013/417] Add custom docker entrypoint --- src/docker-entrypoint.sh | 64 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh index 6436d6f..68bd74a 100755 --- a/src/docker-entrypoint.sh +++ b/src/docker-entrypoint.sh @@ -2,15 +2,69 @@ # shellcheck disable=SC2016 set -e +#set -x + +function generate_configs() { + # configure postfix + echo "Generating postfix configurations for ${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf + cp /etc/postfix/master.cf.orig /etc/postfix/master.cf + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendkim.conf > /etc/opendkim.conf + + # configure opendkim + echo "Generating opendkim configurations for ${PRIMARY_DOMAIN}" + mkdir -p "/etc/opendkim/keys/${PRIMARY_DOMAIN}" + opendkim-genkey --verbose --bits=1024 --selector=mail --directory="/etc/opendkim/keys/${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/TrustedHosts > /etc/opendkim/TrustedHosts + cp /etc/default/opendkim.orig /etc/default/opendkim + echo 'SOCKET="inet:12301"' >> /etc/default/opendkim + chown -R opendkim:opendkim /etc/opendkim + + # configure opendmarc + echo "Generating opendmarc configurations for ${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendmarc.conf > /etc/opendmarc.conf + mkdir "/etc/opendmarc/" + echo "localhost" > /etc/opendmarc/ignore.hosts + chown -R opendmarc:opendmarc /etc/opendmarc + cp /etc/default/opendmarc.orig /etc/default/opendmarc + echo 'SOCKET="inet:54321"' >> /etc/default/opendmarc + + # configure dovecot + echo "Generating dovecot configurations for ${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/dovecot.conf > /etc/dovecot/dovecot.conf + + # create a file marking the configuration as completed for this domain + echo "All configurations generated for ${PRIMARY_DOMAIN}" +} if [ "$1" = 'postfix' ]; then + echo "Starting mail server with:" + echo " PRIMARY_DOMAIN=${PRIMARY_DOMAIN}" + echo " RELAY_IP=${RELAY_IP}" + + # check to see if the configuration was completed for this domain + if [[ ! -f conf_gen_done.txt ]] || [[ $(< conf_gen_done.txt) != "${PRIMARY_DOMAIN}" ]]; then + generate_configs + echo "${PRIMARY_DOMAIN}" > conf_gen_done.txt + else + echo "Configurations already generated for ${PRIMARY_DOMAIN}, preserving." + fi + + # set password of mailarchive user to secret + echo "mailarchive:$(< /run/secrets/mailarchive_passwd.txt)" | chpasswd + + # postfix needs fresh copies of files in its chroot jail + cp /etc/{hosts,localtime,nsswitch.conf,resolv.conf,services} /var/spool/postfix/etc/ - # generate confgurations using environment variables - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf - cp /etc/postfix/master.cf.orig /etc/postfix/master.cf - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf + echo "--------------------------------------------" + cat "/etc/opendkim/keys/${PRIMARY_DOMAIN}/mail.txt" + echo "--------------------------------------------" - exec "$@" + opendmarc + opendkim + dovecot + exec "$@" fi exec "$@" From 35f407018ede943e97e38cd07116d849d2e03aed Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 18:52:56 -0400 Subject: [PATCH 014/417] Move templates and secrets into src --- .pre-commit-config.yaml | 2 +- Dockerfile | 2 +- docker-compose.yml | 6 +++--- {secrets => src/secrets}/fullchain.pem | 0 {secrets => src/secrets}/mailarchive_passwd.txt | 0 {secrets => src/secrets}/privkey.pem | 0 {templates => src/templates}/TrustedHosts | 0 {templates => src/templates}/dovecot.conf | 0 {templates => src/templates}/main.cf | 0 {templates => src/templates}/master.cf | 0 {templates => src/templates}/opendkim.conf | 0 {templates => src/templates}/opendmarc.conf | 0 12 files changed, 5 insertions(+), 5 deletions(-) rename {secrets => src/secrets}/fullchain.pem (100%) rename {secrets => src/secrets}/mailarchive_passwd.txt (100%) rename {secrets => src/secrets}/privkey.pem (100%) rename {templates => src/templates}/TrustedHosts (100%) rename {templates => src/templates}/dovecot.conf (100%) rename {templates => src/templates}/main.cf (100%) rename {templates => src/templates}/master.cf (100%) rename {templates => src/templates}/opendkim.conf (100%) rename {templates => src/templates}/opendmarc.conf (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ac7afe..189b5dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: args: - --allow-missing-credentials - id: detect-private-key - exclude: secrets/privkey.pem + exclude: src/secrets/privkey.pem - id: end-of-file-fixer exclude: files/(issue|motd) - id: mixed-line-ending diff --git a/Dockerfile b/Dockerfile index 6dd598e..a7982d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig RUN mv /etc/default/opendkim /etc/default/opendkim.orig RUN mv /etc/default/opendmarc /etc/default/opendmarc.orig -COPY ./templates ./templates/ +COPY ./src/templates ./templates/ COPY ./src/docker-entrypoint.sh . VOLUME ["/var/log", "/var/spool/postfix"] diff --git a/docker-compose.yml b/docker-compose.yml index 1a50d6f..a2abb7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,11 @@ version: "3.7" secrets: fullchain_pem: - file: ./secrets/fullchain.pem + file: ./src/secrets/fullchain.pem privkey_pem: - file: ./secrets/privkey.pem + file: ./src/secrets/privkey.pem mailarchive_passwd_txt: - file: ./secrets/mailarchive_passwd.txt + file: ./src/secrets/mailarchive_passwd.txt services: diff --git a/secrets/fullchain.pem b/src/secrets/fullchain.pem similarity index 100% rename from secrets/fullchain.pem rename to src/secrets/fullchain.pem diff --git a/secrets/mailarchive_passwd.txt b/src/secrets/mailarchive_passwd.txt similarity index 100% rename from secrets/mailarchive_passwd.txt rename to src/secrets/mailarchive_passwd.txt diff --git a/secrets/privkey.pem b/src/secrets/privkey.pem similarity index 100% rename from secrets/privkey.pem rename to src/secrets/privkey.pem diff --git a/templates/TrustedHosts b/src/templates/TrustedHosts similarity index 100% rename from templates/TrustedHosts rename to src/templates/TrustedHosts diff --git a/templates/dovecot.conf b/src/templates/dovecot.conf similarity index 100% rename from templates/dovecot.conf rename to src/templates/dovecot.conf diff --git a/templates/main.cf b/src/templates/main.cf similarity index 100% rename from templates/main.cf rename to src/templates/main.cf diff --git a/templates/master.cf b/src/templates/master.cf similarity index 100% rename from templates/master.cf rename to src/templates/master.cf diff --git a/templates/opendkim.conf b/src/templates/opendkim.conf similarity index 100% rename from templates/opendkim.conf rename to src/templates/opendkim.conf diff --git a/templates/opendmarc.conf b/src/templates/opendmarc.conf similarity index 100% rename from templates/opendmarc.conf rename to src/templates/opendmarc.conf From bf661acd28fd400614dd895fd0291025f7c03b84 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 18:53:20 -0400 Subject: [PATCH 015/417] Update URLs --- CONTRIBUTING.md | 10 +++++----- README.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ede2f47..59a01fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Before contributing, we encourage you to read our CONTRIBUTING policy If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/skeleton-generic/issues) in this +issue](https://github.com/cisagov/docker-postfix/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/skeleton-generic/pulls), you will +request](https://github.com/cisagov/docker-postfix/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -78,9 +78,9 @@ can create and configure the Python virtual environment with these commands: ```bash -cd skeleton-generic -pyenv virtualenv skeleton-generic -pyenv local skeleton-generic +cd docker-postfix +pyenv virtualenv docker-postfix +pyenv local docker-postfix pip install -r requirements-dev.txt ``` diff --git a/README.md b/README.md index 2337970..7239e32 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# skeleton-generic # +# docker-postfix # -[![Build Status](https://travis-ci.com/cisagov/skeleton-generic.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-generic) +[![Build Status](https://travis-ci.com/cisagov/docker-postfix.svg?branch=develop)](https://travis-ci.com/cisagov/docker-postfix) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. From b84b6b1b6d69004108b9cf04f5704dfc6e30346a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 20:36:52 -0400 Subject: [PATCH 016/417] Get ready for docker hub integration --- .travis.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2c70e21..1bbdf65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,11 @@ python: 3.7 # pre-commit hooks can use Docker, so we should go ahead and enable it services: docker +env: + global: + - IMAGE_NAME=dhsncats/postfix + - DOCKER_USER=felddy + # Cache pip packages and pre-commit plugins to speed up builds cache: pip: true @@ -13,5 +18,17 @@ cache: install: - pip install --upgrade -r requirements-test.txt + script: - pre-commit run --all-files + #- travis_scripts/build_docker_image.sh + +after_success: + - coveralls + +# deploy: +# - provider: script +# script: bash travis_scripts/deploy_to_docker_hub.sh +# on: +# tags: true +# python: '3.7' From f7ba54655719794e0169f9547e41581e6029105f Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 20:37:17 -0400 Subject: [PATCH 017/417] Add scripts to support travis --- bump_version.sh | 47 ++++++++++++++++++++++++++ travis_scripts/build_docker_image.sh | 8 +++++ travis_scripts/deploy_to_docker_hub.sh | 9 +++++ 3 files changed, 64 insertions(+) create mode 100755 bump_version.sh create mode 100755 travis_scripts/build_docker_image.sh create mode 100755 travis_scripts/deploy_to_docker_hub.sh diff --git a/bump_version.sh b/bump_version.sh new file mode 100755 index 0000000..0de513d --- /dev/null +++ b/bump_version.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# bump_version.sh (show|major|minor|patch|prerelease|build) + +set -o nounset +set -o errexit +set -o pipefail + +VERSION_FILE=cyhy/mailer/__init__.py + +HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" + +old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) + +if [ $# -ne 1 ] +then + echo "$HELP_INFORMATION" +else + case $1 in + major|minor|patch|prerelease|build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + show) + echo "$old_version" + ;; + *) + echo "$HELP_INFORMATION" + ;; + esac +fi diff --git a/travis_scripts/build_docker_image.sh b/travis_scripts/build_docker_image.sh new file mode 100755 index 0000000..862f220 --- /dev/null +++ b/travis_scripts/build_docker_image.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +version=$(./bump_version.sh show) +docker build -t "$IMAGE_NAME":"$version" . diff --git a/travis_scripts/deploy_to_docker_hub.sh b/travis_scripts/deploy_to_docker_hub.sh new file mode 100755 index 0000000..2be3e20 --- /dev/null +++ b/travis_scripts/deploy_to_docker_hub.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "$DOCKER_PW" | docker login -u "$DOCKER_USER" --password-stdin +version=$(./bump_version.sh show) +docker push "$IMAGE_NAME":"$version" From b1f527b56598b31cdf8909d313e2568c11da62cd Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 20:40:45 -0400 Subject: [PATCH 018/417] Add container info to README --- README.md | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7239e32..0ae8953 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,46 @@ -# docker-postfix # +# docker-postfix 🐳📮 # [![Build Status](https://travis-ci.com/cisagov/docker-postfix.svg?branch=develop)](https://travis-ci.com/cisagov/docker-postfix) -This is a generic skeleton project that can be used to quickly get a -new [cisagov](https://github.com/cisagov) GitHub project started. -This skeleton project contains [licensing information](LICENSE.md), as -well as [pre-commit hooks](https://pre-commit.com) and a [Travis -CI](https://travis-ci.com) configuration appropriate for the major -languages that we use. +Creates a Docker container with an installation of the +[postfix](http://postfix.org) MTA. Additionally it has an IMAP +server ([dovecot](https://dovecot.org)) for accessing the archvies +of sent email. All email is BCC's to the `mailarchive` account. -In many cases you will instead want to use one of the more specific -skeleton projects derived from this one. +## Usage ## + +A sample [docker composition](docker-compose.yml) is included in this repository. +To build and start the container use the command: `docker-compose up` + +### Ports ### + +By default this container will listen on the following ports: + +- 1025: `smtp` +- 1587: `submission` +- 1993: `imaps` + +### Environment Variables ### + +Two environment variables are used to generate the configurations at runtime: + +- `PRIMARY_DOMAIN`: the domain of the mail server +- `RELAY_IP`: (optional) an IP address that is allowed to relay mail without authentication + +### Secrets ### + +- `fullchain.pem`: public key +- `privkey.pem`: private key +- `mailarchive_password.txt`: password for the mailarchive user + +### Volumes ### + +Two optional volumes can be attached to this container to persist the +mail spool directory, as well as the logging directory. (Note that +the mail logs are available using the docker log command.) + +- `/var/spool/postfix`: mail queues +- `/var/log`: system logs ## Contributing ## From 61a2bafb7dd9465330a4c7e4bf34da3c7f1e6552 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:05:31 -0400 Subject: [PATCH 019/417] Remove uneeded --- requirements-dev.txt | 2 -- requirements-test.txt | 1 - 2 files changed, 3 deletions(-) delete mode 100644 requirements-dev.txt delete mode 100644 requirements-test.txt diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index f122cc5..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ --r requirements-test.txt -ipython diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 416634f..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pre-commit From 2f4926b72ffc966af8beb3c8454caa827718c3be Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:05:48 -0400 Subject: [PATCH 020/417] Add a version file to bump --- bump_version.sh | 2 +- src/version.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 src/version.txt diff --git a/bump_version.sh b/bump_version.sh index 0de513d..7c23343 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -6,7 +6,7 @@ set -o nounset set -o errexit set -o pipefail -VERSION_FILE=cyhy/mailer/__init__.py +VERSION_FILE=src/version.txt HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" diff --git a/src/version.txt b/src/version.txt new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/src/version.txt @@ -0,0 +1 @@ +__version__ = "0.0.1" From ce21639ff226d88478d4abe72e833ab8e185b0ea Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:06:39 -0400 Subject: [PATCH 021/417] Add magic keys for travis deploy --- .travis.yml | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bbdf65..82030d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,33 +2,38 @@ dist: xenial language: python python: 3.7 -# pre-commit hooks can use Docker, so we should go ahead and enable it services: docker - env: global: - IMAGE_NAME=dhsncats/postfix - DOCKER_USER=felddy - -# Cache pip packages and pre-commit plugins to speed up builds + - secure: >- + R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e + zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y + 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 + d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 + KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG + qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq + JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 + EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q + 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI + tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR + n2KsiL8piwAEc= cache: pip: true directories: - - $HOME/.cache/pre-commit - + - "$HOME/.cache/pre-commit" install: - pip install --upgrade -r requirements-test.txt - script: - pre-commit run --all-files - #- travis_scripts/build_docker_image.sh + - travis_scripts/build_docker_image.sh after_success: - coveralls - -# deploy: -# - provider: script -# script: bash travis_scripts/deploy_to_docker_hub.sh -# on: -# tags: true -# python: '3.7' +deploy: + - provider: script + script: bash travis_scripts/deploy_to_docker_hub.sh + on: + tags: true + python: '3.7' From 59c0c893aff55e08a010f7259cfde005b532683c Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:09:38 -0400 Subject: [PATCH 022/417] Remove install section --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82030d8..44e5f20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,6 @@ cache: pip: true directories: - "$HOME/.cache/pre-commit" -install: - - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - travis_scripts/build_docker_image.sh From 873d49c7146c7d54b78e84f7e7035cea7540e5a4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:13:02 -0400 Subject: [PATCH 023/417] Re-enable pre-commit tests --- .travis.yml | 2 ++ requirements-test.txt | 1 + 2 files changed, 3 insertions(+) create mode 100644 requirements-test.txt diff --git a/.travis.yml b/.travis.yml index 44e5f20..82030d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ cache: pip: true directories: - "$HOME/.cache/pre-commit" +install: + - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - travis_scripts/build_docker_image.sh diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..416634f --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +pre-commit From 752922cdb2ef2a001690f07cff73903652d6b95d Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 22:12:20 -0400 Subject: [PATCH 024/417] Update README --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0ae8953..312190a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Creates a Docker container with an installation of the [postfix](http://postfix.org) MTA. Additionally it has an IMAP server ([dovecot](https://dovecot.org)) for accessing the archvies -of sent email. All email is BCC's to the `mailarchive` account. +of sent email. All email is BCC'd to the `mailarchive` account. ## Usage ## @@ -14,11 +14,14 @@ To build and start the container use the command: `docker-compose up` ### Ports ### -By default this container will listen on the following ports: +This container exposes the following ports: -- 1025: `smtp` -- 1587: `submission` -- 1993: `imaps` +- 25: `smtp` +- 587: `submission` +- 993: `imaps` + +The sample [docker composition](docker-compose.yml) publishes the +exposed ports at 1025, 1587, and 1993. ### Environment Variables ### From ea6ea18679c93c6bfcaf9d87900e7cc9fbe9ea74 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 22:12:42 -0400 Subject: [PATCH 025/417] Add real image name to docker-compose build --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a2abb7d..c03837d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: build: context: . dockerfile: Dockerfile - image: postfix + image: dhsncats/postfix init: true restart: always environment: From 631f3a03a43752972e7361aafd8de21c8f4bea12 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 09:20:00 -0400 Subject: [PATCH 026/417] Move container build to install instead of script. This allows a hard fail of the container doesn't build. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82030d8..6ffc591 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,10 +25,9 @@ cache: - "$HOME/.cache/pre-commit" install: - pip install --upgrade -r requirements-test.txt + - travis_scripts/build_docker_image.sh script: - pre-commit run --all-files - - travis_scripts/build_docker_image.sh - after_success: - coveralls deploy: From f3b72cf67589c0470f428e0acce6daf55949e86c Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 10:41:19 -0400 Subject: [PATCH 027/417] Limit bandit in tests tree --- .bandit.yml | 2 +- .pre-commit-config.yaml | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 8ba42d1..5ae161d 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -10,4 +10,4 @@ tests: # - B102 skips: -# - B101 # skip "assert used" check since assertions are required in pytests + - B101 # skip "assert used" check since assertions are required in pytests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 189b5dd..3b357eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,12 +49,22 @@ repos: rev: v1.17.1 hooks: - id: pyupgrade + # Run bandit on "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit - rev: 1.6.0 + rev: 2a1dbab hooks: - id: bandit + name: bandit (tests tree) + files: tests args: - --config=.bandit.yml + # Run bandit everything but tests directory + - repo: https://github.com/PyCQA/bandit + rev: 2a1dbab + hooks: + - id: bandit + name: bandit (everything else) + exclude: tests - repo: https://github.com/python/black rev: 19.3b0 hooks: @@ -63,7 +73,6 @@ repos: rev: v4.1.0a0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From f5c2107331b93b74b9bf29470bf008727cbbf334 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 10:41:46 -0400 Subject: [PATCH 028/417] Add container pytests --- .travis.yml | 8 +++- pytest.ini | 2 + requirements-test.txt | 1 + tests/container_test.py | 103 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/container_test.py diff --git a/.travis.yml b/.travis.yml index 6ffc591..bf01d1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,14 @@ cache: install: - pip install --upgrade -r requirements-test.txt - travis_scripts/build_docker_image.sh +before_script: + - docker-compose up -d + - docker-compose ps | grep -q postfix_postfix_1 script: - pre-commit run --all-files -after_success: - - coveralls + - pytest -v +after_script: + - docker-compose down deploy: - provider: script script: bash travis_scripts/deploy_to_docker_hub.sh diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..aada9d7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = -v -ra diff --git a/requirements-test.txt b/requirements-test.txt index 416634f..51f1982 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ pre-commit +pytest diff --git a/tests/container_test.py b/tests/container_test.py new file mode 100644 index 0000000..59d1554 --- /dev/null +++ b/tests/container_test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env pytest -vs +"""Tests for postfix container.""" + +from email.message import EmailMessage +from imaplib import IMAP4_SSL +import smtplib + +import pytest + +MESSAGE = """ +This is a test message sent during the unit tests. +""" +DOMAIN = "example.com" +TEST_USER = "mailarchive" +TEST_PW = "foobar" +IMAP_PORT = 1993 + + +@pytest.mark.parametrize("port", [1025, 1587]) +def test_sending_mail(port): + """Send an email message to the server.""" + msg = EmailMessage() + msg.set_content(MESSAGE) + msg["Subject"] = f"Test Message on port {port}" + msg["From"] = f"test@{DOMAIN}" + msg["To"] = f"mailarchive@{DOMAIN}" + with smtplib.SMTP("localhost", port=port) as s: + s.send_message(msg) + + +def test_imap_login(): + """Test logging in to the IMAP server.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login("mailarchive", "foobar") + + +def test_imap_messages_exist(): + """Test test existence of our test messages.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + message_count = int(data[0]) + print(f"inbox message count: {message_count}") + assert message_count > 0, "Expected message in the inbox" + + +def test_imap_access(): + """Test receiving message from the IMAP server.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + message_count = int(data[0]) + print(f"inbox message count: {message_count}") + typ, data = m.search(None, "ALL") + assert typ == "OK", "Search did not return OK status" + message_numbers = data[0].split() + for num in message_numbers: + typ, data = m.fetch(num, "(RFC822)") + assert typ == "OK", f"Fetch of message {num} did not return OK status" + print("-" * 40) + print(f"Message: {num}") + print(data[0][1].decode("utf-8")) + # mark messag as deleted + typ, data = m.store(num, "+FLAGS", "\\Deleted") + assert ( + typ == "OK" + ), f"Storing '\\deleted' flag on message {num} did not return OK status" + # expunge all deleted messages + typ, data = m.expunge() + assert typ == "OK", "Expunge did not return OK status" + + +def test_imap_delete_all(): + """Test deleting messages from the IMAP server.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + typ, data = m.search(None, "ALL") + assert typ == "OK", "Search did not return OK status" + message_numbers = data[0].split() + for num in message_numbers: + # mark messag as deleted + typ, data = m.store(num, "+FLAGS", "\\Deleted") + assert ( + typ == "OK" + ), f"Storing '\\deleted' flag on message {num} did not return OK status" + # expunge all deleted messages + typ, data = m.expunge() + assert typ == "OK", "Expunge did not return OK status" + + +def test_imap_messages_cleared(): + """Test that all messages were expunged.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + message_count = int(data[0]) + print(f"inbox message count: {message_count}") + assert message_count == 0, "Expected the inbox to be empty" From de8997ea4e7aaf3ac6895eb095e55a559d15310c Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 10:43:04 -0400 Subject: [PATCH 029/417] add ignore for pytest caches --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 073a081..d8e09a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ .python-version +.pytest_cache From 4da142e28468090b461ec893b62771d06141108d Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 11:02:19 -0400 Subject: [PATCH 030/417] add latest tag to build for use by compose --- travis_scripts/build_docker_image.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/travis_scripts/build_docker_image.sh b/travis_scripts/build_docker_image.sh index 862f220..0a89fed 100755 --- a/travis_scripts/build_docker_image.sh +++ b/travis_scripts/build_docker_image.sh @@ -5,4 +5,5 @@ set -o errexit set -o pipefail version=$(./bump_version.sh show) -docker build -t "$IMAGE_NAME":"$version" . +# shellcheck disable=SC2140 +docker build -t "$IMAGE_NAME":"$version" -t "$IMAGE_NAME":"latest" . From 308fdb3d3a8af82e1af3cafa4d9387cffb5481b7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 11:03:33 -0400 Subject: [PATCH 031/417] Remove check that was failing due to race --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bf01d1b..c488104 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,6 @@ install: - travis_scripts/build_docker_image.sh before_script: - docker-compose up -d - - docker-compose ps | grep -q postfix_postfix_1 script: - pre-commit run --all-files - pytest -v From 72b108cc74902bf13af26f713517c88e9374c360 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:31:45 -0400 Subject: [PATCH 032/417] Add diceware for password generator --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a7982d3..6a82707 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ MAINTAINER Mark Feldhousen RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ +diceware \ dovecot-imapd \ dovecot-lmtpd \ gettext-base \ From 4d3f10ea31c64c507ed7803c274bcf9bae40183f Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:32:29 -0400 Subject: [PATCH 033/417] Change secrets to new user.txt file --- docker-compose.yml | 8 ++++---- src/secrets/mailarchive_passwd.txt | 1 - src/secrets/users.txt | 10 ++++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) delete mode 100644 src/secrets/mailarchive_passwd.txt create mode 100644 src/secrets/users.txt diff --git a/docker-compose.yml b/docker-compose.yml index c03837d..1f0d962 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,8 @@ secrets: file: ./src/secrets/fullchain.pem privkey_pem: file: ./src/secrets/privkey.pem - mailarchive_passwd_txt: - file: ./src/secrets/mailarchive_passwd.txt + users_txt: + file: ./src/secrets/users.txt services: @@ -42,8 +42,8 @@ services: target: fullchain.pem - source: privkey_pem target: privkey.pem - - source: mailarchive_passwd_txt - target: mailarchive_passwd.txt + - source: users_txt + target: users.txt networks: front: diff --git a/src/secrets/mailarchive_passwd.txt b/src/secrets/mailarchive_passwd.txt deleted file mode 100644 index 323fae0..0000000 --- a/src/secrets/mailarchive_passwd.txt +++ /dev/null @@ -1 +0,0 @@ -foobar diff --git a/src/secrets/users.txt b/src/secrets/users.txt new file mode 100644 index 0000000..86cd38b --- /dev/null +++ b/src/secrets/users.txt @@ -0,0 +1,10 @@ +# Define the users to be created at container startup. +# If is omitted for a user it will be generated and logged at startup +# username + +# The mailarchive user is mandatory since all mail is BCC'd to this user. +mailarchive foobar + +# define other users below as needed +testsender1 lemmy is god +testsender2 From 5234af1c945714950472ecbcad17efb41caf8321 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:35:23 -0400 Subject: [PATCH 034/417] Add dynamic user generator and tests. --- src/docker-entrypoint.sh | 24 +++++++++++++++-- tests/container_test.py | 58 ++++++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh index 68bd74a..ce91e89 100755 --- a/src/docker-entrypoint.sh +++ b/src/docker-entrypoint.sh @@ -4,6 +4,7 @@ set -e #set -x + function generate_configs() { # configure postfix echo "Generating postfix configurations for ${PRIMARY_DOMAIN}" @@ -38,6 +39,24 @@ function generate_configs() { echo "All configurations generated for ${PRIMARY_DOMAIN}" } + +function generate_users() { + echo "Generating users and passwords:" + echo "--------------------------------------------" + while IFS=" " read -r username password || [ -n "$username" ] + do + if [ -z "$password" ]; then password=$(diceware -d-); + echo -e "$username\t$password" + else + echo -e "$username\t" + fi + adduser "$username" --quiet --disabled-password --shell /usr/sbin/nologin --gecos "" &>/dev/null || true + echo "$username:$password" | chpasswd || true + done + echo "--------------------------------------------" +} + + if [ "$1" = 'postfix' ]; then echo "Starting mail server with:" echo " PRIMARY_DOMAIN=${PRIMARY_DOMAIN}" @@ -51,12 +70,13 @@ if [ "$1" = 'postfix' ]; then echo "Configurations already generated for ${PRIMARY_DOMAIN}, preserving." fi - # set password of mailarchive user to secret - echo "mailarchive:$(< /run/secrets/mailarchive_passwd.txt)" | chpasswd + # generate the users from the secrets + grep -v '^#\|^$' /run/secrets/users.txt | generate_users # postfix needs fresh copies of files in its chroot jail cp /etc/{hosts,localtime,nsswitch.conf,resolv.conf,services} /var/spool/postfix/etc/ + echo "DKIM DNS entry:" echo "--------------------------------------------" cat "/etc/opendkim/keys/${PRIMARY_DOMAIN}/mail.txt" echo "--------------------------------------------" diff --git a/tests/container_test.py b/tests/container_test.py index 59d1554..0ccd9e9 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -11,44 +11,62 @@ This is a test message sent during the unit tests. """ DOMAIN = "example.com" -TEST_USER = "mailarchive" -TEST_PW = "foobar" +ARCHIVE_USER = "mailarchive" +ARCHIVE_PW = "foobar" +TEST_SEND_USER = "testsender1" +TEST_SEND_PW = "lemmy is god" IMAP_PORT = 1993 @pytest.mark.parametrize("port", [1025, 1587]) -def test_sending_mail(port): +@pytest.mark.parametrize("to_user", [ARCHIVE_USER, TEST_SEND_USER]) +def test_sending_mail(port, to_user): """Send an email message to the server.""" msg = EmailMessage() msg.set_content(MESSAGE) msg["Subject"] = f"Test Message on port {port}" msg["From"] = f"test@{DOMAIN}" - msg["To"] = f"mailarchive@{DOMAIN}" + msg["To"] = f"{to_user}@{DOMAIN}" with smtplib.SMTP("localhost", port=port) as s: s.send_message(msg) -def test_imap_login(): +@pytest.mark.parametrize( + "username,password", + [ + (ARCHIVE_USER, ARCHIVE_PW), + (TEST_SEND_USER, TEST_SEND_PW), + pytest.param(ARCHIVE_USER, TEST_SEND_PW, marks=pytest.mark.xfail), + pytest.param("your_mom", "so_fat", marks=pytest.mark.xfail), + ], +) +def test_imap_login(username, password): """Test logging in to the IMAP server.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login("mailarchive", "foobar") + m.login(username, password) -def test_imap_messages_exist(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_messages_exist(username, password): """Test test existence of our test messages.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() - assert typ == "OK", "Select did not return OK status" + assert typ == "OK", f"Select did not return OK status for {username}" message_count = int(data[0]) - print(f"inbox message count: {message_count}") - assert message_count > 0, "Expected message in the inbox" + print(f"{username} inbox message count: {message_count}") + assert message_count > 0, f"Expected message in the {username} inbox" -def test_imap_access(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_reading(username, password): """Test receiving message from the IMAP server.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() assert typ == "OK", "Select did not return OK status" message_count = int(data[0]) @@ -72,10 +90,13 @@ def test_imap_access(): assert typ == "OK", "Expunge did not return OK status" -def test_imap_delete_all(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_delete_all(username, password): """Test deleting messages from the IMAP server.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() assert typ == "OK", "Select did not return OK status" typ, data = m.search(None, "ALL") @@ -92,10 +113,13 @@ def test_imap_delete_all(): assert typ == "OK", "Expunge did not return OK status" -def test_imap_messages_cleared(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_messages_cleared(username, password): """Test that all messages were expunged.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() assert typ == "OK", "Select did not return OK status" message_count = int(data[0]) From f25e910da6c2a3711a13bdf0f78a29a1ed7024e7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:39:40 -0400 Subject: [PATCH 035/417] Update documentation for user secrets --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 312190a..06cfa02 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Two environment variables are used to generate the configurations at runtime: - `fullchain.pem`: public key - `privkey.pem`: private key -- `mailarchive_password.txt`: password for the mailarchive user +- `users.txt`: account credentials to create at startup ### Volumes ### From 5d22bb478e0d020f929fee7a3f558af40dce93a5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:40:23 -0400 Subject: [PATCH 036/417] Bump version --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index f102a9c..3b93d0b 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "0.0.2" From 38e4868e1aa3b1e52765c5337879baba702b5f4f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 3 Jun 2019 21:48:03 -0400 Subject: [PATCH 037/417] Add version slices to docker deploy --- .travis.yml | 16 ++++++++++++++-- travis_scripts/build_docker_image.sh | 9 --------- travis_scripts/deploy_to_docker_hub.sh | 9 --------- 3 files changed, 14 insertions(+), 20 deletions(-) delete mode 100755 travis_scripts/build_docker_image.sh delete mode 100755 travis_scripts/deploy_to_docker_hub.sh diff --git a/.travis.yml b/.travis.yml index c488104..8e952b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ cache: - "$HOME/.cache/pre-commit" install: - pip install --upgrade -r requirements-test.txt - - travis_scripts/build_docker_image.sh + - docker build -t "$IMAGE_NAME" . before_script: - docker-compose up -d script: @@ -33,9 +33,21 @@ script: - pytest -v after_script: - docker-compose down +before_deploy: + - version=$(./bump_version.sh show) + - IFS='.' read -r -a version_array <<< "$version" + - docker login -u "$DOCKER_USER" -p "$DOCKER_PW" + - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" + - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version}" + - docker tag "$IMAGE_NAME" + "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" + - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" deploy: - provider: script - script: bash travis_scripts/deploy_to_docker_hub.sh + script: docker push "${IMAGE_NAME}:latest" && + docker push "${IMAGE_NAME}:${version}" && + docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" && + docker push "${IMAGE_NAME}:${version_array[0]}" on: tags: true python: '3.7' diff --git a/travis_scripts/build_docker_image.sh b/travis_scripts/build_docker_image.sh deleted file mode 100755 index 0a89fed..0000000 --- a/travis_scripts/build_docker_image.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o pipefail - -version=$(./bump_version.sh show) -# shellcheck disable=SC2140 -docker build -t "$IMAGE_NAME":"$version" -t "$IMAGE_NAME":"latest" . diff --git a/travis_scripts/deploy_to_docker_hub.sh b/travis_scripts/deploy_to_docker_hub.sh deleted file mode 100755 index 2be3e20..0000000 --- a/travis_scripts/deploy_to_docker_hub.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o pipefail - -echo "$DOCKER_PW" | docker login -u "$DOCKER_USER" --password-stdin -version=$(./bump_version.sh show) -docker push "$IMAGE_NAME":"$version" From 2c479d1434cab334d07e8b205746b5cef2748486 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 4 Jun 2019 14:00:22 -0400 Subject: [PATCH 038/417] Spell out flag names --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e952b9..1f956d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,19 +24,19 @@ cache: directories: - "$HOME/.cache/pre-commit" install: - - pip install --upgrade -r requirements-test.txt - - docker build -t "$IMAGE_NAME" . + - pip install --upgrade --requirement requirements-test.txt + - docker build --tag "$IMAGE_NAME" . before_script: - - docker-compose up -d + - docker-compose up --detach script: - pre-commit run --all-files - - pytest -v + - pytest --verbose after_script: - docker-compose down before_deploy: - version=$(./bump_version.sh show) - IFS='.' read -r -a version_array <<< "$version" - - docker login -u "$DOCKER_USER" -p "$DOCKER_PW" + - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version}" - docker tag "$IMAGE_NAME" From 06c04b94beda6bc253022c1dfc2d6ef3ab7f4074 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 4 Jun 2019 14:17:16 -0400 Subject: [PATCH 039/417] Rename repo --- CONTRIBUTING.md | 10 +++++----- README.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 59a01fc..f03cb56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Before contributing, we encourage you to read our CONTRIBUTING policy If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/docker-postfix/issues) in this +issue](https://github.com/cisagov/postfix-docker/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/docker-postfix/pulls), you will +request](https://github.com/cisagov/postfix-docker/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -78,9 +78,9 @@ can create and configure the Python virtual environment with these commands: ```bash -cd docker-postfix -pyenv virtualenv docker-postfix -pyenv local docker-postfix +cd postfix-docker +pyenv virtualenv postfix-docker +pyenv local postfix-docker pip install -r requirements-dev.txt ``` diff --git a/README.md b/README.md index 06cfa02..20012dd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# docker-postfix 🐳📮 # +# postfix-docker 📮🐳 # -[![Build Status](https://travis-ci.com/cisagov/docker-postfix.svg?branch=develop)](https://travis-ci.com/cisagov/docker-postfix) +[![Build Status](https://travis-ci.com/cisagov/postfix-docker.svg?branch=develop)](https://travis-ci.com/cisagov/postfix-docker) Creates a Docker container with an installation of the [postfix](http://postfix.org) MTA. Additionally it has an IMAP From b36365282560d5f0787136f48a338ad1a3a0493b Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 4 Jun 2019 15:00:13 -0400 Subject: [PATCH 040/417] Spell out flag names --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index aada9d7..d302749 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = -v -ra +addopts = --verbose -ra From 121f27c2f048e07a581d3e4651655a4b89f4ba51 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 10:49:22 -0400 Subject: [PATCH 041/417] Remove non-skel files --- src/docker-entrypoint.sh | 90 ------------------------------------ src/secrets/fullchain.pem | 31 ------------- src/secrets/privkey.pem | 52 --------------------- src/secrets/users.txt | 10 ---- src/templates/TrustedHosts | 4 -- src/templates/dovecot.conf | 38 --------------- src/templates/main.cf | 32 ------------- src/templates/master.cf | 9 ---- src/templates/opendkim.conf | 17 ------- src/templates/opendmarc.conf | 10 ---- 10 files changed, 293 deletions(-) delete mode 100755 src/docker-entrypoint.sh delete mode 100644 src/secrets/fullchain.pem delete mode 100644 src/secrets/privkey.pem delete mode 100644 src/secrets/users.txt delete mode 100644 src/templates/TrustedHosts delete mode 100644 src/templates/dovecot.conf delete mode 100644 src/templates/main.cf delete mode 100644 src/templates/master.cf delete mode 100644 src/templates/opendkim.conf delete mode 100644 src/templates/opendmarc.conf diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh deleted file mode 100755 index ce91e89..0000000 --- a/src/docker-entrypoint.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2016 - -set -e -#set -x - - -function generate_configs() { - # configure postfix - echo "Generating postfix configurations for ${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf - cp /etc/postfix/master.cf.orig /etc/postfix/master.cf - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendkim.conf > /etc/opendkim.conf - - # configure opendkim - echo "Generating opendkim configurations for ${PRIMARY_DOMAIN}" - mkdir -p "/etc/opendkim/keys/${PRIMARY_DOMAIN}" - opendkim-genkey --verbose --bits=1024 --selector=mail --directory="/etc/opendkim/keys/${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/TrustedHosts > /etc/opendkim/TrustedHosts - cp /etc/default/opendkim.orig /etc/default/opendkim - echo 'SOCKET="inet:12301"' >> /etc/default/opendkim - chown -R opendkim:opendkim /etc/opendkim - - # configure opendmarc - echo "Generating opendmarc configurations for ${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendmarc.conf > /etc/opendmarc.conf - mkdir "/etc/opendmarc/" - echo "localhost" > /etc/opendmarc/ignore.hosts - chown -R opendmarc:opendmarc /etc/opendmarc - cp /etc/default/opendmarc.orig /etc/default/opendmarc - echo 'SOCKET="inet:54321"' >> /etc/default/opendmarc - - # configure dovecot - echo "Generating dovecot configurations for ${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/dovecot.conf > /etc/dovecot/dovecot.conf - - # create a file marking the configuration as completed for this domain - echo "All configurations generated for ${PRIMARY_DOMAIN}" -} - - -function generate_users() { - echo "Generating users and passwords:" - echo "--------------------------------------------" - while IFS=" " read -r username password || [ -n "$username" ] - do - if [ -z "$password" ]; then password=$(diceware -d-); - echo -e "$username\t$password" - else - echo -e "$username\t" - fi - adduser "$username" --quiet --disabled-password --shell /usr/sbin/nologin --gecos "" &>/dev/null || true - echo "$username:$password" | chpasswd || true - done - echo "--------------------------------------------" -} - - -if [ "$1" = 'postfix' ]; then - echo "Starting mail server with:" - echo " PRIMARY_DOMAIN=${PRIMARY_DOMAIN}" - echo " RELAY_IP=${RELAY_IP}" - - # check to see if the configuration was completed for this domain - if [[ ! -f conf_gen_done.txt ]] || [[ $(< conf_gen_done.txt) != "${PRIMARY_DOMAIN}" ]]; then - generate_configs - echo "${PRIMARY_DOMAIN}" > conf_gen_done.txt - else - echo "Configurations already generated for ${PRIMARY_DOMAIN}, preserving." - fi - - # generate the users from the secrets - grep -v '^#\|^$' /run/secrets/users.txt | generate_users - - # postfix needs fresh copies of files in its chroot jail - cp /etc/{hosts,localtime,nsswitch.conf,resolv.conf,services} /var/spool/postfix/etc/ - - echo "DKIM DNS entry:" - echo "--------------------------------------------" - cat "/etc/opendkim/keys/${PRIMARY_DOMAIN}/mail.txt" - echo "--------------------------------------------" - - opendmarc - opendkim - dovecot - exec "$@" -fi - -exec "$@" diff --git a/src/secrets/fullchain.pem b/src/secrets/fullchain.pem deleted file mode 100644 index ddc24ba..0000000 --- a/src/secrets/fullchain.pem +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFXTCCA0WgAwIBAgIJAPWv/2ssPwHVMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQwHhcNMTkwNTI4MjAxNDM0WhcNMjAwNTI3MjAxNDM0WjBF -MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 -ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC -CgKCAgEAvclFvQ6WAkQXpwNksjjojlvAKkqTnHJ8vHaM0C3yrSm+aMPH0/lzLTuT -pv5eaSBUUzi5f/VjBFslH7kAGct4m1MJUfxRYdP4uZXqnfkiMyT8x7z+k6SbD22U -6Xxa7yV+hwkbhNDFOmcCWawgrERvfkSdyp/l94u+TWg5v/LvmkmsFRixT+U5dl/g -vSdXbAvjdrn+x/IRVMFrEDTm5QNCHrx1lTQf4giFl7VU820HQiNT3Y3JD7v+JHLO -DUyYZAA6bpg5vYYMbhxfegZu/C7DkT3/ZtpRXEPv1mR+koS7nPAtZmJ8t76GR/eU -A5rYLv1P6a4KvUf0/uRPGHQZsj57lVRqyR3TDi37aDywvcdTBQZHe9fNyYYe7g6s -ToKzY/Z+KwVuI+KcamVH6QLLhHOZ7IFIdB5PrSCMLHlDRP22GWV5lbFjxP+9H/2P -B5QkoBuPS4vV/GNxMlGlnbCoVDdUluypnsl0pdpae36PiFPlkA3dzJ1OcOt803jR -E3HHBg4Mq9rO51NQfxx3LsnKPVvajVChcEie1UEH+DQLEVciRBIp9Jho4SMh9xS9 -VXFRXXrnEKkfLOXwkd3Vy4upvyQrxpEMmj1SHSsFqJ4xz1uo5akpgwzB5I14oe+k -OK/sLP+B4GryWp78cASQQ/0ldbvOWxBxm7OVSPQB6F6vHVKNExsCAwEAAaNQME4w -HQYDVR0OBBYEFOAh81Tup7BSdYBjAPGyd1GW5GJhMB8GA1UdIwQYMBaAFOAh81Tu -p7BSdYBjAPGyd1GW5GJhMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB -ALBpfqJWpTf58mv1n6HuSLI7n/FGA0wxoVPmvG9qBV0uBx0kbsuJVBUXfO+FE/3x -aiTiCwKweqdRhA2H7TnGs12D1Dweh3jd1gs+inClN7w4Ge9hIMiDcc9K45wgP/Sk -IIJYgLfctGfzc5EetqWQFq1GEPpNPNpfBMC7Z/KcgiZ8+RmoIACDlJ+EwSkDuXgf -045n45cF2xqfpeq/qKZDfWwg5+js/LN7abDRxX+JjH1i678Wx+SOlxsRK2plmmIL -c46vSErWoRddwxyZyP4TqBCTgV6ZcoZSWKFvHrn4c2YZHMuagsBaDIBg05jfxv72 -ewuKeWIC0/2PkbJ+W15X+/Ltgru7gIcidt8Xm+JnBjvz0bCiS2qVGxu0DxBmbmjv -CW3pvaNhc7NWxnTbJdpC/G2wH1RV06CS2WIWuBXu/AlkVlI/HHqYD4fWdETx9VjM -821dJY0oGBVqK+2/2d9Q1J1bfzs/J2kLaNocERog2RMapygHoNI0qtfsQIyEecGw -LoDcYltzKM+tPbWsYc1lI1rNo0v0/Y2TjQ6Jq+P4eZQ9gf6XgmdLg5nIDPkO4til -epYT2sgG6TAMyzf475BX92ect9KWD1efAavL+aSxwpngWBc7uqYvvbX8w1EePX7G -EoxN6uctuyBtDpzhdKxEEVBZ7NfU6X/91ZgVdlOR/rN4 ------END CERTIFICATE----- diff --git a/src/secrets/privkey.pem b/src/secrets/privkey.pem deleted file mode 100644 index 52e5024..0000000 --- a/src/secrets/privkey.pem +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC9yUW9DpYCRBen -A2SyOOiOW8AqSpOccny8dozQLfKtKb5ow8fT+XMtO5Om/l5pIFRTOLl/9WMEWyUf -uQAZy3ibUwlR/FFh0/i5leqd+SIzJPzHvP6TpJsPbZTpfFrvJX6HCRuE0MU6ZwJZ -rCCsRG9+RJ3Kn+X3i75NaDm/8u+aSawVGLFP5Tl2X+C9J1dsC+N2uf7H8hFUwWsQ -NOblA0IevHWVNB/iCIWXtVTzbQdCI1PdjckPu/4kcs4NTJhkADpumDm9hgxuHF96 -Bm78LsORPf9m2lFcQ+/WZH6ShLuc8C1mYny3voZH95QDmtgu/U/prgq9R/T+5E8Y -dBmyPnuVVGrJHdMOLftoPLC9x1MFBkd7183Jhh7uDqxOgrNj9n4rBW4j4pxqZUfp -AsuEc5nsgUh0Hk+tIIwseUNE/bYZZXmVsWPE/70f/Y8HlCSgG49Li9X8Y3EyUaWd -sKhUN1SW7KmeyXSl2lp7fo+IU+WQDd3MnU5w63zTeNETcccGDgyr2s7nU1B/HHcu -yco9W9qNUKFwSJ7VQQf4NAsRVyJEEin0mGjhIyH3FL1VcVFdeucQqR8s5fCR3dXL -i6m/JCvGkQyaPVIdKwWonjHPW6jlqSmDDMHkjXih76Q4r+ws/4HgavJanvxwBJBD -/SV1u85bEHGbs5VI9AHoXq8dUo0TGwIDAQABAoICAQCRaDhKVXaRXeJRT8RC2F81 -Uw60WFcoMn9nVd0lU07vZWBBnF7qBeE88rx54cIsAV0aNgfKBhRLLhoPaAqvuLk7 -KC+n5Q3lSiby6e3MAyk0zk3uKttR+3fiJi9FhMWXHL8Ibu3qoJm72Vhvo/WUhwp1 -T9UlfcUQGL1BSW2Vp2f0aiWyNC0F7bZM/8CMrCvK2ID6Yh7WypyEt3xz+lQ9enWa -XwInwrv6zlSsm33u08YP4klLImq952ccPempPtozJAmg2njCwIWdh5ePQoaeKKYm -Db4062gSrOqA9JYVZCTqZQoju6majhsL4KBC8sxXlDU58OLBivQmpn4DWlClxEGi -IbY/FIE6WEhOrdoGPzIjAcC3OYYTasIMBDLdA0tODmtv9Nvst30IGZc4Pm/QIJOk -EGJo4hqWbxiy4gisWxHwYeQ9/EEwrrc3FP94VscVkT8x0i22w5WMLtcrnCGpwzMg -E10+9v4ZUZ7cu9V+IeWQUkeuP3xhumI7RIDVRHpGC6TfEk/Q2gNdsPL2E8ng2Ytx -KMI3Pj5FuYi7enIR9AWdBVmVc2u7nzJMF/ODAwY6GmqHxni7PD97cnYwCy7Gxp/S -DZqiiD32RHwUwBm0AgdLhftkgqyTN/qo/Bhmj9ieO2CkuAvTYoXG0VMzxCb9wBG/ -7BJSGcbwtTJOJGK7LvrDAQKCAQEA6Q45teOKcmOSw5ne2cXzXuaXZ0OOCkjJ2ens -M89YmKXDVEZRbGoHVtftInUpr0H2UJ/N268Ogfzw62enZ40WIGwNALvp9PkLvdT0 -6LD/4MhcgZGQ5WDwqfqwkOanHdw9HJb752yEJ+3OG+fojmKkOs6OoQk1Ypxv5+5K -OuG/qtiKKpSLbG/nKAbPsPObArBxyfH9pV5F2E6vy38lYoDTURlA2BXHPoXu9M4c -/K2BMmO5zvGu5VOpAtnag5CWUwVvnX9DKDYs+k+exErluEj+U8GbKNQUTE+1p6fT -j4KKNVZBgnavOST3Xm/i4qVbccF/CwUc387HPdK5FU6kn3evewKCAQEA0HiEAytq -jzlBBHm892tojRzvpQa65fT7khsxETLhABvqeWZ2h9lE8TJTLC46N4cG1MC/hnWB -Q7XzKd7jAeht41Lp0mlDWv6eqKN4VyXSpAYzATcEO739eja7WNTgkYB91eDSyT+K -DVaElaXMjw/uX9tBnqaVyEe8JDqHw9E3Gl0MLWi89ztYptaWvKjt0+QqENBc6o+G -K/qzO+B4o9AyjyYkUYVA87tRrDk746LA5DbkpLQKPmQ3lb1hvVysJOnEdRabu5ly -mC0HR9n2UwcU98Op/EX3D4MuCUoFB/HQNMXq7oRMg+AcfsG0/ENcbiY6o0yRhxHu -ACgcjTi/QKAI4QKCAQBbgzB6EZ0diafpkpQFI0uLKjStYcN2mlpYbRhIx9RcLErk -3q++SGwVV7hP3X2+ycH0qqtk5fpmZHIdnZgIe0gC9yqr7R3TCa/onKSGcmonU8Wv -Qv+IcmZN+Jg4bbmVahO9FDRaDSxfmWtjXc7dijI+vTkYVstVq2PtyI3xTQ+8AEdQ -rP+KVu6HsxT+wMlPZwVnbNRSiRAX/d3dpFGDul4/7BCgSPzxuhm4mu6a8W5X4Pzn -G9O3TQCClBTPsIi2lN3dFEnEknFa4MTRAy/tCwyCyvUoNQ67YFlOOgJCydmHVBVp -Kz1mzPMta/XFVXTw2DAQnbNW1pU523K9wSG3VIHdAoIBACJTZbE76dzRWZJKFUJM -DjgGBrOOiyGoF/Azx/2D+iZRcmcw5t1xefeZCLbimbVg51AKuL6EBJfIktRXHdvH -kKh4k4WQzYVjHW65E+yNjsRxPN67V1ga7Wy9LFXxH1T16kJYNXzrmGif0U7usOLx -hZeE+6YK2ejTXvg8JvSoM0GFBqdHcq3muK8n8EP6MMbN79s648G/hiEhs3dte4/F -jT2i0yIVJd+7/TO1bNYLi2VIYJd6CaHCUKC4QSqz4qhlUXLSGSxnlMXXzDYZfoSn -St2M+yVNw+Nq/x6KcI+hUl4OJKPHZu3j7e01Kf7LfKGqa8dNqTyrSBwAfssGB/+1 -GiECggEAJD0KWTfJrSbgCkMfp1fNkwNExW2+neB+MI1eIR1sWsu8rz1a5d/NIdQq -pkoJp4FQUgRFEK+CzPWbKBDOxDVwpZ5o84JzxAEc78tL8/QIYwbtw5ZOiHNZ+wS6 -OYk6weY7rro7PwzqsTXcGdg/yxtphwguveSQM8y6McqBNZKqlN2fvXY8a4KZtt8O -RXBwpsqYulHpMGPh2MsMJBGEEII7Y2WKZG41oU1SGb5J2tBdGixW0buQnr6qwBgL -Ie8VV5kgbei97WK1lwvosn3HetBYSEE0GWMvjx93yoeozV8L/IF1rf7xss2BSqzF -UjgsHxWMDJWcER8NHXkE5DQORLtKCA== ------END PRIVATE KEY----- diff --git a/src/secrets/users.txt b/src/secrets/users.txt deleted file mode 100644 index 86cd38b..0000000 --- a/src/secrets/users.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Define the users to be created at container startup. -# If is omitted for a user it will be generated and logged at startup -# username - -# The mailarchive user is mandatory since all mail is BCC'd to this user. -mailarchive foobar - -# define other users below as needed -testsender1 lemmy is god -testsender2 diff --git a/src/templates/TrustedHosts b/src/templates/TrustedHosts deleted file mode 100644 index 53aaa67..0000000 --- a/src/templates/TrustedHosts +++ /dev/null @@ -1,4 +0,0 @@ -127.0.0.1 -localhost -${PRIMARY_DOMAIN} -${RELAY_IP} diff --git a/src/templates/dovecot.conf b/src/templates/dovecot.conf deleted file mode 100644 index 167b8a6..0000000 --- a/src/templates/dovecot.conf +++ /dev/null @@ -1,38 +0,0 @@ -disable_plaintext_auth = no -mail_privileged_group = mail -mail_location = mbox:~/mail:INBOX=/var/mail/%u -userdb { - driver = passwd -} -passdb { - args = %s - driver = pam -} -protocols = " imap" -protocol imap { - mail_plugins = " autocreate" -} -plugin { - autocreate = Trash - autocreate2 = Sent - autosubscribe = Trash - autosubscribe2 = Sent -} -service imap-login { - inet_listener imap { - port = 0 - } - inet_listener imaps { - port = 993 - } -} -service auth { - unix_listener /var/spool/postfix/private/auth { - group = postfix - mode = 0660 - user = postfix - } -} -ssl=required -ssl_cert = Date: Thu, 6 Jun 2019 10:49:32 -0400 Subject: [PATCH 042/417] revert version --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 3b93d0b..f102a9c 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.2" +__version__ = "0.0.1" From bb0911a323f43938d590b149af93838781e1257a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 10:49:52 -0400 Subject: [PATCH 043/417] Update project name --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f03cb56..958c559 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Before contributing, we encourage you to read our CONTRIBUTING policy If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/postfix-docker/issues) in this +issue](https://github.com/cisagov/skeleton-docker/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/postfix-docker/pulls), you will +request](https://github.com/cisagov/skeleton-docker/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -78,9 +78,9 @@ can create and configure the Python virtual environment with these commands: ```bash -cd postfix-docker -pyenv virtualenv postfix-docker -pyenv local postfix-docker +cd skeleton-docker +pyenv virtualenv skeleton-docker +pyenv local skeleton-docker pip install -r requirements-dev.txt ``` From e5adf50959f5cf2a598bd61d251b30a08592ceb4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:45:02 -0400 Subject: [PATCH 044/417] Rename project and stow ciphertext --- .travis.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1f956d5..f60659a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,20 +5,20 @@ python: 3.7 services: docker env: global: - - IMAGE_NAME=dhsncats/postfix + - IMAGE_NAME=dhsncats/example - DOCKER_USER=felddy - - secure: >- - R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e - zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y - 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 - d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 - KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG - qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq - JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 - EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q - 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI - tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR - n2KsiL8piwAEc= + # - secure: >- + # R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e + # zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y + # 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 + # d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 + # KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG + # qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq + # JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 + # EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q + # 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI + # tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR + # n2KsiL8piwAEc= cache: pip: true directories: From 57ef61d743fb04952d3e7d6e38d476e663185af9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:45:35 -0400 Subject: [PATCH 045/417] Add example docker composition for testing --- docker-compose.yml | 49 ++++++++++------------------------------------ 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1f0d962..baa392a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,53 +2,24 @@ version: "3.7" secrets: - fullchain_pem: - file: ./src/secrets/fullchain.pem - privkey_pem: - file: ./src/secrets/privkey.pem - users_txt: - file: ./src/secrets/users.txt - + quote_txt: + file: ./src/secrets/quote.txt services: - postfix: + example: build: context: . dockerfile: Dockerfile - image: dhsncats/postfix + image: dhsncats/example init: true - restart: always + restart: on-failure environment: - - PRIMARY_DOMAIN=example.com - - RELAY_IP=172.16.202.1/32 - networks: - front: - ipv4_address: 172.16.202.2 + - ECHO_MESSAGE=Hello World from docker-compose! ports: - - target: "25" - published: "1025" - protocol: tcp - mode: host - - target: "587" - published: "1587" - protocol: tcp - mode: host - - target: "993" - published: "1993" + - target: "8080" + published: "8080" protocol: tcp mode: host secrets: - - source: fullchain_pem - target: fullchain.pem - - source: privkey_pem - target: privkey.pem - - source: users_txt - target: users.txt - -networks: - front: - driver: bridge - ipam: - driver: default - config: - - subnet: 172.16.202.0/24 + - source: quote_txt + target: quote.txt From 55b43a49f199c817ab47a6abab82f6518417acb3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:52:04 -0400 Subject: [PATCH 046/417] Add sample tests for container using pytest-dockerc --- requirements-test.txt | 1 + tests/conftest.py | 23 +++++++ tests/container_test.py | 145 ++++++---------------------------------- 3 files changed, 45 insertions(+), 124 deletions(-) create mode 100644 tests/conftest.py diff --git a/requirements-test.txt b/requirements-test.txt index 51f1982..fc5504e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ pre-commit pytest +pytest-dockerc diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b060645 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +"""pytest plugin configuration. + +https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins +""" +import pytest + + +def pytest_addoption(parser): + """Add new commandline options to pytest.""" + parser.addoption( + "--runslow", action="store_true", default=False, help="run slow tests" + ) + + +def pytest_collection_modifyitems(config, items): + """Modify collected tests based on custom marks and commandline options.""" + if config.getoption("--runslow"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "slow" in item.keywords: + item.add_marker(skip_slow) diff --git a/tests/container_test.py b/tests/container_test.py index 0ccd9e9..18de7c7 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,127 +1,24 @@ #!/usr/bin/env pytest -vs -"""Tests for postfix container.""" +"""Tests for example container.""" -from email.message import EmailMessage -from imaplib import IMAP4_SSL -import smtplib - -import pytest - -MESSAGE = """ -This is a test message sent during the unit tests. -""" -DOMAIN = "example.com" -ARCHIVE_USER = "mailarchive" -ARCHIVE_PW = "foobar" -TEST_SEND_USER = "testsender1" -TEST_SEND_PW = "lemmy is god" -IMAP_PORT = 1993 - - -@pytest.mark.parametrize("port", [1025, 1587]) -@pytest.mark.parametrize("to_user", [ARCHIVE_USER, TEST_SEND_USER]) -def test_sending_mail(port, to_user): - """Send an email message to the server.""" - msg = EmailMessage() - msg.set_content(MESSAGE) - msg["Subject"] = f"Test Message on port {port}" - msg["From"] = f"test@{DOMAIN}" - msg["To"] = f"{to_user}@{DOMAIN}" - with smtplib.SMTP("localhost", port=port) as s: - s.send_message(msg) - - -@pytest.mark.parametrize( - "username,password", - [ - (ARCHIVE_USER, ARCHIVE_PW), - (TEST_SEND_USER, TEST_SEND_PW), - pytest.param(ARCHIVE_USER, TEST_SEND_PW, marks=pytest.mark.xfail), - pytest.param("your_mom", "so_fat", marks=pytest.mark.xfail), - ], -) -def test_imap_login(username, password): - """Test logging in to the IMAP server.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] -) -def test_imap_messages_exist(username, password): - """Test test existence of our test messages.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", f"Select did not return OK status for {username}" - message_count = int(data[0]) - print(f"{username} inbox message count: {message_count}") - assert message_count > 0, f"Expected message in the {username} inbox" - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] -) -def test_imap_reading(username, password): - """Test receiving message from the IMAP server.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", "Select did not return OK status" - message_count = int(data[0]) - print(f"inbox message count: {message_count}") - typ, data = m.search(None, "ALL") - assert typ == "OK", "Search did not return OK status" - message_numbers = data[0].split() - for num in message_numbers: - typ, data = m.fetch(num, "(RFC822)") - assert typ == "OK", f"Fetch of message {num} did not return OK status" - print("-" * 40) - print(f"Message: {num}") - print(data[0][1].decode("utf-8")) - # mark messag as deleted - typ, data = m.store(num, "+FLAGS", "\\Deleted") - assert ( - typ == "OK" - ), f"Storing '\\deleted' flag on message {num} did not return OK status" - # expunge all deleted messages - typ, data = m.expunge() - assert typ == "OK", "Expunge did not return OK status" - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] -) -def test_imap_delete_all(username, password): - """Test deleting messages from the IMAP server.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", "Select did not return OK status" - typ, data = m.search(None, "ALL") - assert typ == "OK", "Search did not return OK status" - message_numbers = data[0].split() - for num in message_numbers: - # mark messag as deleted - typ, data = m.store(num, "+FLAGS", "\\Deleted") - assert ( - typ == "OK" - ), f"Storing '\\deleted' flag on message {num} did not return OK status" - # expunge all deleted messages - typ, data = m.expunge() - assert typ == "OK", "Expunge did not return OK status" - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +SECRET_PW = ( + "There are no secrets better kept than the secrets everybody guesses." # nosec ) -def test_imap_messages_cleared(username, password): - """Test that all messages were expunged.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", "Select did not return OK status" - message_count = int(data[0]) - print(f"inbox message count: {message_count}") - assert message_count == 0, "Expected the inbox to be empty" +ENV_VAR = "ECHO_MESSAGE" +ENV_VAR_VAL = "Hello World from docker-compose!" + + +def test_container(dockerc): + """Verify the test composition and container.""" + assert len(dockerc.containers()) == 1, "Wrong number of containers were running." + container = dockerc.containers()[0] + assert container.is_running is True, "Expected container to be running" + assert container.wait() == 0, "Container did not exit cleanly" + assert ( + container.labels["com.docker.compose.service"] == "example" + ), "Service name was different than expected" + assert ( + container.environment[ENV_VAR] == ENV_VAR_VAL + ), f"{ENV_VAR} value was different than expected" + log_output = container.logs().decode("utf-8") + assert SECRET_PW in log_output, "Secret not found in log output." From dee0fcd9b1a35ab77da14b9dda30eccfa09bdeee Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:52:29 -0400 Subject: [PATCH 047/417] Add a secret quote --- src/secrets/quote.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/secrets/quote.txt diff --git a/src/secrets/quote.txt b/src/secrets/quote.txt new file mode 100644 index 0000000..93ee1a8 --- /dev/null +++ b/src/secrets/quote.txt @@ -0,0 +1 @@ +There are no secrets better kept than the secrets everybody guesses. From 2a6d14888784b73860a9806cf3a4b8fc91c24046 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:53:12 -0400 Subject: [PATCH 048/417] Add dockerfile to build example container --- Dockerfile | 54 ++++++++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a82707..4f844ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,28 @@ -FROM debian:buster-slim -MAINTAINER Mark Feldhousen +FROM python:3.7-alpine +LABEL maintainer="mark.feldhousen@trio.dhs.gov" -RUN apt-get update && \ -DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ -ca-certificates \ -diceware \ -dovecot-imapd \ -dovecot-lmtpd \ -gettext-base \ -mailutils \ -opendkim \ -opendkim-tools \ -opendmarc \ -postfix \ -procmail \ -sasl2-bin \ -&& apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -RUN adduser mailarchive --quiet --disabled-password \ ---shell /usr/sbin/nologin --gecos "Mail Archive" +ARG CISA_UID=421 +ENV CISA_HOME="/home/cisa" +ENV ECHO_MESSAGE="Hello World from Dockerfile" -USER root -WORKDIR /root +RUN addgroup --system --gid ${CISA_UID} cisa \ + && adduser --system --uid ${CISA_UID} --ingroup cisa cisa -# make backups of configurations. These are modified at startup. -RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig -RUN mv /etc/default/opendkim /etc/default/opendkim.orig -RUN mv /etc/default/opendmarc /etc/default/opendmarc.orig +RUN apk --update --no-cache add \ +ca-certificates \ +git \ +openssl \ +py-pip -COPY ./src/templates ./templates/ -COPY ./src/docker-entrypoint.sh . +WORKDIR ${CISA_HOME} +# TODO remove branch +RUN git clone --branch improvement/data https://github.com/cisagov/skeleton-python-library.git . && \ +pip install --requirement requirements.txt && \ +ln -snf /run/secrets/quote.txt src/example/data/secret.txt -VOLUME ["/var/log", "/var/spool/postfix"] -EXPOSE 25/TCP 587/TCP 993/TCP +USER cisa -ENTRYPOINT ["./docker-entrypoint.sh"] -CMD ["postfix", "-v", "start-fg"] +EXPOSE 8080/TCP +VOLUME ["/var/log"] +ENTRYPOINT ["example"] +CMD ["--log-level", "DEBUG"] From c231cc679252062043c937386e4e3696ea110be3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:53:43 -0400 Subject: [PATCH 049/417] Update README --- README.md | 69 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 20012dd..4e1eaaa 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,60 @@ -# postfix-docker 📮🐳 # +# skeleton-docker 💀🐳 # -[![Build Status](https://travis-ci.com/cisagov/postfix-docker.svg?branch=develop)](https://travis-ci.com/cisagov/postfix-docker) +[![Build Status](https://travis-ci.com/cisagov/skeleton-docker.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-docker) -Creates a Docker container with an installation of the -[postfix](http://postfix.org) MTA. Additionally it has an IMAP -server ([dovecot](https://dovecot.org)) for accessing the archvies -of sent email. All email is BCC'd to the `mailarchive` account. +## Docker Image ## + +![MicroBadger Layers](https://img.shields.io/microbadger/layers/dhsncats/example.svg) +![MicroBadger Size](https://img.shields.io/microbadger/image-size/dhsncats/example.svg) + +This is a docker skeleton project that can be used to quickly get a +new [cisagov](https://github.com/cisagov) GitHub docker project started. +This skeleton project contains [licensing information](LICENSE.md), as +well as [pre-commit hooks](https://pre-commit.com) and a [Travis +CI](https://travis-ci.com) configuration appropriate for docker +containers and the major languages that we use. ## Usage ## -A sample [docker composition](docker-compose.yml) is included in this repository. -To build and start the container use the command: `docker-compose up` +### Install ### -### Ports ### +Pull `dhsncats/example` from the Docker repository: -This container exposes the following ports: + docker pull dhsncats/example -- 25: `smtp` -- 587: `submission` -- 993: `imaps` +Or build `dhsncats/example` from source: -The sample [docker composition](docker-compose.yml) publishes the -exposed ports at 1025, 1587, and 1993. + git clone https://github.com/cisagov/skeleton-docker.git + cd skeleton-docker + docker-compose build -### Environment Variables ### +### Run ### + +## Ports ## + +This container exposes the following ports: -Two environment variables are used to generate the configurations at runtime: +| Port | Protocol | Service | +|-------|----------|----------| +| 8080 | TCP | http | -- `PRIMARY_DOMAIN`: the domain of the mail server -- `RELAY_IP`: (optional) an IP address that is allowed to relay mail without authentication +## Environment Variables ## -### Secrets ### +| Variable | Default Value | Purpose | +|---------------|-------------------------------|--------------| +| ECHO_MESSAGE | `Hello World from Dockerfile` | Text to echo | -- `fullchain.pem`: public key -- `privkey.pem`: private key -- `users.txt`: account credentials to create at startup +## Secrets ## -### Volumes ### +| Filename | Purpose | +|---------------|----------------------| +| quote.txt | Secret text to echo | -Two optional volumes can be attached to this container to persist the -mail spool directory, as well as the logging directory. (Note that -the mail logs are available using the docker log command.) +## Volumes ## -- `/var/spool/postfix`: mail queues -- `/var/log`: system logs +| Mount point | Purpose | +|-------------|----------------| +| /var/log | logging output | ## Contributing ## From 4c3aeda22523270b8ee53ed37b76fa4db5ce9e86 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:55:11 -0400 Subject: [PATCH 050/417] Remove composition up and down. This is now handled by pytest itself --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f60659a..b5a5e27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,13 +26,9 @@ cache: install: - pip install --upgrade --requirement requirements-test.txt - docker build --tag "$IMAGE_NAME" . -before_script: - - docker-compose up --detach script: - pre-commit run --all-files - pytest --verbose -after_script: - - docker-compose down before_deploy: - version=$(./bump_version.sh show) - IFS='.' read -r -a version_array <<< "$version" From 340f6f7809db2d0294b14eb07ed8e48fbdd78d65 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:58:47 -0400 Subject: [PATCH 051/417] Update ciphertext for DOCKER_PW --- .travis.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5a5e27..861470d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,18 +7,18 @@ env: global: - IMAGE_NAME=dhsncats/example - DOCKER_USER=felddy - # - secure: >- - # R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e - # zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y - # 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 - # d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 - # KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG - # qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq - # JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 - # EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q - # 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI - # tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR - # n2KsiL8piwAEc= + - secure: >- + GWxja6rCcevNRay3uOGEt1wpzGOAoKgfquyror60VtkG07GCJ/rCBa/jNsTRRwwDxYW + MqY6xQlAU+GzNeuBxKAlmyrLRRYZDmcBpPgQQPbiC0b5ftGCzTuhdYAj4Hg+LyHKbNP + H0zsXfQRy2NdWPJ8rr3RFL89a/If0hJVL5OEEHfmzpfvHoDvBJQwhmL2sPZizgCsgh6 + eRStRvUWj3taVb5VZ1uIlet8P3g7azkD7tTNx+wAxhjlLpaQfBcy/CC27ogw+QOvgdF + kefskYz54LLzTyXYhRhACjuGaukpBvQBFzBi3MOFYoLj4kb+9b3mqHlnQZLOSo2zlWV + 30mTC7eed5KE64MuQ6I1E72B0rnAoVpnwAjXVIvzfhKBuzbhS54tWjKbqK6UAGxbg6t + wJDiNaB+nYk6zozJ93TQT02XDUOdDS45hO2Kc/CUwJbrZc7V4aYN6Dj30tWy8d2TI8s + GMPuJm8tyxpT/h2eUUPhVzhMCk+4efIBa6cb99wZgkphLmWQefLLpZy8pMwOH7c3ywU + +6tVPBZxSEm4tFRYyPwgl9qvzE8E0WM3FPNgwVO9aypx61HcO8NVD7alUW2j+s3taHX + aVpaCtsf+bpsXPro3/ofst+FHiFPZGBL2iiSIYtFYI4hkDFq9iGGGexo9zgYS6CMexu + YoZtThDw6kaLI= cache: pip: true directories: From 5cd2569407fed07f1b90c7a40820fb4f5f94c518 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 15:08:58 -0400 Subject: [PATCH 052/417] Remove branch checkout --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4f844ce..033fe0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,8 @@ openssl \ py-pip WORKDIR ${CISA_HOME} -# TODO remove branch -RUN git clone --branch improvement/data https://github.com/cisagov/skeleton-python-library.git . && \ + +RUN git clone https://github.com/cisagov/skeleton-python-library.git . && \ pip install --requirement requirements.txt && \ ln -snf /run/secrets/quote.txt src/example/data/secret.txt From b58fa62c004d4ffe42188f258653211ee85d6405 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 6 Jun 2019 16:12:58 -0400 Subject: [PATCH 053/417] Add changes from running pre-commit autoupdate --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6a0939..e9029c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.17.1 + rev: v1.18.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -64,7 +64,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.11.0 + rev: v1.12.0 hooks: - id: terraform_fmt - id: terraform_validate_no_variables From 740457694ab4f14fbe89583f14de86aab064765b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 6 Jun 2019 16:33:07 -0400 Subject: [PATCH 054/417] Remove the terraform-docs hook terraform-docs is currently broken with respect to terraform 0.12. --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9029c0..dd98ddc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,6 @@ repos: hooks: - id: terraform_fmt - id: terraform_validate_no_variables - - id: terraform_docs - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.0 hooks: From 0d10121e65a1083c56eb6720338f8ce490a14e47 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 18:52:59 -0400 Subject: [PATCH 055/417] Add version label --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 033fe0b..6cc894e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.7-alpine LABEL maintainer="mark.feldhousen@trio.dhs.gov" +LABEL version="0.0.1" ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" From bd4206bbcd94f00aaa3ae8a4b1daf8db002120f1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 7 Jun 2019 11:19:41 -0400 Subject: [PATCH 056/417] Configure prettier to ignore JSON files JSON files are already being linted by the pretty-format-json hook. --- .prettierignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.prettierignore b/.prettierignore index f1e840c..738d402 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,5 @@ +# Already being linted by pretty-format-json +*.json # Already being linted by mdl *.md # Already being linted by yamllint From 365fff926c7c11d266d4cc64148f2d2d9b283245 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:28:22 -0400 Subject: [PATCH 057/417] Add robust container tests --- tests/conftest.py | 21 ++++++++++++++++++ tests/container_test.py | 49 +++++++++++++++++++++++++++++------------ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b060645..d62dff6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,27 @@ import pytest +MAIN_SERVICE_NAME = "example" +VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" + + +@pytest.fixture(scope="session") +def main_container(dockerc): + """Return the main container from the docker composition.""" + # find the container by name even if it is stopped already + return dockerc.containers(service_names=[MAIN_SERVICE_NAME], stopped=True)[0] + + +@pytest.fixture(scope="session") +def version_container(dockerc): + """Return the version container from the docker composition. + + The version container should just output the version of its underlying contents. + """ + # find the container by name even if it is stopped already + return dockerc.containers(service_names=[VERSION_SERVICE_NAME], stopped=True)[0] + + def pytest_addoption(parser): """Add new commandline options to pytest.""" parser.addoption( diff --git a/tests/container_test.py b/tests/container_test.py index 18de7c7..494ad45 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,24 +1,45 @@ #!/usr/bin/env pytest -vs """Tests for example container.""" -SECRET_PW = ( - "There are no secrets better kept than the secrets everybody guesses." # nosec -) ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" +SECRET_QUOTE = ( + "There are no secrets better kept than the secrets everybody guesses." # nosec +) +VERSION_FILE = "src/version.txt" -def test_container(dockerc): +def test_container_count(dockerc): """Verify the test composition and container.""" - assert len(dockerc.containers()) == 1, "Wrong number of containers were running." - container = dockerc.containers()[0] - assert container.is_running is True, "Expected container to be running" - assert container.wait() == 0, "Container did not exit cleanly" + assert len(dockerc.containers()) == 2, "Wrong number of containers were running." + + +def test_wait_for_exits(main_container, version_container): + """Wait for containers to exit.""" + assert main_container.wait() == 0, "Container service (main) did not exit cleanly" + assert ( + version_container.wait() == 0 + ), "Container service (version) did not exit cleanly" + + +def test_output(main_container): + """Verify the container had the correct output.""" + main_container.wait() # make sure container exited if running test isolated + log_output = main_container.logs().decode("utf-8") + assert SECRET_QUOTE in log_output, "Secret not found in log output." + + +def test_version_matches(version_container): + """Verify the container outputs the correct version.""" + version_container.wait() # make sure container exited if running test isolated + log_output = version_container.logs().decode("utf-8").strip() + pkg_vars = {} + with open(VERSION_FILE) as f: + exec(f.read(), pkg_vars) # nosec + project_version = pkg_vars["__version__"] assert ( - container.labels["com.docker.compose.service"] == "example" - ), "Service name was different than expected" + log_output == project_version + ), f"Container version output does not match project version file {VERSION_FILE}" assert ( - container.environment[ENV_VAR] == ENV_VAR_VAL - ), f"{ENV_VAR} value was different than expected" - log_output = container.logs().decode("utf-8") - assert SECRET_PW in log_output, "Secret not found in log output." + version_container.labels["version"] == project_version + ), "Dockerfile version label does not match project version" From adcf39e6e86c105ba0c02072a8accdee704734a8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:28:45 -0400 Subject: [PATCH 058/417] Add second instance of container for version test --- docker-compose.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index baa392a..8579d0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,21 @@ --- version: "3.7" +# This docker-compose file is used to build and test the container + secrets: quote_txt: file: ./src/secrets/quote.txt services: example: + # Run the container normally build: context: . dockerfile: Dockerfile image: dhsncats/example init: true - restart: on-failure + restart: "no" environment: - ECHO_MESSAGE=Hello World from docker-compose! ports: @@ -23,3 +26,10 @@ services: secrets: - source: quote_txt target: quote.txt + + example-version: + # Run the container to collect version information + image: dhsncats/example + init: true + restart: "no" + command: --version From b63c80ffc29c85a24b0a08f2b8ae9aeb610d0011 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:29:22 -0400 Subject: [PATCH 059/417] Add useful labels --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6cc894e..88a6a82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM python:3.7-alpine +ARG VERSION="0.0.1" LABEL maintainer="mark.feldhousen@trio.dhs.gov" -LABEL version="0.0.1" +LABEL vendor="Cyber and Infrastructure Security Agency" +LABEL version=${VERSION} ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" From 5b0d395d1f47be2c3264337430908707c4d2cb2d Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:29:58 -0400 Subject: [PATCH 060/417] Use release of code instead of repo --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 88a6a82..b4dd7f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,15 +13,16 @@ RUN addgroup --system --gid ${CISA_UID} cisa \ RUN apk --update --no-cache add \ ca-certificates \ -git \ openssl \ py-pip WORKDIR ${CISA_HOME} -RUN git clone https://github.com/cisagov/skeleton-python-library.git . && \ -pip install --requirement requirements.txt && \ -ln -snf /run/secrets/quote.txt src/example/data/secret.txt +RUN wget -O sourcecode.tgz https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ + tar xzf sourcecode.tgz --strip-components=1 && \ + pip install --requirement requirements.txt && \ + ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ + rm sourcecode.tgz USER cisa From 8380ecd53b6b4a6f9103c2eb776241a5f1ff8a0b Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:25:16 -0400 Subject: [PATCH 061/417] Expose ARGs for external configuration --- Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b4dd7f9..ba75023 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,15 @@ +ARG GIT_COMMIT=unspecified +ARG GIT_REMOTE=unspecified +ARG VERSION=unspecified + FROM python:3.7-alpine -ARG VERSION="0.0.1" + +ARG GIT_COMMIT +ARG GIT_REMOTE +ARG VERSION + +LABEL git_commit=${GIT_COMMIT} +LABEL git_remote=${GIT_REMOTE} LABEL maintainer="mark.feldhousen@trio.dhs.gov" LABEL vendor="Cyber and Infrastructure Security Agency" LABEL version=${VERSION} From 17751059c7e461777be8327edbc013ef940988ed Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:26:47 -0400 Subject: [PATCH 062/417] Update to show version specification for manual build --- README.md | 2 +- docker-compose.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e1eaaa..ac29e5c 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Or build `dhsncats/example` from source: git clone https://github.com/cisagov/skeleton-docker.git cd skeleton-docker - docker-compose build + docker-compose build --build-arg VERSION=0.0.1 ### Run ### diff --git a/docker-compose.yml b/docker-compose.yml index 8579d0a..f9a54ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: example: # Run the container normally build: + # VERSION must be specified on the command line: + # e.g., --build-arg VERSION=0.0.1 context: . dockerfile: Dockerfile image: dhsncats/example From f40541cd8d6ae567df08e7ec694860eb824ea2c8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:27:19 -0400 Subject: [PATCH 063/417] Add build arguments for image creation --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 861470d..a06bc93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,12 +25,17 @@ cache: - "$HOME/.cache/pre-commit" install: - pip install --upgrade --requirement requirements-test.txt - - docker build --tag "$IMAGE_NAME" . + - version=$(./bump_version.sh show) + - docker build + --tag "$IMAGE_NAME" + --build-arg GIT_COMMIT=$(git log -1 --format=%H) + --build-arg GIT_REMOTE=$(git remote get-url origin) + --build-arg VERSION=${version} + . script: - pre-commit run --all-files - pytest --verbose before_deploy: - - version=$(./bump_version.sh show) - IFS='.' read -r -a version_array <<< "$version" - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" From befbec0e6163845d1393b06bbabd6493c04cefb4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:36:27 -0400 Subject: [PATCH 064/417] Fix for travis being a bit pokey at startup --- tests/container_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index 494ad45..b6835aa 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -11,7 +11,10 @@ def test_container_count(dockerc): """Verify the test composition and container.""" - assert len(dockerc.containers()) == 2, "Wrong number of containers were running." + # stopped parameter allows non-running containers in results + assert ( + len(dockerc.containers(stopped=True)) == 2 + ), "Wrong number of containers were started." def test_wait_for_exits(main_container, version_container): From 7bd93f1ef735a1ba1025f6c7d5a4198172234f2f Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:50:01 -0400 Subject: [PATCH 065/417] Add version check for TRAVIS_TAG when available --- tests/container_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/container_test.py b/tests/container_test.py index b6835aa..fd7b085 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,6 +1,8 @@ #!/usr/bin/env pytest -vs """Tests for example container.""" +import os + ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" SECRET_QUOTE = ( @@ -46,3 +48,8 @@ def test_version_matches(version_container): assert ( version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" + travis_tag = os.getenv("TRAVIS_TAG") + if travis_tag is not None: + assert ( + travis_tag == project_version + ), "TRAVIS_TAG does not match the project version" From c88c5dc535645f0128254248f3e042d2387aa718 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:54:15 -0400 Subject: [PATCH 066/417] Travis you biatch! --- tests/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index fd7b085..45a9588 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -49,7 +49,7 @@ def test_version_matches(version_container): version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" travis_tag = os.getenv("TRAVIS_TAG") - if travis_tag is not None: + if travis_tag not in [None, ""]: assert ( travis_tag == project_version ), "TRAVIS_TAG does not match the project version" From 285d520ca239f3bc1d12b3ad8fb1c005f7cde38c Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 13:05:08 -0400 Subject: [PATCH 067/417] Travis you biatch!! --- tests/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index 45a9588..4c80ca8 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -49,7 +49,7 @@ def test_version_matches(version_container): version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" travis_tag = os.getenv("TRAVIS_TAG") - if travis_tag not in [None, ""]: + if travis_tag: assert ( travis_tag == project_version ), "TRAVIS_TAG does not match the project version" From 4913c2bece0d780db30aa497a97603827e2476b8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 13:15:50 -0400 Subject: [PATCH 068/417] =?UTF-8?q?You're=20not=20wrong=20Travis,=20you're?= =?UTF-8?q?=20just=20an=20=F0=9F=90=B4=F0=9F=95=B3.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index 4c80ca8..216d392 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -51,5 +51,5 @@ def test_version_matches(version_container): travis_tag = os.getenv("TRAVIS_TAG") if travis_tag: assert ( - travis_tag == project_version + travis_tag == project_version or travis_tag == f"v{project_version}" ), "TRAVIS_TAG does not match the project version" From fb35101cd7f6a8f206594eaf5cf98ebc03004c02 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 14:57:40 -0400 Subject: [PATCH 069/417] Add missing run instructions --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ac29e5c..aa74a59 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Or build `dhsncats/example` from source: ### Run ### + docker-compose run --rm example + ## Ports ## This container exposes the following ports: From c6f2adb6abc41b0bbb6abae88aba10cb6ab2e515 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell Date: Thu, 20 Jun 2019 15:14:12 -0400 Subject: [PATCH 070/417] Update two text blocks to use the console tag instead of the bash tag for formatting. --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ede2f47..117baba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,7 +77,7 @@ Once `pyenv` and `pyenv-virtualenv` are installed on your system, you can create and configure the Python virtual environment with these commands: -```bash +```console cd skeleton-generic pyenv virtualenv skeleton-generic pyenv local skeleton-generic @@ -88,7 +88,7 @@ pip install -r requirements-dev.txt Now setting up pre-commit is as simple as: -```bash +```console pre-commit install ``` From d90ce5509cd7da08b7d5187481561133a8bbc1dd Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Jun 2019 15:24:46 -0400 Subject: [PATCH 071/417] Add lgtm configuration for requirements-test.txt file --- .lgtm.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .lgtm.yml diff --git a/.lgtm.yml b/.lgtm.yml new file mode 100644 index 0000000..8950263 --- /dev/null +++ b/.lgtm.yml @@ -0,0 +1,8 @@ +--- +extraction: + python: + python_setup: + version: 3 + requirements_files: + - requirements-test.txt + setup_py: false From 06a2b906cef6a8f25677b251b14806b6473d12ed Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Jun 2019 15:25:32 -0400 Subject: [PATCH 072/417] Add LGTM shields --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aa74a59..0bd4127 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # skeleton-docker 💀🐳 # [![Build Status](https://travis-ci.com/cisagov/skeleton-docker.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-docker) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/context:python) ## Docker Image ## From a13597d2330f947797fce78bed3ac665c0623ddc Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:11:45 -0400 Subject: [PATCH 073/417] Update pre-commit hooks via pre-commit autoupdate --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd98ddc..eeddaed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.16.0 + rev: v0.17.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -31,7 +31,7 @@ repos: # alone. exclude: LICENSE.md - repo: https://github.com/adrienverge/yamllint - rev: v1.15.0 + rev: v1.16.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -45,11 +45,11 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.18.0 + rev: v1.19.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 1.6.0 + rev: 1.6.1 hooks: - id: bandit args: @@ -73,6 +73,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.17.1 + rev: 1.18.2 hooks: - id: prettier From 1843ca0b482624b867828c95c99faa9cb01253b7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:14:02 -0400 Subject: [PATCH 074/417] Add config file for markdownlint This config file turns off the "line too long" rule when within a code block or a table. Often there is no way to avoid going over 80 lines in such cases. --- .mdl_config.json | 7 +++++++ .pre-commit-config.yaml | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 .mdl_config.json diff --git a/.mdl_config.json b/.mdl_config.json new file mode 100644 index 0000000..492955a --- /dev/null +++ b/.mdl_config.json @@ -0,0 +1,7 @@ +{ + "MD013": { + "code_blocks": false, + "tables": false + }, + "default": true +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eeddaed..491b94c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,8 @@ repos: # GitHub's autorecognition fu to work, so we should leave it # alone. exclude: LICENSE.md + args: + - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint rev: v1.16.0 hooks: From 3fee0b86874070e9c9cb41a6d82b7904a3352590 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:17:54 -0400 Subject: [PATCH 075/417] Rename LICENSE.md to LICENSE, make the other changes that requires The license text is not actually in Markdown, and adding the .md extension actually makes it render a little differently than intended. --- .pre-commit-config.yaml | 4 ---- CONTRIBUTING.md | 4 ++-- LICENSE.md => LICENSE | 0 README.md | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) rename LICENSE.md => LICENSE (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 491b94c..a4b6299 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,10 +26,6 @@ repos: rev: v0.17.0 hooks: - id: markdownlint - # The LICENSE.md must match the license text exactly for - # GitHub's autorecognition fu to work, so we should leave it - # alone. - exclude: LICENSE.md args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 117baba..93addc2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,8 +8,8 @@ of contribution, and don't want a wall of rules to get in the way of that. Before contributing, we encourage you to read our CONTRIBUTING policy -(you are here), our [LICENSE](LICENSE.md), and our -[README](README.md), all of which should be in this repository. +(you are here), our [LICENSE](LICENSE), and our [README](README.md), +all of which should be in this repository. ## Issues ## diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE diff --git a/README.md b/README.md index 2337970..b880f4d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. -This skeleton project contains [licensing information](LICENSE.md), as +This skeleton project contains [licensing information](LICENSE), as well as [pre-commit hooks](https://pre-commit.com) and a [Travis CI](https://travis-ci.com) configuration appropriate for the major languages that we use. @@ -19,7 +19,7 @@ details. ## License ## -This project is in the worldwide [public domain](LICENSE.md). +This project is in the worldwide [public domain](LICENSE). This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through From 74ff0acee52e57f6d82458bc6ab4264701d5dd79 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 11:32:50 -0400 Subject: [PATCH 076/417] Add test to check for container readiness. --- tests/container_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/container_test.py b/tests/container_test.py index 216d392..c1cf95e 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -2,9 +2,11 @@ """Tests for example container.""" import os +import time ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" +READY_MESSAGE = "This is a debug message" SECRET_QUOTE = ( "There are no secrets better kept than the secrets everybody guesses." # nosec ) @@ -19,6 +21,20 @@ def test_container_count(dockerc): ), "Wrong number of containers were started." +def test_wait_for_ready(main_container): + """Wait for container to be ready.""" + TIMEOUT = 10 + for i in range(TIMEOUT): + if READY_MESSAGE in main_container.logs().decode("utf-8"): + break + time.sleep(1) + else: + raise Exception( + f"Container does not seem ready. " + f'Expected "{READY_MESSAGE}" in the log within {TIMEOUT} seconds.' + ) + + def test_wait_for_exits(main_container, version_container): """Wait for containers to exit.""" assert main_container.wait() == 0, "Container service (main) did not exit cleanly" From 2c6377123f3600ee8c8cec2d6c7298413a07a613 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 17:07:16 -0400 Subject: [PATCH 077/417] Add script to push readme to Docker Hub. Snarfed from: https://github.com/moikot/golang-dep/blob/aab3ea8462a19407544f1ce9daa11c3f0924394c/.travis/push.sh#L127 --- push_readme.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 push_readme.sh diff --git a/push_readme.sh b/push_readme.sh new file mode 100755 index 0000000..880b95b --- /dev/null +++ b/push_readme.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# push the README.md file to the docker hub repository + +set -o nounset +set -o errexit +set -o pipefail + +token=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d '{"username": "'"$DOCKER_USER"'", "password": "'"$DOCKER_PW"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token) + +code=$(jq -n --arg msg "$( Date: Thu, 27 Jun 2019 17:07:57 -0400 Subject: [PATCH 078/417] Add comment to identify ciphertext blob purpose --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a06bc93..112e965 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ env: global: - IMAGE_NAME=dhsncats/example - DOCKER_USER=felddy + # DOCKER_PW - secure: >- GWxja6rCcevNRay3uOGEt1wpzGOAoKgfquyror60VtkG07GCJ/rCBa/jNsTRRwwDxYW MqY6xQlAU+GzNeuBxKAlmyrLRRYZDmcBpPgQQPbiC0b5ftGCzTuhdYAj4Hg+LyHKbNP From 28ecdae1f3e12c5d1f7f358fa3baea6391ea79dc Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 17:08:39 -0400 Subject: [PATCH 079/417] Add call to push_readme script --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 112e965..cedc956 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,8 @@ deploy: script: docker push "${IMAGE_NAME}:latest" && docker push "${IMAGE_NAME}:${version}" && docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" && - docker push "${IMAGE_NAME}:${version_array[0]}" + docker push "${IMAGE_NAME}:${version_array[0]}" && + ./push_readme.sh on: tags: true python: '3.7' From 60f495c69ed5c524177c81c848d687a42fa0feeb Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 17:10:54 -0400 Subject: [PATCH 080/417] Document script inputs --- push_readme.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/push_readme.sh b/push_readme.sh index 880b95b..0b6d07a 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash -# push the README.md file to the docker hub repository +# Push the README.md file to the docker hub repository + +# Requires the following environment variables to be set: +# DOCKER_PW, DOCKER_USER, IMAGE_NAME set -o nounset set -o errexit From cb8e8ffd5d07f248f6b39d433620f0207df69fb7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 19 Jul 2019 15:45:57 -0400 Subject: [PATCH 081/417] fix issue 5 --- tests/container_test.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/container_test.py b/tests/container_test.py index c1cf95e..50b62d8 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -4,12 +4,15 @@ import os import time +import pytest + ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" READY_MESSAGE = "This is a debug message" SECRET_QUOTE = ( "There are no secrets better kept than the secrets everybody guesses." # nosec ) +TRAVIS_TAG = os.getenv("TRAVIS_TAG") VERSION_FILE = "src/version.txt" @@ -50,8 +53,22 @@ def test_output(main_container): assert SECRET_QUOTE in log_output, "Secret not found in log output." -def test_version_matches(version_container): - """Verify the container outputs the correct version.""" +@pytest.mark.skipif( + TRAVIS_TAG in [None, ""], reason="this is not a release (TRAVIS_TAG not set)" +) +def test_release_version(): + """Verify that release tag version agrees with the module version.""" + pkg_vars = {} + with open(VERSION_FILE) as f: + exec(f.read(), pkg_vars) # nosec + project_version = pkg_vars["__version__"] + assert ( + TRAVIS_TAG == f"v{project_version}" + ), "TRAVIS_TAG does not match the project version" + + +def test_log_version(version_container): + """Verify the container outputs the correct version to the logs.""" version_container.wait() # make sure container exited if running test isolated log_output = version_container.logs().decode("utf-8").strip() pkg_vars = {} @@ -60,12 +77,15 @@ def test_version_matches(version_container): project_version = pkg_vars["__version__"] assert ( log_output == project_version - ), f"Container version output does not match project version file {VERSION_FILE}" + ), f"Container version output to log does not match project version file {VERSION_FILE}" + + +def test_container_version_label_matches(version_container): + """Verify the container version label is the correct version.""" + pkg_vars = {} + with open(VERSION_FILE) as f: + exec(f.read(), pkg_vars) # nosec + project_version = pkg_vars["__version__"] assert ( version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" - travis_tag = os.getenv("TRAVIS_TAG") - if travis_tag: - assert ( - travis_tag == project_version or travis_tag == f"v{project_version}" - ), "TRAVIS_TAG does not match the project version" From 2932d35483cb8b7be71da69b7006e58298846792 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 11 Sep 2019 15:58:44 -0400 Subject: [PATCH 082/417] Add updates from running pre-commit autoupdate --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4b6299..9580afa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.3 + rev: v2.3.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,13 +23,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.17.0 + rev: v0.18.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.16.0 + rev: v1.17.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -37,17 +37,17 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.7 + rev: 3.7.8 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.19.0 + rev: v1.23.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 1.6.1 + rev: 1.6.2 hooks: - id: bandit args: @@ -57,7 +57,7 @@ repos: hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.0a0 + rev: v4.1.1a0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From 78a230d46c532176c0aa1af70b33f3bca4664243 Mon Sep 17 00:00:00 2001 From: Dave Redmin Date: Fri, 18 Oct 2019 10:10:19 -0400 Subject: [PATCH 083/417] Add updates from running pre-commit autoupdate --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9580afa..0057d09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.17.0 + rev: v1.18.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -43,7 +43,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.23.0 + rev: v1.25.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit From 05a136bb117b9fb966e0ce3bd0d9784a32f08c51 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:39:22 -0400 Subject: [PATCH 084/417] Replace Travis-CI with GitHub actions. --- .github/workflows/build.yml | 26 ++++++++++++++++++++++++++ .travis.yml | 17 ----------------- README.md | 8 ++++---- 3 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..b3e5b2f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,26 @@ +--- +name: build + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + + - name: Run pre-commit on all files + run: pre-commit run --all-files + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2c70e21..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -dist: xenial -language: python -python: 3.7 -# pre-commit hooks can use Docker, so we should go ahead and enable it -services: docker - -# Cache pip packages and pre-commit plugins to speed up builds -cache: - pip: true - directories: - - $HOME/.cache/pre-commit - -install: - - pip install --upgrade -r requirements-test.txt -script: - - pre-commit run --all-files diff --git a/README.md b/README.md index b880f4d..36c7336 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # skeleton-generic # -[![Build Status](https://travis-ci.com/cisagov/skeleton-generic.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-generic) +[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. This skeleton project contains [licensing information](LICENSE), as -well as [pre-commit hooks](https://pre-commit.com) and a [Travis -CI](https://travis-ci.com) configuration appropriate for the major -languages that we use. +well as [pre-commit hooks](https://pre-commit.com) and +[GitHub Actions](https://github.com/features/actions) configurations +appropriate for the major languages that we use. In many cases you will instead want to use one of the more specific skeleton projects derived from this one. From 02fb008d62ca07a78a0103a7af1d81b4712613c1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:40:58 -0400 Subject: [PATCH 085/417] Fix action shield link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 36c7336..606df0a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # skeleton-generic # -[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) +[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-generic/actions) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. From b31ad8591f030b2300b82775dfce2fb3c85fc8db Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:46:32 -0400 Subject: [PATCH 086/417] Disable ansible-lint until they fix their setup.py See: https://github.com/ansible/ansible-lint/issues/590 --- .pre-commit-config.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0057d09..fca1e26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,11 +56,12 @@ repos: rev: 19.3b0 hooks: - id: black - - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a0 - hooks: - - id: ansible-lint - # files: molecule/default/playbook.yml + # Disabled until: https://github.com/ansible/ansible-lint/issues/590 + # - repo: https://github.com/ansible/ansible-lint.git + # rev: v4.1.1a0 + # hooks: + # - id: ansible-lint + # # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From b11e39abd7e62b93f9a14158919bd524317ff34e Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 11:00:39 -0400 Subject: [PATCH 087/417] Remove extra line. --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3e5b2f..013abd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,4 +23,3 @@ jobs: - name: Run pre-commit on all files run: pre-commit run --all-files - From 3c78dd98a6ee54b1a102f4e4bddb6c0a31cf4c1d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 2 Nov 2019 19:48:42 -0400 Subject: [PATCH 088/417] Re-enabled ansible-lint and ran pre-commit autoupgrade. --- .pre-commit-config.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fca1e26..6d7bcbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v2.4.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.18.0 + rev: v0.19.0 hooks: - id: markdownlint args: @@ -37,13 +37,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.8 + rev: 3.7.9 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.0 + rev: v1.25.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -53,15 +53,14 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/python/black - rev: 19.3b0 + rev: 19.10b0 hooks: - id: black - # Disabled until: https://github.com/ansible/ansible-lint/issues/590 - # - repo: https://github.com/ansible/ansible-lint.git - # rev: v4.1.1a0 - # hooks: - # - id: ansible-lint - # # files: molecule/default/playbook.yml + - repo: https://github.com/ansible/ansible-lint.git + rev: v4.1.1a3 + hooks: + - id: ansible-lint + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From 5d5567d303c8ef4c1074ba86bfc7a73dad63b4c8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 5 Nov 2019 14:27:20 -0500 Subject: [PATCH 089/417] Add GitHub action caching of pre-commit hooks and pip packages. --- .github/workflows/build.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 013abd4..fe47599 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,9 +5,7 @@ on: [push] jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v1 @@ -16,6 +14,25 @@ jobs: with: python-version: 3.7 + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" + restore-keys: | + ${{ runner.os }}-pre-commit- + + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- + - name: Install dependencies run: | python -m pip install --upgrade pip From 99c611557e432fe6615f7495b6dbdf5c78ac9ae6 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 5 Nov 2019 14:51:49 -0500 Subject: [PATCH 090/417] Remove search for previous pre-commit caches. These seem to be very large caches. Restoring an old one and updating it results in a cache larger than the maximum allowed cache size: 200MB. "Cache size of 254757924 bytes is over the 200MB limit, not saving cache." So if the config changes it is best to just take the cache-miss and start from scratch. --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe47599..d12843d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,8 +20,6 @@ jobs: path: ~/.cache/pre-commit key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - restore-keys: | - ${{ runner.os }}-pre-commit- - name: Cache pip test requirements uses: actions/cache@v1 From 398e7f5ded6b896cf970b69d2024d858260f3e34 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:09:52 -0500 Subject: [PATCH 091/417] Add new workflow: lint, test, and build artifact. --- .github/workflows/build.yml | 71 ++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d12843d..df36bee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,38 +3,85 @@ name: build on: [push] +env: + IMAGE_NAME: dhsncats/example + PIP_CACHE_DIR: ~/.cache/pip + PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + + jobs: - build: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/setup-python@v1 with: python-version: 3.7 - + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- - name: Cache pre-commit hooks uses: actions/cache@v1 with: - path: ~/.cache/pre-commit + path: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + - name: Run linters on all files + run: pre-commit run --all-files + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 - name: Cache pip test requirements uses: actions/cache@v1 with: - path: ~/.cache/pip + path: ${{ env.PIP_CACHE_DIR }} key: "${{ runner.os }}-pip-test-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - - - name: Run pre-commit on all files - run: pre-commit run --all-files + - name: Run tests + run: pytest + build: + runs-on: ubuntu-latest + needs: [lint, test] + steps: + - uses: actions/checkout@v1 + - name: Build docker image + run: | + version=$(./bump_version.sh show) + docker build \ + --tag "$IMAGE_NAME" \ + --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ + --build-arg GIT_REMOTE=$(git remote get-url origin) \ + --build-arg VERSION=${version} \ + . + - name: Save docker image + run: | + mkdir dist + version=$(./bump_version.sh show) + docker save $IMAGE_NAME:latest | \ + gzip > dist/"${IMAGE_NAME}"_"${version}".tar.gz + - name: Upload artifacts + uses: actions/upload-artifact@v1 + with: + name: dist + path: dist From 6bdab7d11cdb371d0612392d4f8ec34a7a01ef87 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:50:22 -0500 Subject: [PATCH 092/417] Add release workflow based on Travis. --- .github/workflows/release.yml | 53 +++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8fcc8e1 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,53 @@ +--- +name: release + +on: + release: + types: [prereleased, released] + +env: + IMAGE_NAME: dhsncats/example + DOCKER_PW: ${{ secrets.DOCKER_PW }} + DOCKER_USER: ${{ secrets.DOCKER_USER }} + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Determine image version + id: get_ver + run: | + echo "##[set-output name=version;]$(./bump_version.sh show)" + - name: Build Docker image + run: | + docker build \ + --tag "$IMAGE_NAME" \ + --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ + --build-arg GIT_REMOTE=$(git remote get-url origin) \ + --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ + . + - name: Tag Docker image + run: | + IFS='.' read -r -a version_array \ + <<< "${{ steps.get_ver.outputs.version }}" + docker login --username "$DOCKER_USER" --password "$DOCKER_PW" + docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" + docker tag "$IMAGE_NAME" \ + "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + docker tag "$IMAGE_NAME" \ + "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" + docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" + - name: Publish image to Docker Hub + run: | + IFS='.' read -r -a version_array \ + <<< "${{ steps.get_ver.outputs.version }}" + docker push "${IMAGE_NAME}:latest" + docker push "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" + docker push "${IMAGE_NAME}:${version_array[0]}" + - name: Publish README.md to Docker Hub + run: ./push_readme.sh From 5686bbee8c2fb1f2a52959d1ec169ff7cbc4ea16 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:50:43 -0500 Subject: [PATCH 093/417] Remove Travis configuration. --- .travis.yml | 56 ----------------------------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cedc956..0000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -dist: xenial -language: python -python: 3.7 -services: docker -env: - global: - - IMAGE_NAME=dhsncats/example - - DOCKER_USER=felddy - # DOCKER_PW - - secure: >- - GWxja6rCcevNRay3uOGEt1wpzGOAoKgfquyror60VtkG07GCJ/rCBa/jNsTRRwwDxYW - MqY6xQlAU+GzNeuBxKAlmyrLRRYZDmcBpPgQQPbiC0b5ftGCzTuhdYAj4Hg+LyHKbNP - H0zsXfQRy2NdWPJ8rr3RFL89a/If0hJVL5OEEHfmzpfvHoDvBJQwhmL2sPZizgCsgh6 - eRStRvUWj3taVb5VZ1uIlet8P3g7azkD7tTNx+wAxhjlLpaQfBcy/CC27ogw+QOvgdF - kefskYz54LLzTyXYhRhACjuGaukpBvQBFzBi3MOFYoLj4kb+9b3mqHlnQZLOSo2zlWV - 30mTC7eed5KE64MuQ6I1E72B0rnAoVpnwAjXVIvzfhKBuzbhS54tWjKbqK6UAGxbg6t - wJDiNaB+nYk6zozJ93TQT02XDUOdDS45hO2Kc/CUwJbrZc7V4aYN6Dj30tWy8d2TI8s - GMPuJm8tyxpT/h2eUUPhVzhMCk+4efIBa6cb99wZgkphLmWQefLLpZy8pMwOH7c3ywU - +6tVPBZxSEm4tFRYyPwgl9qvzE8E0WM3FPNgwVO9aypx61HcO8NVD7alUW2j+s3taHX - aVpaCtsf+bpsXPro3/ofst+FHiFPZGBL2iiSIYtFYI4hkDFq9iGGGexo9zgYS6CMexu - YoZtThDw6kaLI= -cache: - pip: true - directories: - - "$HOME/.cache/pre-commit" -install: - - pip install --upgrade --requirement requirements-test.txt - - version=$(./bump_version.sh show) - - docker build - --tag "$IMAGE_NAME" - --build-arg GIT_COMMIT=$(git log -1 --format=%H) - --build-arg GIT_REMOTE=$(git remote get-url origin) - --build-arg VERSION=${version} - . -script: - - pre-commit run --all-files - - pytest --verbose -before_deploy: - - IFS='.' read -r -a version_array <<< "$version" - - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" - - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version}" - - docker tag "$IMAGE_NAME" - "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" -deploy: - - provider: script - script: docker push "${IMAGE_NAME}:latest" && - docker push "${IMAGE_NAME}:${version}" && - docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" && - docker push "${IMAGE_NAME}:${version_array[0]}" && - ./push_readme.sh - on: - tags: true - python: '3.7' From 1a174ba02813b2301658e10d6017a36256438ac4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:56:56 -0500 Subject: [PATCH 094/417] Create a version lookup step with output. --- .github/workflows/build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df36bee..87a4050 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,6 @@ env: PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit - jobs: lint: runs-on: ubuntu-latest @@ -65,6 +64,10 @@ jobs: needs: [lint, test] steps: - uses: actions/checkout@v1 + - name: Determine image version + id: get_ver + run: | + echo "##[set-output name=version;]$(./bump_version.sh show)" - name: Build docker image run: | version=$(./bump_version.sh show) @@ -72,14 +75,14 @@ jobs: --tag "$IMAGE_NAME" \ --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${version} \ + --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ . - name: Save docker image run: | mkdir dist version=$(./bump_version.sh show) - docker save $IMAGE_NAME:latest | \ - gzip > dist/"${IMAGE_NAME}"_"${version}".tar.gz + docker save $IMAGE_NAME:latest | gzip > \ + dist/"${IMAGE_NAME}"_"${{ steps.get_ver.outputs.version }}".tar.gz - name: Upload artifacts uses: actions/upload-artifact@v1 with: From 57dfd9ff1e7743552483de99f0302c649851c522 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 11:47:01 -0500 Subject: [PATCH 095/417] Reorder jobs and pass image artifacts. --- .github/workflows/build.yml | 54 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87a4050..1100fd8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,31 +37,8 @@ jobs: pip install --upgrade -r requirements-test.txt - name: Run linters on all files run: pre-commit run --all-files - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Cache pip test requirements - uses: actions/cache@v1 - with: - path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ - ${{ hashFiles('**/requirements-test.txt') }}" - restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade -r requirements-test.txt - - name: Run tests - run: pytest build: runs-on: ubuntu-latest - needs: [lint, test] steps: - uses: actions/checkout@v1 - name: Determine image version @@ -77,7 +54,7 @@ jobs: --build-arg GIT_REMOTE=$(git remote get-url origin) \ --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ . - - name: Save docker image + - name: Save docker image artifact run: | mkdir dist version=$(./bump_version.sh show) @@ -88,3 +65,32 @@ jobs: with: name: dist path: dist + test: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + - name: Download docker image artifact + uses: actions/download-artifact@v1 + with: + name: dist + - name: Load docker image + run: docker load < dist/*.tar.gz + - name: Run tests + run: pytest From 78348f67d7780dae0b7b3359d654dd3025923f90 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 11:54:46 -0500 Subject: [PATCH 096/417] Simplify artifact naming. --- .github/workflows/build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1100fd8..7c42afb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,8 +58,7 @@ jobs: run: | mkdir dist version=$(./bump_version.sh show) - docker save $IMAGE_NAME:latest | gzip > \ - dist/"${IMAGE_NAME}"_"${{ steps.get_ver.outputs.version }}".tar.gz + docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts uses: actions/upload-artifact@v1 with: @@ -91,6 +90,6 @@ jobs: with: name: dist - name: Load docker image - run: docker load < dist/*.tar.gz + run: docker load < dist/image.tar.gz - name: Run tests run: pytest From 9251add910953eb9651268681439b274a4104769 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 12:05:21 -0500 Subject: [PATCH 097/417] Change environment variable name to be CI agnostic. --- tests/container_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/container_test.py b/tests/container_test.py index 50b62d8..524d1d3 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -12,7 +12,7 @@ SECRET_QUOTE = ( "There are no secrets better kept than the secrets everybody guesses." # nosec ) -TRAVIS_TAG = os.getenv("TRAVIS_TAG") +RELEASE_TAG = os.getenv("RELEASE_TAG") VERSION_FILE = "src/version.txt" @@ -54,7 +54,7 @@ def test_output(main_container): @pytest.mark.skipif( - TRAVIS_TAG in [None, ""], reason="this is not a release (TRAVIS_TAG not set)" + RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" ) def test_release_version(): """Verify that release tag version agrees with the module version.""" @@ -63,8 +63,8 @@ def test_release_version(): exec(f.read(), pkg_vars) # nosec project_version = pkg_vars["__version__"] assert ( - TRAVIS_TAG == f"v{project_version}" - ), "TRAVIS_TAG does not match the project version" + RELEASE_TAG == f"v{project_version}" + ), "RELEASE_TAG does not match the project version" def test_log_version(version_container): From 7ed5c76f8b249e04f7c9b8b7ec684fad395a9c12 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 12:19:44 -0500 Subject: [PATCH 098/417] Move images from dhsncats to cisagov docker hub org. --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- README.md | 10 +++++----- docker-compose.yml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c42afb..1eddebe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ name: build on: [push] env: - IMAGE_NAME: dhsncats/example + IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8fcc8e1..6fe7223 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ on: types: [prereleased, released] env: - IMAGE_NAME: dhsncats/example + IMAGE_NAME: cisagov/example DOCKER_PW: ${{ secrets.DOCKER_PW }} DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/README.md b/README.md index 5b2da6b..5b39478 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ ## Docker Image ## -![MicroBadger Layers](https://img.shields.io/microbadger/layers/dhsncats/example.svg) -![MicroBadger Size](https://img.shields.io/microbadger/image-size/dhsncats/example.svg) +![MicroBadger Layers](https://img.shields.io/microbadger/layers/cisagov/example.svg) +![MicroBadger Size](https://img.shields.io/microbadger/image-size/cisagov/example.svg) This is a docker skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub docker project @@ -20,11 +20,11 @@ appropriate for docker containers and the major languages that we use. ### Install ### -Pull `dhsncats/example` from the Docker repository: +Pull `cisagov/example` from the Docker repository: - docker pull dhsncats/example + docker pull cisagov/example -Or build `dhsncats/example` from source: +Or build `cisagov/example` from source: git clone https://github.com/cisagov/skeleton-docker.git cd skeleton-docker diff --git a/docker-compose.yml b/docker-compose.yml index f9a54ad..6774387 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: # e.g., --build-arg VERSION=0.0.1 context: . dockerfile: Dockerfile - image: dhsncats/example + image: cisagov/example init: true restart: "no" environment: @@ -31,7 +31,7 @@ services: example-version: # Run the container to collect version information - image: dhsncats/example + image: cisagov/example init: true restart: "no" command: --version From e97faf6bd11ecf8753cf757e1cf182c77b04bfdc Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 14:15:35 -0500 Subject: [PATCH 099/417] Pass in the github release tag for testing. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1eddebe..9643f76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,4 +92,6 @@ jobs: - name: Load docker image run: docker load < dist/image.tar.gz - name: Run tests + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest From 7c0b30b495e689b84317b390eff36425d7c70cdd Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 15 Nov 2019 11:50:38 -0500 Subject: [PATCH 100/417] Force pre-commit hooks to use python3. Fixes #23 --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d7bcbc..7626c79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,8 @@ --- +default_language_version: + # force all unspecified python hooks to run python3 + python: python3 + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 From 30a16a38420cca91cda0cd2543484aa4a2e125cc Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 15 Nov 2019 11:51:46 -0500 Subject: [PATCH 101/417] Autoupdate pre-commit hooks. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7626c79..27477d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a3 + rev: v4.1.1a5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -75,6 +75,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.18.2 + rev: 1.19.1 hooks: - id: prettier From e39dc8881915f36fdb70c9c58f0278700523b294 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 18 Nov 2019 10:33:28 -0500 Subject: [PATCH 102/417] Add codeowners file with team OIS maintainers. See: https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ca9c8dc --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# these owners will be requested for review when someone +# opens a pull request. +* @dav3r @felddy @jsf9k @mcdonnnj From b89eecb788084c95930554679cb38b32548c3c8d Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Mon, 18 Nov 2019 11:11:06 -0500 Subject: [PATCH 103/417] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca9c8dc..32918e8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj @cisagov/team-ois From 18b5d12c784dad59c461af36922cc2082f1dcffd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 19 Dec 2019 15:32:36 -0500 Subject: [PATCH 104/417] Add seed-isort-config and isort to pre-commit hooks. Add isort configuration file. --- .isort.cfg | 12 ++++++++++++ .pre-commit-config.yaml | 10 ++++++++++ 2 files changed, 22 insertions(+) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..5216e47 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,12 @@ +[settings] +combine_star=true +force_sort_within_sections=true + +import_heading_stdlib=Standard Python Libraries +import_heading_thirdparty=Third-Party Libraries +import_heading_firstparty=cisagov Libraries + +# Should be auto-populated by seed-isort-config hook +known_third_party= +# These must be manually set to correctly separate them from third party libraries +known_first_party= diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27477d3..c8fc88a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,6 +60,16 @@ repos: rev: 19.10b0 hooks: - id: black + - repo: https://github.com/asottile/seed-isort-config + rev: v1.9.3 + hooks: + - id: seed-isort-config + - repo: https://github.com/pre-commit/mirrors-isort + # pick the isort version you'd like to use from + # https://github.com/pre-commit/mirrors-isort/releases + rev: v4.3.21 + hooks: + - id: isort - repo: https://github.com/ansible/ansible-lint.git rev: v4.1.1a5 hooks: From 2fa4cbe194ac05ff89e377ad91adfa3fc7eddb6f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Jan 2020 16:45:48 -0500 Subject: [PATCH 105/417] Make workflow run when a PR is opened, synchronized, or reopened A user forked cisagov/scan-target-data and created a pull request, but the required GitHub Action(s) did not run. This is presumably because the user does not have Actions enabled in his or her fork. Ideally, the required Action(s) would run in cisagov/scan-target-data when a PR to merge changes back is created. Based on my reading of this link, adding the "pull_request" event type should make this happen: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/events-that-trigger-workflows#pull-request-events-for-forked-repositories --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d12843d..c9cc05b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,7 @@ --- name: build -on: [push] +on: [push, pull_request] jobs: build: From dbd589d2e27b517726fec167987058b3d687992f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Jan 2020 17:10:49 -0500 Subject: [PATCH 106/417] Improve list formatting --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c9cc05b..e6c14e6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,10 @@ --- name: build -on: [push, pull_request] +on: [ + push, + pull_request +] jobs: build: From deb186c98b5539654402c2c252bc3d55d2190aeb Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 27 Jan 2020 11:14:04 -0500 Subject: [PATCH 107/417] Add changes made by isort pre-commit hook --- .isort.cfg | 2 +- tests/conftest.py | 2 +- tests/container_test.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 5216e47..3b2f682 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,6 +7,6 @@ import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries # Should be auto-populated by seed-isort-config hook -known_third_party= +known_third_party=pytest # These must be manually set to correctly separate them from third party libraries known_first_party= diff --git a/tests/conftest.py b/tests/conftest.py index d62dff6..28d6c42 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,9 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ +# Third-Party Libraries import pytest - MAIN_SERVICE_NAME = "example" VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" diff --git a/tests/container_test.py b/tests/container_test.py index 524d1d3..90ea1a3 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,9 +1,11 @@ #!/usr/bin/env pytest -vs """Tests for example container.""" +# Standard Python Libraries import os import time +# Third-Party Libraries import pytest ENV_VAR = "ECHO_MESSAGE" From 532751627a347637814e8a7f626dad487a1b6053 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 10 Feb 2020 10:39:07 -0500 Subject: [PATCH 108/417] Backported changes to CONTRIBUTING.md from the development guide. --- CONTRIBUTING.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93addc2..eb00ca9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,10 +56,31 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux (or on the Mac, if you don't want to use `brew`) you can use +For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to -install the necessary tools. When you are finished you will need to -add the same two lines above to your profile. +install the necessary tools. Before running this ensure that you have +installed the prerequisites for your platform according to the +[`pyenv` wiki +page](https://github.com/pyenv/pyenv/wiki/common-build-problems). + +On WSL you should treat your platform as whatever Linux distribution +you've chosen to install. + +Once you have installed `pyenv` you will need to add the following +lines to your `.bashrc`: + +```bash +export PATH="$PATH:$HOME/.pyenv/bin" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +``` + +If you are using a shell other than `bash` you should follow the +instructions that the `pyenv-installer` script outputs. + +You will need to reload your shell for these changes to take effect so +you can begin to use `pyenv`. For a list of Python versions that are already installed and ready to use with `pyenv`, use the command `pyenv versions`. To see a list of From f7a4166ad67d961324bc44130e092eb1ddebd320 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 11 Feb 2020 10:41:17 -0500 Subject: [PATCH 109/417] Update Python version used to 3.8 Update actions/checkout to v2 Update formatting to match downstream children --- .github/workflows/build.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6c14e6..4953f7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,20 +10,16 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 with: - python-version: 3.7 - + python-version: 3.8 - name: Cache pre-commit hooks uses: actions/cache@v1 with: path: ~/.cache/pre-commit key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -33,11 +29,9 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - - name: Run pre-commit on all files run: pre-commit run --all-files From b857939b3d0c5393b8b5528a6bb9bce2ff0f736f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:01:16 -0500 Subject: [PATCH 110/417] Run pre-commit autoupdate. --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8fc88a..7856658 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v2.5.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.19.0 + rev: v0.22.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.18.0 + rev: v1.20.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.1 + rev: v1.26.2 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.3 + rev: v1.9.4 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -71,7 +71,7 @@ repos: hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a5 + rev: v4.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -81,7 +81,7 @@ repos: - id: terraform_fmt - id: terraform_validate_no_variables - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.0 + rev: v1.0.1 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier From d99fd00bc2e5c4a0afeb2d6717dac7fe77f64d33 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:14:11 -0500 Subject: [PATCH 111/417] Flip cache order to mirror how it is done downstream. --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4953f7c..76801a8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,12 +14,6 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.8 - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: "${{ runner.os }}-pre-commit-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -29,6 +23,12 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | python -m pip install --upgrade pip From bf7f97bd480df8dfc0e7de063173cd4ccc9e8827 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 16:36:17 -0500 Subject: [PATCH 112/417] Add missing space in .bandit.yml --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index 5ae161d..2b618f6 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -10,4 +10,4 @@ tests: # - B102 skips: - - B101 # skip "assert used" check since assertions are required in pytests + - B101 # skip "assert used" check since assertions are required in pytests From e96577bce4b3b6aefa044943e478301a7d11288f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 18 Feb 2020 18:04:06 -0500 Subject: [PATCH 113/417] All references to '-r' for pip calls have been replaced with the more verbose '--requirement'. --- .github/workflows/build.yml | 2 +- CONTRIBUTING.md | 2 +- requirements-dev.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 76801a8..aff7e7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade -r requirements-test.txt + pip install --upgrade --requirement requirements-test.txt - name: Run pre-commit on all files run: pre-commit run --all-files diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb00ca9..dacaaad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ commands: cd skeleton-generic pyenv virtualenv skeleton-generic pyenv local skeleton-generic -pip install -r requirements-dev.txt +pip install --requirement requirements-dev.txt ``` #### Installing the pre-commit hook #### diff --git a/requirements-dev.txt b/requirements-dev.txt index f122cc5..d84ee68 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ --r requirements-test.txt +--requirement requirements-test.txt ipython From 067ee0850c154845b7de623988c5a1bd5ce67d3a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Feb 2020 17:29:09 -0500 Subject: [PATCH 114/417] Autoupdate pre-commit hooks. Add mypy. --- .pre-commit-config.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7856658..46cea9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.26.2 + rev: v2.0.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -74,7 +74,7 @@ repos: rev: v4.2.0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: @@ -88,3 +88,7 @@ repos: rev: 1.19.1 hooks: - id: prettier + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.761 + hooks: + - id: mypy From bf366086f48d9b7a7e49b9f44f33b44f6a23aeb3 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 2 Mar 2020 08:11:32 -0500 Subject: [PATCH 115/417] Add .mypy_cache to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 073a081..95b74cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.mypy_cache __pycache__ .python-version From 454864bc029ce968cbeb8f9f5b88ad1c54522335 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 3 Mar 2020 16:49:53 -0500 Subject: [PATCH 116/417] Incorporate the Python version into keys for pip and pre-commit caches. This should resolve the issue seen when the Python version changes before there is an update to .pre-commit-config.yml which results in pre-commit pointing to a non-existent Python installation. --- .github/workflows/build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aff7e7a..a92cd83 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,20 +14,25 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.8 + - name: Store installed Python version + run: | + echo "::set-env name=PY_VERSION::"\ + "$(python -c "import platform;print(platform.python_version())")" - name: Cache pip test requirements uses: actions/cache@v1 with: path: ~/.cache/pip - key: "${{ runner.os }}-pip-test-\ + key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | + ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - name: Cache pre-commit hooks uses: actions/cache@v1 with: path: ~/.cache/pre-commit - key: "${{ runner.os }}-pre-commit-\ + key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | From 478f1688a108dd0f6e634c5f03c7ba6cb816e3f2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 4 Mar 2020 08:41:59 -0500 Subject: [PATCH 117/417] Change the cache paths from hardcoded values in their appropriate blocks to environment variables declared before the job block. --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a92cd83..298e93c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,10 @@ on: [ pull_request ] +env: + PIP_CACHE_DIR: ~/.cache/pip + PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + jobs: build: runs-on: ubuntu-latest @@ -21,7 +25,7 @@ jobs: - name: Cache pip test requirements uses: actions/cache@v1 with: - path: ~/.cache/pip + path: ${{ env.PIP_CACHE_DIR }} key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | @@ -31,7 +35,7 @@ jobs: - name: Cache pre-commit hooks uses: actions/cache@v1 with: - path: ~/.cache/pre-commit + path: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies From d7913343523e9841e1d295a5203e2043f055064b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 4 Mar 2020 10:56:39 -0500 Subject: [PATCH 118/417] Add a rule for markdownlint to allow multiple headers with the same name as long as they are not nested in the same heading group. --- .mdl_config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.mdl_config.json b/.mdl_config.json index 492955a..7a6f3f8 100644 --- a/.mdl_config.json +++ b/.mdl_config.json @@ -3,5 +3,8 @@ "code_blocks": false, "tables": false }, + "MD024": { + "allow_different_nesting": true + }, "default": true } From 61790a9bbd22cede3cd41d94115d0e8ad8ba51d1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Mar 2020 14:41:10 -0400 Subject: [PATCH 119/417] Fixed broken hook id for pre-commit-terraform: terraform_validate_no_variables was changed to terraform_validate in the following commit: https://github.com/antonbabenko/pre-commit-terraform/commit/35e0356188b64a4c5af9a4e7200d936e514cba71. Ran pre-commit autoupdate. --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46cea9e..42b824f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.20.0 + rev: v1.21.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.0.0 + rev: v2.1.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.4 + rev: v2.1.0 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -76,19 +76,19 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.12.0 + rev: v1.27.0 hooks: - id: terraform_fmt - - id: terraform_validate_no_variables + - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.19.1 + rev: 2.0.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.761 + rev: v0.770 hooks: - id: mypy From 0bc1aab45e2338e658a63a9b92e9f9d14eafdec8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 12:05:57 -0400 Subject: [PATCH 120/417] Ran `pre-commit autoupdate` to get the latest version of the `pre-commit-terraform` hook since the PR I submitted, https://github.com/antonbabenko/pre-commit-terraform/pull/100, was approved. This will fix issues with `skeleton-tf-module` related to multiple directories with Terraform code. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42b824f..129aa39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.27.0 + rev: v1.29.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -85,7 +85,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.2 + rev: 2.0.4 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From b01a0ee7bd1e0854b4c4c6bb999316c21221c97f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 13:47:18 -0400 Subject: [PATCH 121/417] Explicitly install pre-commit hooks as its own step so it's clearer when the failure is with setup rather than hooks running. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 298e93c..6026d47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,5 +42,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt + - name: Install pre-commit hooks + run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files From 1f3d440a1af6283621b5584da9b56e2747d40a0b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 19:02:31 -0400 Subject: [PATCH 122/417] Rename action to accurately reflect that we are only linting, not building. Adjust description for pre-commit hook setup to better convey what is being done. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6026d47..9c6b03a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ env: PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit jobs: - build: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -42,7 +42,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - - name: Install pre-commit hooks + - name: Set up pre-commit hook environments run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files From 647b351997ab7cafd391bdabda7b1b5e68214b8a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 19:04:21 -0400 Subject: [PATCH 123/417] Sort .gitignore entries. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 95b74cd..e00826d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .mypy_cache -__pycache__ .python-version +__pycache__ From c8b4bcf126989abf35e4ce226263ef7da139c959 Mon Sep 17 00:00:00 2001 From: Hillary Date: Tue, 14 Apr 2020 12:43:18 -0400 Subject: [PATCH 124/417] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32918e8..a3619be 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj @cisagov/team-ois +* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj @cisagov/team-ois From 8c99c5294a798bca98a68212507fce924505a289 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 Apr 2020 23:31:59 -0400 Subject: [PATCH 125/417] Allow events from apb to rebuild this repository weekly. --- .github/workflows/build.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c6b03a..5fe2f8b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,11 @@ --- name: build -on: [ - push, - pull_request -] +on: + push: + pull_request: + repository_dispatch: + types: [apb] env: PIP_CACHE_DIR: ~/.cache/pip From 7d7a6774571c4727b4858b1117c31f97160131da Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 May 2020 17:49:17 -0400 Subject: [PATCH 126/417] Updated pre-commit configuration with 'pre-commit autoupdate'. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 129aa39..de0152b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.22.0 + rev: v0.23.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.21.0 + rev: v1.23.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -41,13 +41,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.9 + rev: 3.8.0a2 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.1.0 + rev: v2.4.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.0 + rev: v2.1.1 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -71,12 +71,12 @@ repos: hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.2.0 + rev: v4.3.0a0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.29.0 + rev: v1.30.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -85,7 +85,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.4 + rev: 2.0.5 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From cda200cfd1be3dfa44fbb2f271e571b43d560691 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 11 May 2020 21:52:31 -0400 Subject: [PATCH 127/417] Add Lineage configuration. --- .github/lineage.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/lineage.yml diff --git a/.github/lineage.yml b/.github/lineage.yml new file mode 100644 index 0000000..8dfc20b --- /dev/null +++ b/.github/lineage.yml @@ -0,0 +1,6 @@ +--- +version: "1" + +lineage: + skeleton: + remote-url: https://github.com/cisagov/skeleton-generic.git From ce6658abc69d1444e51f45c20441c98e6b54c086 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:25:31 -0400 Subject: [PATCH 128/417] Update isort pre-commit hook source repository. The mirrors-isort respository has been deprecated per this Pull Request: https://github.com/pre-commit/mirrors-isort/pull/13 Since isort includes pre-commit configuration in current versions we will switch to using it directly. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de0152b..5be35e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,10 +64,10 @@ repos: rev: v2.1.1 hooks: - id: seed-isort-config - - repo: https://github.com/pre-commit/mirrors-isort + - repo: https://github.com/timothycrosley/isort # pick the isort version you'd like to use from # https://github.com/pre-commit/mirrors-isort/releases - rev: v4.3.21 + rev: 4.3.21 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git From daaebc1cc13782e8fb39be6815aa53bf3efd44f3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:30:04 -0400 Subject: [PATCH 129/417] Ran pre-commit autoupdate. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5be35e4..a2e4f76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.0a2 + rev: 3.8.1 hooks: - id: flake8 additional_dependencies: From 2399f24622b1d6be92c494653d9308e9694c5496 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:35:36 -0400 Subject: [PATCH 130/417] Remove legacy comment from the isort hook declaration. This comment references the mirrors-isort repository that we are no longer using. --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2e4f76..bf8ae6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,8 +65,6 @@ repos: hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - # pick the isort version you'd like to use from - # https://github.com/pre-commit/mirrors-isort/releases rev: 4.3.21 hooks: - id: isort From fe4d5eb72a8f6ce9e23daae98d7a0c87675a9da4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 16:07:18 -0400 Subject: [PATCH 131/417] Disable the terraform_validate hook. We have seen a number of issues related to this hook ever since it was re-enabled. It will need to remain disabled until at least the 0.13 Terraform release, and can only be re-enabled if all issues we have seen have been resolved in how `terraform validate` operates. --- .pre-commit-config.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf8ae6e..74af27c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,19 @@ repos: rev: v1.30.0 hooks: - id: terraform_fmt - - id: terraform_validate + # There are ongoing issues with how this command works. This issue + # documents the core issue: + # https://github.com/hashicorp/terraform/issues/21408 + # We have seen issues primarily with proxy providers and Terraform code + # that uses remote state. The PR + # https://github.com/hashicorp/terraform/pull/24887 + # has been approved and is part of the 0.13 release to resolve the issue + # with remote states. + # The PR + # https://github.com/hashicorp/terraform/pull/24896 + # is a proprosed fix to deal with `terraform validate` with proxy + # providers (among other configurations). + # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 hooks: From f831ef827cae16fd288f2e7a16fedc53c6e06873 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 21:34:58 -0400 Subject: [PATCH 132/417] Add final statement to descriptionf or why terraform_validate is disabled. Review noticed that there lacked a determination for what we were doing about the problem with the terraform_validate hook. I described the problems but failed to mention what our path forward would be. This commit rectifies that oversight. --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74af27c..b7c5518 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -89,6 +89,9 @@ repos: # https://github.com/hashicorp/terraform/pull/24896 # is a proprosed fix to deal with `terraform validate` with proxy # providers (among other configurations). + # We have decided to disable the terraform_validate hook until the issues + # above have been resolved, which we hope will be with the release of + # Terraform 0.13. # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 From 87ea53022c39f3adead2688fbae5698147b279d8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:10:11 -0400 Subject: [PATCH 133/417] Add setuptools and wheel as pip dependencies setuptools usually comes along with pip, but wheel does not. Using wheel where possible to build python extensions is more modern and more security conscious than using setup.py. --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0a8547b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +setuptools +wheel From 7f8d430a309b22a3410949e64ccd60ad5a1dc79f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:19:05 -0400 Subject: [PATCH 134/417] Pull in requirements.txt from requirements-test.txt --- requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-test.txt b/requirements-test.txt index 416634f..66f74db 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ +--requirement requirements.txt pre-commit From af06915b05674695cb5d9bd8418c6b37f77ec3bc Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 14:57:04 -0400 Subject: [PATCH 135/417] Add tool and docs to automate development environment setup. --- CONTRIBUTING.md | 11 +++ setup-env | 175 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100755 setup-env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dacaaad..2bd2451 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,17 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. +If you already have `pyenv` and `pyenv-virtualenv` configured you can +take advantage of the `setup-env` tool in this repo to automate the +entire environment configuration process. + +```bash +./setup-env +``` + +Otherwise, follow the steps below to manually configure your +environment. + #### Installing and using `pyenv` and `pyenv-virtualenv` #### On the Mac, installation is as simple as `brew install pyenv diff --git a/setup-env b/setup-env new file mode 100755 index 0000000..65c5b6f --- /dev/null +++ b/setup-env @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +USAGE=$(cat << 'END_OF_LINE' +This script is used to configure a developement environment for this repo. + +It does the following: + - Verifies pyenv and pyenv-virtualenv are installed. + - Creates a Python virtual environment. + - Configures the activation of the virtual enviroment for the repo directory. + - Installs the requirements required for development. + - Installs git pre-commit hooks. + - Configures git upstream remote "lineage" repositories. + +usage: setup-env [--force] [--help] [virt_env_name] + +END_OF_LINE +) + +# Flag to force deletion and creation of virtual environment +FORCE=0 + +# Positional parameters +PARAMS="" + +# Parse command line arguments +while (( "$#" )); do + case "$1" in + -f|--force) + FORCE=1 + shift + ;; + -h|--help) + echo "${USAGE}" + exit 0 + ;; + -*) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# set positional arguments in their proper place +eval set -- "$PARAMS" + +# Check to see if pyenv is installed +if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, installation is as simple as "brew install pyenv + pyenv-virtualenv" and adding this to your profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + +set +o nounset +# Determine the virtual environment name +if [ "$1" ]; then + # Use the user-provided environment name + env_name=$1 +else + # Set the environment name to the last part of the working directory. + env_name=${PWD##*/} +fi +set -o nounset + +# Remove any lingering local configuration. +if [ $FORCE -ne 0 ]; then + rm -f .python-version + pyenv virtualenv-delete --force "${env_name}" || true +elif [[ -f .python-version ]]; then + cat << 'END_OF_LINE' + An existing .python-version file was found. Either remove this file yourself + or re-run with --force option to have it deleted along with the associated + virtual environment. + + rm .python-version + +END_OF_LINE + exit 1 +fi + +# Create a new virutal environment for this project +if ! pyenv virtualenv "${env_name}"; then + cat << END_OF_LINE + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. + + pyenv virtualenv-delete ${env_name} + +END_OF_LINE + exit 1 +fi + +# Activate the new virtual environment +pyenv local "${env_name}" + +# Upgrade pip and friends +python -m pip install --upgrade pip setuptools wheel + +# Find a requirements file (if possible) and install +for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do + if [[ -f $req_file ]] + then + pip install -r $req_file + break + fi +done + +# Install git pre-commit hooks +pre-commit install + +# Setup git remotes from lineage configuration +# This could fail if the remotes are already setup, but that is ok. +set +o errexit + +eval "$(python3 << 'END_OF_LINE' +from pathlib import Path +import yaml +import sys + +LINEAGE_CONFIG = Path(".github/lineage.yml") + +if not LINEAGE_CONFIG.exists(): + print('No lineage configuration found.', file=sys.stderr) + sys.exit(0) + +with LINEAGE_CONFIG.open("r") as f: + lineage = yaml.safe_load(stream=f) + +if lineage["version"] == "1": + for parent_name, v in lineage["lineage"].items(): + remote_url = v["remote-url"] + print(f"git remote add {parent_name} {remote_url};") + print(f"git remote set-url --push {parent_name} no_push;") +else: + print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) +END_OF_LINE +)" + +# Qapla +echo "Success!" From c50094a8dee2573462c355e82e01eba13fb1fb14 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:15:18 -0400 Subject: [PATCH 136/417] Fix typo. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 65c5b6f..5015beb 100755 --- a/setup-env +++ b/setup-env @@ -113,7 +113,7 @@ END_OF_LINE exit 1 fi -# Create a new virutal environment for this project +# Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this From 3c2781c8cc3d87d4e1433100e243679e899d7fe1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:25:59 -0400 Subject: [PATCH 137/417] Correct syntax on fence. Closes CVE-2020-3.14159 --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bd2451..8635fea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ If you already have `pyenv` and `pyenv-virtualenv` configured you can take advantage of the `setup-env` tool in this repo to automate the entire environment configuration process. -```bash +```console ./setup-env ``` From 23ef29c8a301a6851664ae0ccd8633363f95648f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:30:46 -0400 Subject: [PATCH 138/417] Clarify comment with text from `pyenv local --help` --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5015beb..5f404b3 100755 --- a/setup-env +++ b/setup-env @@ -125,7 +125,8 @@ END_OF_LINE exit 1 fi -# Activate the new virtual environment +# Set the local application-specific Python version(s) by writing the +# version name to a file named `.python-version'. pyenv local "${env_name}" # Upgrade pip and friends From 5d1d134c88aa203d69d4baf88e9b1e5d4b7a2eab Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:34:53 -0400 Subject: [PATCH 139/417] Change to python3 to standardize how we call python. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5f404b3..a767b3d 100755 --- a/setup-env +++ b/setup-env @@ -130,7 +130,7 @@ fi pyenv local "${env_name}" # Upgrade pip and friends -python -m pip install --upgrade pip setuptools wheel +python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do From 8a105ca04010ca92cd066a525b840fe1c6de66f5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:50:42 -0400 Subject: [PATCH 140/417] Respect customs. --- setup-env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index a767b3d..713687a 100755 --- a/setup-env +++ b/setup-env @@ -134,9 +134,8 @@ python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do - if [[ -f $req_file ]] - then - pip install -r $req_file + if [[ -f $req_file ]]; then + pip install --requirement $req_file break fi done From ae21af108405bde6ea772d804f71ecf3cd677895 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:01 -0400 Subject: [PATCH 141/417] Improve usage message. --- setup-env | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 713687a..7bc99f9 100755 --- a/setup-env +++ b/setup-env @@ -5,17 +5,25 @@ set -o errexit set -o pipefail USAGE=$(cat << 'END_OF_LINE' -This script is used to configure a developement environment for this repo. +Configure a developement environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - - Installs the requirements required for development. + - Installs the requirements needed for development. - Installs git pre-commit hooks. - Configures git upstream remote "lineage" repositories. -usage: setup-env [--force] [--help] [virt_env_name] +Usage: + setup-env [--force] [virt_env_name] + setup-env (-h | --help) + +Options: + -f --force Delete virtual enviroment if it already exists. + -h --help Show this message. + -i --install-hooks Install hook environments for all environments in the + pre-commit config file. END_OF_LINE ) From c5e7edf46b736965216b258ac562cbf6ad6e1218 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:34 -0400 Subject: [PATCH 142/417] Add flag to install pre-commit hooks now. --- setup-env | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 7bc99f9..5b6dacd 100755 --- a/setup-env +++ b/setup-env @@ -37,6 +37,10 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -f|--force) FORCE=1 shift @@ -148,8 +152,8 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt fi done -# Install git pre-commit hooks -pre-commit install +# Install git pre-commit hooks now or later. +pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # Setup git remotes from lineage configuration # This could fail if the remotes are already setup, but that is ok. From 482a370679570e55233c13ef6df644ee3437694e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:02:03 -0400 Subject: [PATCH 143/417] Switch to a POSIX compliant command... command. This is the way. I learned something today. https://unix.stackexchange.com/questions/85249/why-not-use-which-what-to-use-then --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5b6dacd..9d1ed6a 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 466ef0cce560fc18b4d657caef943f9c07626fc8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:05:19 -0400 Subject: [PATCH 144/417] Sort command line flag case statement. --- setup-env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup-env b/setup-env index 9d1ed6a..0b44a24 100755 --- a/setup-env +++ b/setup-env @@ -37,10 +37,6 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in - -i|--install-hooks) - INSTALL_HOOKS=1 - shift - ;; -f|--force) FORCE=1 shift @@ -49,6 +45,10 @@ while (( "$#" )); do echo "${USAGE}" exit 0 ;; + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 996ae2677a33e1631c8ac80a6e9c682be22e6ac9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:13:51 -0400 Subject: [PATCH 145/417] Add recommendation to install brew, instead of assuming it is available. --- CONTRIBUTING.md | 5 +++-- setup-env | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8635fea..6434d30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,8 +59,9 @@ environment. #### Installing and using `pyenv` and `pyenv-virtualenv` #### -On the Mac, installation is as simple as `brew install pyenv -pyenv-virtualenv` and adding this to your profile: +On the Mac, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install pyenv pyenv-virtualenv` and +adding this to your profile: ```bash eval "$(pyenv init -)" diff --git a/setup-env b/setup-env index 0b44a24..46620e0 100755 --- a/setup-env +++ b/setup-env @@ -69,8 +69,9 @@ if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, installation is as simple as "brew install pyenv - pyenv-virtualenv" and adding this to your profile: + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 31980ed4a8624baa682735d0ecfef611582e55a1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:18:19 -0400 Subject: [PATCH 146/417] Clean up usage syntax since we now have multiple options. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 46620e0..57abe56 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [--force] [virt_env_name] + setup-env [options] [virt_env_name] setup-env (-h | --help) Options: From db362b912512418280e8b74891de812016e4fb9f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:37:34 -0400 Subject: [PATCH 147/417] Normalize quotes. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 57abe56..4d822c4 100755 --- a/setup-env +++ b/setup-env @@ -168,7 +168,7 @@ import sys LINEAGE_CONFIG = Path(".github/lineage.yml") if not LINEAGE_CONFIG.exists(): - print('No lineage configuration found.', file=sys.stderr) + print("No lineage configuration found.", file=sys.stderr) sys.exit(0) with LINEAGE_CONFIG.open("r") as f: From 91279d3ac2a17383acbea2f82a193b3fefdaa895 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jul 2020 21:28:58 -0400 Subject: [PATCH 148/417] Perform a pre-commit autoupdate for 2020-07 Notable version changes: - docker-pre-commit v2.0.0 This release includes the addition of two new hooks: hadolint and hadolint-system. These allow for linting Dockerfiles with the hadolint Docker image or a system installation of hadolint respectively. - isort 5.0.7 This release takes us from isort 4 to isort 5. There are a multitude of changes with information available on the isort 5 release page at https://timothycrosley.github.io/isort/docs/major_releases/introducing_isort_5/ The most notable feature for us is the addition of built-in profiles for isort to run under. These profiles provide settings for conforming to various popular styles, and black is one of the profiles. --- .pre-commit-config.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7c5518..52b6473 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.1.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -27,7 +27,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.23.0 + rev: v0.23.2 hooks: - id: markdownlint args: @@ -41,13 +41,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.1 + rev: 3.8.3 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.4.1 + rev: v2.7.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,20 +61,20 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.1 + rev: v2.2.0 hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.0.7 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.0a0 + rev: v4.3.0a3 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.30.0 + rev: v1.31.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -94,7 +94,7 @@ repos: # Terraform 0.13. # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.1 + rev: v2.0.0 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier @@ -102,6 +102,6 @@ repos: hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.770 + rev: v0.782 hooks: - id: mypy From 907b89a20b17b5f8ffadbdc7ee71ed6cc31fd63b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jul 2020 21:34:22 -0400 Subject: [PATCH 149/417] Add the black profile to the isort configuration --- .isort.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.isort.cfg b/.isort.cfg index 5216e47..2fd6b08 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -10,3 +10,6 @@ import_heading_firstparty=cisagov Libraries known_third_party= # These must be manually set to correctly separate them from third party libraries known_first_party= + +# Run isort under the black profile to align with our other Python linting +profile=black From d6fe79dce37d8ecab907d7ba49fd580d4191819b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 02:19:02 -0400 Subject: [PATCH 150/417] Update actions/cache to v2 Switch the workflow to use cachev2. This puts us up to date and allows us to consolidate into one cache item. --- .github/workflows/build.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5fe2f8b..f6f4474 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,22 +23,18 @@ jobs: run: | echo "::set-env name=PY_VERSION::"\ "$(python -c "import platform;print(platform.python_version())")" - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache linting environments + uses: actions/cache@v2 with: - path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ - ${{ hashFiles('**/requirements-test.txt') }}" - restore-keys: | - ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ + path: | + ${{ env.PIP_CACHE_DIR }} + ${{ env.PRE_COMMIT_CACHE_DIR }} + key: "${{ runner.os }}-lint-py${{ env.PY_VERSION }}-\ + ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" + restore-keys: | + ${{ runner.os }}-lint-py${{ env.PY_VERSION }}- + ${{ runner.os }}-lint- - name: Install dependencies run: | python -m pip install --upgrade pip From 6ab69f4f5158c3b2681f1b78a8fd71a57f0eb4d0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 02:21:04 -0400 Subject: [PATCH 151/417] Bump actions/setup-python to v2 Keep it up-to-date even though we are not currently using features added in v2. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6f4474..2f58ed6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - name: Store installed Python version From 969efdd0bbc8cfad942ab0b731e24dcd43b8864f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 11:08:22 -0400 Subject: [PATCH 152/417] Adjust cache key name Make the job name the first part of the cache key since it is the only part that isn't variable. --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f58ed6..36a908a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,12 +29,12 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "${{ runner.os }}-lint-py${{ env.PY_VERSION }}-\ + key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - ${{ runner.os }}-lint-py${{ env.PY_VERSION }}- - ${{ runner.os }}-lint- + lint-${{ runner.os }}-py${{ env.PY_VERSION }}- + lint-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip From 6edaffdded081d9149e22b957cfada94fd4831be Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 14:39:19 -0400 Subject: [PATCH 153/417] Update test job in build workflow to mirror updates to lint job --- .github/workflows/build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57bafc0..4fef5ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,18 +76,17 @@ jobs: needs: [build] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache testing environments + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ + key: "test-${{ runner.os }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- + test-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip From 46593bf715e34730062b6343ee12efba63e35115 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 14:40:48 -0400 Subject: [PATCH 154/417] Update release job in release workflow to use setup-python v2 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f851822..563ab08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - name: Determine image version From 332e5a7669c39eff0feef36d054acb5766069fd7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:27:01 -0400 Subject: [PATCH 155/417] Run pre-commit autoupdate --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 52b6473..2689bb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.2.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -33,7 +33,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.23.0 + rev: v1.24.2 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.7.0 + rev: v2.7.2 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -57,7 +57,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/python/black - rev: 19.10b0 + rev: 20.8b1 hooks: - id: black - repo: https://github.com/asottile/seed-isort-config @@ -65,16 +65,16 @@ repos: hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - rev: 5.0.7 + rev: 5.5.0 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.0a3 + rev: v4.3.4 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.31.0 + rev: v1.37.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -98,7 +98,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.5 + rev: 2.1.1 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From 3e9441c837ce7d34a4043a8348f1bbcccaf2608b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:29:38 -0400 Subject: [PATCH 156/417] Update CODEOWNERS to resolve #50 --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a3619be..7735a52 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj @cisagov/team-ois +* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From ae4aa123c81c0915bb1ce3da2ee19b41a98c9846 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:31:23 -0400 Subject: [PATCH 157/417] Update lint job to resolve #49 --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36a908a..7a92841 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,7 @@ jobs: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | lint-${{ runner.os }}-py${{ env.PY_VERSION }}- From d8f14e845b4054e8a45fc1f9a38c8537bcf30bfb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:32:30 -0400 Subject: [PATCH 158/417] Sort .gitignore while I'm here --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e00826d..4b15d77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ +__pycache__ .mypy_cache .python-version -__pycache__ From 6a4164dfbac372fde6babd98ece2ab61c9b5e380 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:41:47 -0400 Subject: [PATCH 159/417] Bump build workflow python-version to 3.9 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a92841..3493928 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ From c9c00f1d03d2185dfa4a7cab0ec45bd76387c95a Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:42:59 -0400 Subject: [PATCH 160/417] Replace deprecated set-env workflow syntax See: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3493928..d0563b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,8 +21,9 @@ jobs: python-version: 3.9 - name: Store installed Python version run: | - echo "::set-env name=PY_VERSION::"\ - "$(python -c "import platform;print(platform.python_version())")" + echo "PY_VERSION="\ + "$(python -c "import platform;print(platform.python_version())")" \ + >> $GITHUB_ENV - name: Cache linting environments uses: actions/cache@v2 with: From 7389a1b0cfa7e6d1b50fccef2381d3c28d31e49c Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:43:54 -0400 Subject: [PATCH 161/417] Enable weekly dependabot scans of github-actions and pip --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1fd2ed3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +--- + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" From 04c046a6fae54f0584f82133ac31fa7f01a6d857 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 14:37:56 -0400 Subject: [PATCH 162/417] Add Docker ecosystem to dependabot configuration --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1fd2ed3..fa93c02 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,6 +2,11 @@ version: 2 updates: + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" directory: "/" schedule: From e64d87783d6fac5fd6eed881dfcbbf47556a82a0 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 13:26:11 -0400 Subject: [PATCH 163/417] Update build.yml Bump workflow test Python version to 3.9. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 071e31e..786781a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Cache testing environments uses: actions/cache@v2 with: From 3de4b852b8635bf8948d4736156149e0ac701677 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2020 17:31:12 +0000 Subject: [PATCH 164/417] Bump actions/download-artifact from v1 to v2.0.5 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from v1 to v2.0.5. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v1...c3f5d00c8784369c43779f3d2611769594a61f7a) Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 786781a..c796b4c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,7 +94,7 @@ jobs: python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2.0.5 with: name: dist - name: Load docker image From d28c382f48f7d48bd963c68626679a72cdbf7109 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2020 17:31:12 +0000 Subject: [PATCH 165/417] Bump actions/upload-artifact from v1 to v2.2.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from v1 to v2.2.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v1...27bce4eee761b5bc643f46a8dfb41b430c8d05f6) Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 786781a..a7d7387 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,7 +69,7 @@ jobs: version=$(./bump_version.sh show) docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2.2.0 with: name: dist path: dist From df407a7fc08bcbf41963ae41cf94c6a2dcea4a00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2020 17:31:12 +0000 Subject: [PATCH 166/417] Bump python from 3.7-alpine to 3.9.0-alpine Bumps python from 3.7-alpine to 3.9.0-alpine. Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ba75023..67d0295 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG GIT_COMMIT=unspecified ARG GIT_REMOTE=unspecified ARG VERSION=unspecified -FROM python:3.7-alpine +FROM python:3.9.0-alpine ARG GIT_COMMIT ARG GIT_REMOTE From 1dcbc83d8c23e63d22fd25957374afb1b46b7497 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 13:42:01 -0400 Subject: [PATCH 167/417] Fix download-artifact incompatibility See: https://github.com/actions/download-artifact#compatibility-between-v1-and-v2 --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c796b4c..3a7d8e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,6 +97,7 @@ jobs: uses: actions/download-artifact@v2.0.5 with: name: dist + path: dist - name: Load docker image run: docker load < dist/image.tar.gz - name: Run tests From 53501f578ed38c7387746763c9efc648364b9a87 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 13:57:54 -0400 Subject: [PATCH 168/417] Relax upload-artifact version pin. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a7d7387..60bccc5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,7 +69,7 @@ jobs: version=$(./bump_version.sh show) docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts - uses: actions/upload-artifact@v2.2.0 + uses: actions/upload-artifact@v2 with: name: dist path: dist From 5307661b4c8bb163f32852b9420c6a333749e6d0 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 14:03:56 -0400 Subject: [PATCH 169/417] Relax base image version pin. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 67d0295..b4f4e61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG GIT_COMMIT=unspecified ARG GIT_REMOTE=unspecified ARG VERSION=unspecified -FROM python:3.9.0-alpine +FROM python:3.9-alpine ARG GIT_COMMIT ARG GIT_REMOTE From 98f237f180b95060c2efe1cff1ab2e64521c048d Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 8 Oct 2020 14:33:04 -0400 Subject: [PATCH 170/417] Add missing cache keys for test job. Fixes #30 --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b52302a..f8267a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,7 +86,9 @@ jobs: with: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-\ - ${{ hashFiles('**/requirements-test.txt') }}" + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | test-${{ runner.os }}- - name: Install dependencies From e14ecde029ecfbc39759055f7c03d7bc4ac7f458 Mon Sep 17 00:00:00 2001 From: Hillary Date: Fri, 9 Oct 2020 10:07:04 -0400 Subject: [PATCH 171/417] Add link to how to set up a new repo with skeletons --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 606df0a..66e6444 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ appropriate for the major languages that we use. In many cases you will instead want to use one of the more specific skeleton projects derived from this one. +## New Repositories from a Skeleton ## + +Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) for step-by-step instructions on how to start a new repository from a skeleton. This will save you time and effort when configuring a new repository! + ## Contributing ## We welcome contributions! Please see [here](CONTRIBUTING.md) for From 57d927d7d17f9b41aa49e5a57173005eb69719c9 Mon Sep 17 00:00:00 2001 From: Hillary Date: Fri, 9 Oct 2020 10:10:12 -0400 Subject: [PATCH 172/417] Add line breaks for linting Pre-commit is now installed properly. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 66e6444..83da801 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,10 @@ skeleton projects derived from this one. ## New Repositories from a Skeleton ## -Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) for step-by-step instructions on how to start a new repository from a skeleton. This will save you time and effort when configuring a new repository! +Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) +for step-by-step instructions on how to start a new repository from +a skeleton. This will save you time and effort when configuring a +new repository! ## Contributing ## From e0beef276313fde355efc51fa2a56571763ff598 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Fri, 9 Oct 2020 14:22:05 -0400 Subject: [PATCH 173/417] Update .github/workflows/build.yml Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8267a2..4ff7cec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,8 +87,7 @@ jobs: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" + ${{ hashFiles('**/requirements.txt') }}" restore-keys: | test-${{ runner.os }}- - name: Install dependencies From 0b979a796b02266599391c9a2ffe121af93b0754 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Oct 2020 17:01:17 -0400 Subject: [PATCH 174/417] Use python 3.9 instead of 3.8 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 563ab08..01970b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Determine image version id: get_ver run: | From 7d59f7e91d011bcbd5ea043999e6f7488078a7b9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 12 Oct 2020 14:56:53 -0400 Subject: [PATCH 175/417] Fix several places where the old-style set-env was being used These changes all have to do with the version of the Docker image. --- .github/workflows/build.yml | 10 ++++------ .github/workflows/release.yml | 15 +++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ff7cec..bf84602 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,22 +51,20 @@ jobs: steps: - uses: actions/checkout@v2 - name: Determine image version - id: get_ver - run: | - echo "##[set-output name=version;]$(./bump_version.sh show)" + run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - name: Build docker image run: | - version=$(./bump_version.sh show) + version=${{ env.IMAGE_VERSION }} docker build \ --tag "$IMAGE_NAME" \ --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ + --build-arg VERSION=${{ env.IMAGE_VERSION }} \ . - name: Save docker image artifact run: | mkdir dist - version=$(./bump_version.sh show) + version=${{ env.IMAGE_VERSION }} docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts uses: actions/upload-artifact@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01970b7..dca4c36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,34 +19,33 @@ jobs: with: python-version: 3.9 - name: Determine image version - id: get_ver - run: | - echo "##[set-output name=version;]$(./bump_version.sh show)" + run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - name: Build Docker image run: | + version=${{ env.IMAGE_VERSION }} docker build \ --tag "$IMAGE_NAME" \ --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ + --build-arg VERSION=${{ env.IMAGE_VERSION }} \ . - name: Tag Docker image run: | IFS='.' read -r -a version_array \ - <<< "${{ steps.get_ver.outputs.version }}" + <<< "${{ env.IMAGE_VERSION }}" docker login --username "$DOCKER_USER" --password "$DOCKER_PW" docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" docker tag "$IMAGE_NAME" \ "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" - name: Publish image to Docker Hub run: | IFS='.' read -r -a version_array \ - <<< "${{ steps.get_ver.outputs.version }}" + <<< "${{ env.IMAGE_VERSION }}" docker push "${IMAGE_NAME}:latest" - docker push "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + docker push "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" docker push "${IMAGE_NAME}:${version_array[0]}" - name: Publish README.md to Docker Hub From b5ef97dd4bfb008e08c5abbc0e18848798b1c803 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:49:41 -0400 Subject: [PATCH 176/417] Remove seed-isort-config pre-commit hook As of isort >=5, which we use, the seed-isort-config hook has been deprecated, and the repository archived. I have removed the hook and the configuration options it updated as a result. --- .isort.cfg | 5 ----- .pre-commit-config.yaml | 4 ---- 2 files changed, 9 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 2fd6b08..46d45f3 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -6,10 +6,5 @@ import_heading_stdlib=Standard Python Libraries import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries -# Should be auto-populated by seed-isort-config hook -known_third_party= -# These must be manually set to correctly separate them from third party libraries -known_first_party= - # Run isort under the black profile to align with our other Python linting profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2689bb0..84fbb49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,10 +60,6 @@ repos: rev: 20.8b1 hooks: - id: black - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - repo: https://github.com/timothycrosley/isort rev: 5.5.0 hooks: From de409db3f7567c342f6942ffef9a98dded20ede0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:52:09 -0400 Subject: [PATCH 177/417] Use local filename in README link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 606df0a..b445f38 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ skeleton projects derived from this one. ## Contributing ## -We welcome contributions! Please see [here](CONTRIBUTING.md) for +We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for details. ## License ## From 59eef4cc771b2340e131b216e4417b822e8d0b3f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:53:09 -0400 Subject: [PATCH 178/417] Run pre-commit autoupdate --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84fbb49..6ab5b5b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.23.2 + rev: v0.24.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.24.2 + rev: v1.25.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -41,7 +41,7 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 additional_dependencies: @@ -61,16 +61,16 @@ repos: hooks: - id: black - repo: https://github.com/timothycrosley/isort - rev: 5.5.0 + rev: 5.6.4 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.4 + rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.37.0 + rev: v1.43.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -94,10 +94,10 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.1.1 + rev: 2.1.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.790 hooks: - id: mypy From 919460cb213ac2e3c507f1c2fb4c8e7cff54783b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 22 Oct 2020 13:42:10 -0400 Subject: [PATCH 179/417] Change repo for the prettier pre-commit hook Per https://github.com/prettier/prettier/issues/9459 and specifically https://github.com/prettier/prettier/issues/9459#issuecomment-713223710 the prettier hook for pre-commit is being broken out into its own repo. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ab5b5b..7362f89 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,8 +93,8 @@ repos: rev: v2.0.0 hooks: - id: docker-compose-check - - repo: https://github.com/prettier/prettier - rev: 2.1.2 + - repo: https://github.com/prettier/pre-commit + rev: v2.1.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From 4f14cc6213bfa811f85283124bced8ddb87b4feb Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:34:57 -0400 Subject: [PATCH 180/417] Add script to generate cross-platform Dockerfile. --- buildx-dockerfile.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 buildx-dockerfile.sh diff --git a/buildx-dockerfile.sh b/buildx-dockerfile.sh new file mode 100755 index 0000000..46710e9 --- /dev/null +++ b/buildx-dockerfile.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Create a Dockerfile suitable for a multi-platform build using buildx +# See: https://docs.docker.com/buildx/working-with-buildx/ + +set -o nounset +set -o errexit +set -o pipefail + +DOCKERFILE=Dockerfile +DOCKERFILEX=Dockerfile-x + +# We don't want this expression to expand. +# shellcheck disable=SC2016 +sed 's/^FROM /FROM --platform=$TARGETPLATFORM /g' < $DOCKERFILE > $DOCKERFILEX From ba039e872f453b14731863f6f6fb411ffcd5f33f Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:40:21 -0400 Subject: [PATCH 181/417] Add git ignore for generated Dockerfile-x. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9f81d3f..bceb4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ .mypy_cache .pytest_cache .python-version +Dockerfile-x From 3d0720bae1c0928402b11cf4174291c7929e8e03 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:44:59 -0400 Subject: [PATCH 182/417] Replace previous Docker workflows with a moderinzed, complete-workflow. These changes are based on the "complete workflow" from https://github.com/docker/build-push-action Additionally it adds support for the new "workflow_dispatch" event type. --- .github/workflows/build.yml | 229 ++++++++++++++++++++++++++++++---- .github/workflows/release.yml | 52 -------- push_readme.sh | 4 +- 3 files changed, 207 insertions(+), 78 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf84602..a489565 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,17 +3,37 @@ name: build on: push: + branches: + - '**' + tags: + - 'v*.*.*' pull_request: + schedule: + - cron: '0 10 * * *' # everyday at 10am repository_dispatch: types: [apb] + workflow_dispatch: + inputs: + remote-shell: + description: "Debug with remote shell" + required: true + default: false + image-tag: + description: "Tag to apply to pushed images" + required: true + default: dispatch env: + BUILDX_CACHE_DIR: ~/.cache/buildx IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip + PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ + linux/arm64,linux/ppc64le,linux/s390x" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit jobs: lint: + name: "Lint sources" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -46,32 +66,126 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files - build: + prepare: + name: "Prepare build variables" runs-on: ubuntu-latest + outputs: + created: ${{ steps.prep.outputs.created }} + repometa: ${{ steps.repo.outputs.result }} + source_version: ${{ steps.prep.outputs.source_version }} + tags: ${{ steps.prep.outputs.tags }} steps: - uses: actions/checkout@v2 - - name: Determine image version - run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - - name: Build docker image - run: | - version=${{ env.IMAGE_VERSION }} - docker build \ - --tag "$IMAGE_NAME" \ - --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ - --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ env.IMAGE_VERSION }} \ - . - - name: Save docker image artifact + - name: Gather repository metadata + id: repo + uses: actions/github-script@v3 + with: + script: | + const repo = await github.repos.get(context.repo) + return repo.data + - name: Calculate output values + id: prep run: | - mkdir dist - version=${{ env.IMAGE_VERSION }} - docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz + VERSION=noop + SEMVER="^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$" + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION=${{ github.event.inputs.image-tag }} + elif [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; + then + VERSION=edge + fi + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + if [[ $VERSION =~ $SEMVER ]]; then + VERSION_NO_V=${VERSION#v} + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" + TAGS="${IMAGE_NAME}:${VERSION_NO_V//+/_},${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH},${IMAGE_NAME}:${MAJOR}.${MINOR},${IMAGE_NAME}:${MAJOR},${IMAGE_NAME}:latest" + else + TAGS="${IMAGE_NAME}:${VERSION}" + fi + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=source_version::$(./bump_version.sh show) + echo ::set-output name=tags::${TAGS} + echo tags=${TAGS} + - name: Setup debug session remote shell + uses: mxschmitt/action-tmate@v3 + if: github.event.inputs.remote-shell == 'true' + build: + name: "Build test image" + runs-on: ubuntu-latest + needs: [prepare] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: ${{ env.BUILDX_CACHE_DIR }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Create dist directory + run: mkdir -p dist + - name: Build image + id: docker_build + uses: docker/build-push-action@v2 + with: + build-args: | + VERSION=${{ needs.prepare.outputs.source_version }} + cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} + cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} + context: . + file: ./Dockerfile + outputs: type=docker,dest=dist/image.tar + tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + labels: "\ + org.opencontainers.image.created=${{ + needs.prepare.outputs.created }} + + org.opencontainers.image.description=${{ + fromJson(needs.prepare.outputs.repometa).description }} + + org.opencontainers.image.licenses=${{ + fromJson(needs.prepare.outputs.repometa).license.spdx_id }} + + org.opencontainers.image.revision=${{ github.sha }} + + org.opencontainers.image.source=${{ + fromJson(needs.prepare.outputs.repometa).clone_url }} + + org.opencontainers.image.title=${{ + fromJson(needs.prepare.outputs.repometa).name }} + + org.opencontainers.image.url=${{ + fromJson(needs.prepare.outputs.repometa).html_url }} + + org.opencontainers.image.version=${{ + needs.prepare.outputs.source_version }}" + - name: Compress image + run: gzip dist/image.tar - name: Upload artifacts uses: actions/upload-artifact@v2 with: name: dist path: dist test: + name: "Test image" runs-on: ubuntu-latest needs: [build] steps: @@ -79,21 +193,21 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache testing environments - uses: actions/cache@v2 + - name: Cache pip test requirements + uses: actions/cache@v1 with: path: ${{ env.PIP_CACHE_DIR }} - key: "test-${{ runner.os }}-\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}" + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - test-${{ runner.os }}- + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v2.0.5 + uses: actions/download-artifact@v2 with: name: dist path: dist @@ -102,4 +216,71 @@ jobs: - name: Run tests env: RELEASE_TAG: ${{ github.event.release.tag_name }} - run: pytest + run: pytest --runslow + build-push-all: + name: "Build and push all platforms" + runs-on: ubuntu-latest + needs: [prepare, test] + if: github.event_name != 'pull_request' + steps: + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: ${{ env.BUILDX_CACHE_DIR }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Create cross-platform support Dockerfile-x + run: ./buildx-dockerfile.sh + - name: Build and push platform images to Docker Hub + id: docker_build + uses: docker/build-push-action@v2 + with: + build-args: | + VERSION=${{ needs.prepare.outputs.source_version }} + cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} + cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} + context: . + file: ./Dockerfile-x + platforms: ${{ env.PLATFORMS }} + push: true + tags: ${{ needs.prepare.outputs.tags }} + labels: "\ + org.opencontainers.image.created=${{ + needs.prepare.outputs.created }} + + org.opencontainers.image.description=${{ + fromJson(needs.prepare.outputs.repometa).description }} + + org.opencontainers.image.licenses=${{ + fromJson(needs.prepare.outputs.repometa).license.spdx_id }} + + org.opencontainers.image.revision=${{ github.sha }} + + org.opencontainers.image.source=${{ + fromJson(needs.prepare.outputs.repometa).clone_url }} + + org.opencontainers.image.title=${{ + fromJson(needs.prepare.outputs.repometa).name }} + + org.opencontainers.image.url=${{ + fromJson(needs.prepare.outputs.repometa).html_url }} + + org.opencontainers.image.version=${{ + needs.prepare.outputs.source_version }}" + - name: Publish README.md to Docker Hub + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + run: ./push_readme.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index dca4c36..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: release - -on: - release: - types: [prereleased, released] - -env: - IMAGE_NAME: cisagov/example - DOCKER_PW: ${{ secrets.DOCKER_PW }} - DOCKER_USER: ${{ secrets.DOCKER_USER }} - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Determine image version - run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - - name: Build Docker image - run: | - version=${{ env.IMAGE_VERSION }} - docker build \ - --tag "$IMAGE_NAME" \ - --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ - --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ env.IMAGE_VERSION }} \ - . - - name: Tag Docker image - run: | - IFS='.' read -r -a version_array \ - <<< "${{ env.IMAGE_VERSION }}" - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" - docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" - docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" - - name: Publish image to Docker Hub - run: | - IFS='.' read -r -a version_array \ - <<< "${{ env.IMAGE_VERSION }}" - docker push "${IMAGE_NAME}:latest" - docker push "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" - docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - docker push "${IMAGE_NAME}:${version_array[0]}" - - name: Publish README.md to Docker Hub - run: ./push_readme.sh diff --git a/push_readme.sh b/push_readme.sh index 0b6d07a..4532701 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -3,7 +3,7 @@ # Push the README.md file to the docker hub repository # Requires the following environment variables to be set: -# DOCKER_PW, DOCKER_USER, IMAGE_NAME +# DOCKER_PASSWORD, DOCKER_USERNAME, IMAGE_NAME set -o nounset set -o errexit @@ -11,7 +11,7 @@ set -o pipefail token=$(curl -s -X POST \ -H "Content-Type: application/json" \ - -d '{"username": "'"$DOCKER_USER"'", "password": "'"$DOCKER_PW"'"}' \ + -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq -r .token) code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 10:55:24 -0400 Subject: [PATCH 183/417] Add CodeQL workflow. Needed to check tests written in Python. --- .github/workflows/codeql-analysis.yml | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..561e8c1 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,75 @@ +--- + +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + schedule: + - cron: '0 21 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', + # 'python'] + language: ['python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a + # config file. By default, queries listed here will override any + # specified in a config file. Prefix the list here with "+" to use + # these queries and those in the config file. queries: + # ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Java). If this step fails, then you should remove it and run the build + # manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + # - run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 7530392ec4ab87301360f91577f80dbc064da4bd Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 14:54:59 -0400 Subject: [PATCH 184/417] Update README publisher to use new Docker URL. --- push_readme.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 4532701..2e55623 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -9,15 +9,17 @@ set -o nounset set -o errexit set -o pipefail +echo "Logging in and requesting JWT..." token=$(curl -s -X POST \ -H "Content-Type: application/json" \ -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq -r .token) +echo "Pushing README file..." code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 15:06:29 -0400 Subject: [PATCH 185/417] Expand command option names to long form. --- push_readme.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 2e55623..13155cd 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -10,19 +10,20 @@ set -o errexit set -o pipefail echo "Logging in and requesting JWT..." -token=$(curl -s -X POST \ - -H "Content-Type: application/json" \ - -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ - https://hub.docker.com/v2/users/login/ | jq -r .token) +token=$(curl --silent --request POST \ + --header "Content-Type: application/json" \ + --data \ + '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ + https://hub.docker.com/v2/users/login/ | jq --raw-output .token) echo "Pushing README file..." -code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 18:44:23 -0400 Subject: [PATCH 186/417] Update Dockerfile to use opencontainers metadata and build-time labels. --- Dockerfile | 11 ++--------- tests/container_test.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b4f4e61..c9184c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,11 @@ -ARG GIT_COMMIT=unspecified -ARG GIT_REMOTE=unspecified ARG VERSION=unspecified FROM python:3.9-alpine -ARG GIT_COMMIT -ARG GIT_REMOTE ARG VERSION -LABEL git_commit=${GIT_COMMIT} -LABEL git_remote=${GIT_REMOTE} -LABEL maintainer="mark.feldhousen@trio.dhs.gov" -LABEL vendor="Cyber and Infrastructure Security Agency" -LABEL version=${VERSION} +LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" +LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" diff --git a/tests/container_test.py b/tests/container_test.py index 90ea1a3..6153028 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -89,5 +89,5 @@ def test_container_version_label_matches(version_container): exec(f.read(), pkg_vars) # nosec project_version = pkg_vars["__version__"] assert ( - version_container.labels["version"] == project_version + version_container.labels["org.opencontainers.image.version"] == project_version ), "Dockerfile version label does not match project version" From e01481ba5b532010969242ff5d51d386c1ad56b7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 10:44:02 -0500 Subject: [PATCH 187/417] Prevent image pushes when linter checks fail. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a489565..7b79adb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -220,7 +220,7 @@ jobs: build-push-all: name: "Build and push all platforms" runs-on: ubuntu-latest - needs: [prepare, test] + needs: [lint, prepare, test] if: github.event_name != 'pull_request' steps: - name: Login to DockerHub From 297e52758195303ed6aa193f9b40acec7a56a98a Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 12:13:17 -0500 Subject: [PATCH 188/417] Modify cache keys to follow team-established naming conventions. Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b79adb..7bb0a2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,9 +137,9 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.BUILDX_CACHE_DIR }} - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: buildx-${{ runner.os }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx- + buildx-${{ runner.os }}- - name: Create dist directory run: mkdir -p dist - name: Build image @@ -194,14 +194,14 @@ jobs: with: python-version: 3.9 - name: Cache pip test requirements - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ - ${{ hashFiles('**/requirements-test.txt') }}" + key: "test-${{ runner.os }}-\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- + test-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip @@ -238,9 +238,9 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.BUILDX_CACHE_DIR }} - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: buildx-${{ runner.os }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx- + buildx-${{ runner.os }}- - name: Create cross-platform support Dockerfile-x run: ./buildx-dockerfile.sh - name: Build and push platform images to Docker Hub From f1e097a8641df648bd31673bfe911aae8540a05e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 12:29:06 -0500 Subject: [PATCH 189/417] Add comments pointing to the opencontainers image-spec documentation. --- .github/workflows/build.yml | 2 ++ Dockerfile | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7bb0a2f..e80f5d4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -256,6 +256,8 @@ jobs: platforms: ${{ env.PLATFORMS }} push: true tags: ${{ needs.prepare.outputs.tags }} + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} diff --git a/Dockerfile b/Dockerfile index c9184c8..8819053 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ FROM python:3.9-alpine ARG VERSION +# For a list of pre-defined annotation keys and value types see: +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# Note: Additional labels are added by the build workflow. LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" From a88d550bd9ea10b572b58945981a76a804c00263 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 16:36:43 -0500 Subject: [PATCH 190/417] Add comments to help document and break up a monolitic file. --- .github/workflows/build.yml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e80f5d4..b5dd214 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,7 @@ on: schedule: - cron: '0 10 * * *' # everyday at 10am repository_dispatch: + # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ types: [apb] workflow_dispatch: inputs: @@ -33,6 +34,8 @@ env: jobs: lint: + # Checks out the source and runs pre-commit hooks. Detects coding errors + # and style deviations. name: "Lint sources" runs-on: ubuntu-latest steps: @@ -66,7 +69,45 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files + prepare: + # Calculates and publishes outputs that are used by other jobs. + # + # Outputs: + # created: + # The current date-time in RFC3339 format. + # repometa: + # The json metadata describing this repository. + # source_version: + # The source version as reported by the `bump_version.sh show` command. + # tags: + # A comma separated list of Docker tags to be applied to the images on + # DockerHub. The tags will vary depending on: + # - The event that triggered the build. + # - The branch the build is based upon. + # - The git tag the build is based upon. + # + # When a build is based on a git tag of the form `v*.*.*` the image will + # be tagged on DockerHub with multiple levels of version specificity. + # For example, a git tag of `v1.2.3+a` will generate Docker tags of + # `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`. + # + # Builds targeting the default branch will be tagged with `:edge`. + # + # Builds from other branches will be tagged with the branch name. + # Solidi `/` in branch names are replaced with hyphens `-` in the Docker + # tag. + # + # Builds triggered by a push event are tagged with a short hash in the + # form: sha-12345678 + # + # Builds triggered by a pull request are tagged with the pull request + # number in the form pr-123. + # + # Builds triggered using the GitHub GUI (workflow_dispatch) are tagged + # with the value specified by the user. + # + # Scheduled builds are tagged with `:nightly`. name: "Prepare build variables" runs-on: ubuntu-latest outputs: @@ -122,7 +163,10 @@ jobs: - name: Setup debug session remote shell uses: mxschmitt/action-tmate@v3 if: github.event.inputs.remote-shell == 'true' + build: + # Builds a single test image for the native platform. This image is saved + # as an artifact and loaded by the test job. name: "Build test image" runs-on: ubuntu-latest needs: [prepare] @@ -154,6 +198,8 @@ jobs: file: ./Dockerfile outputs: type=docker,dest=dist/image.tar tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} @@ -184,7 +230,9 @@ jobs: with: name: dist path: dist + test: + # Executes tests on the single-platform image created in the "build" job. name: "Test image" runs-on: ubuntu-latest needs: [build] @@ -217,7 +265,13 @@ jobs: env: RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest --runslow + build-push-all: + # Builds the final set of images for each of the platforms listed in + # PLATFORMS environment variable. These images are tagged with the Docker + # tags calculated in the "prepare" job and pushed to DockerHub. The + # contents of README.md is pushed as the image's description. This job is + # skipped when the triggering event is a pull request. name: "Build and push all platforms" runs-on: ubuntu-latest needs: [lint, prepare, test] From a842abbfefefdb03bd1661dfb5956d7b09eeae25 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 3 Nov 2020 12:40:54 -0500 Subject: [PATCH 191/417] Insist that the cisagov devs are the owners of the .github directory This additional clause must remain at the _end_ of the CODEOWNERS file so that it cannot be overridden by a later clause. We want to make it so that all the .github files including CODEOWNERS are protected so only code owners (the dev team) can approve modifications to them. This will prevent configuration changes from breaking Actions and other management-type functions that the files in this directory control. By setting the .github files/folder to require code owner approval for changes, workflow and management changes will require dev team review and checking. Resolves #56. --- .github/CODEOWNERS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7735a52..5671d70 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,10 @@ # Each line is a file pattern followed by one or more owners. -# These owners will be the default owners for everything in -# the repo. Unless a later match takes precedence, -# these owners will be requested for review when someone -# opens a pull request. +# These owners will be the default owners for everything in the +# repo. Unless a later match takes precedence, these owners will be +# requested for review when someone opens a pull request. * @dav3r @felddy @hillaryj @jsf9k @mcdonnnj + +# These folks own any files in the /.github directory at the root of +# the repository and any of its subdirectories. +/.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From 5b199bc86974d09044df2e510d7ac4c6ba4ad850 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 3 Nov 2020 14:51:18 -0500 Subject: [PATCH 192/417] Remove offending slash Thanks to @dav3r and @mcdonnnj for the suggestion. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5671d70..9c3d21f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,6 +5,6 @@ # requested for review when someone opens a pull request. * @dav3r @felddy @hillaryj @jsf9k @mcdonnnj -# These folks own any files in the /.github directory at the root of +# These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. /.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From a29dc98d6f00c9d6ad6e05fc9f6b99163eb512db Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 5 Nov 2020 11:54:24 -0500 Subject: [PATCH 193/417] Apply correct typographic designation to character replacement comment. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5dd214..63afd50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,8 +94,8 @@ jobs: # # Builds targeting the default branch will be tagged with `:edge`. # - # Builds from other branches will be tagged with the branch name. - # Solidi `/` in branch names are replaced with hyphens `-` in the Docker + # Builds from other branches will be tagged with the branch name. Solidi + # `/` in branch names are replaced with hyphen-minuses `-` in the Docker # tag. # # Builds triggered by a push event are tagged with a short hash in the From e9426727634b5a8c198f83395262966290364f85 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 5 Nov 2020 12:22:58 -0500 Subject: [PATCH 194/417] Modernize Docker shields in README to point to working links. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index abfe21e..99246a9 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ ## Docker Image ## -![MicroBadger Layers](https://img.shields.io/microbadger/layers/cisagov/example.svg) -![MicroBadger Size](https://img.shields.io/microbadger/image-size/cisagov/example.svg) +[![Docker Pulls](https://img.shields.io/docker/pulls/cisagov/example)](https://hub.docker.com/r/cisagov/example) +[![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/cisagov/example)](https://hub.docker.com/r/cisagov/example) +[![Platforms](https://img.shields.io/badge/platforms-amd64%20%7C%20arm%2Fv6%20%7C%20arm%2Fv7%20%7C%20arm64%20%7C%20ppc64le%20%7C%20s390x-blue)](https://hub.docker.com/r/cisagov/skeleton-docker/tags) This is a docker skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub docker project From da037f5045c34927038f264733abc51816e2b449 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 5 Nov 2020 13:04:26 -0500 Subject: [PATCH 195/417] Elaborate on the Solidus. Add some additional comments in case anyone is confused about the nature of the solidus and its place within our tagging system. Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 63afd50..d56e3eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -95,8 +95,11 @@ jobs: # Builds targeting the default branch will be tagged with `:edge`. # # Builds from other branches will be tagged with the branch name. Solidi - # `/` in branch names are replaced with hyphen-minuses `-` in the Docker - # tag. + # (`/` characters - commonly known as slashes) in branch names are + # replaced with hyphen-minuses (`-` characters) in the Docker tag. For + # more information about the solidus see these links: + # * https://www.compart.com/en/unicode/U+002F + # * https://en.wikipedia.org/wiki/Slash_(punctuation)#Encoding # # Builds triggered by a push event are tagged with a short hash in the # form: sha-12345678 From 02c5a6f8bf72e7695cc5957cd62a7e137f55f80e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 10 Nov 2020 22:10:38 -0500 Subject: [PATCH 196/417] Use the python version output by actions/setup-python There is no need to run python code to determine the python version. Resolves #58. See here for details: https://github.com/actions/setup-python/blob/main/action.yml#L14-L16 --- .github/workflows/build.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0563b8..bae558f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,26 +16,25 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - id: setup-python + uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Store installed Python version - run: | - echo "PY_VERSION="\ - "$(python -c "import platform;print(platform.python_version())")" \ - >> $GITHUB_ENV - name: Cache linting environments uses: actions/cache@v2 with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ + key: | + lint-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - lint-${{ runner.os }}-py${{ env.PY_VERSION }}- + lint-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}- lint-${{ runner.os }}- - name: Install dependencies run: | From 64c3fb61ab1eb832821d57882f42407e88970740 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 15 Nov 2020 23:02:59 -0500 Subject: [PATCH 197/417] Organize pre-commit hooks Group related pre-commit hooks together. Make sure that hooks are alphabetically sorted within those groups. --- .pre-commit-config.yaml | 48 +++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7362f89..e9dde69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,30 +26,30 @@ repos: - --autofix - id: requirements-txt-fixer - id: trailing-whitespace + + # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.24.0 hooks: - id: markdownlint args: - --config=.mdl_config.json + - repo: https://github.com/prettier/pre-commit + rev: v2.1.2 + hooks: + - id: prettier - repo: https://github.com/adrienverge/yamllint rev: v1.25.0 hooks: - id: yamllint + + # Shell script hooks - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: - id: shell-lint - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 - hooks: - - id: flake8 - additional_dependencies: - - flake8-docstrings - - repo: https://github.com/asottile/pyupgrade - rev: v2.7.2 - hooks: - - id: pyupgrade + + # Python hooks - repo: https://github.com/PyCQA/bandit rev: 1.6.2 hooks: @@ -60,15 +60,33 @@ repos: rev: 20.8b1 hooks: - id: black + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + additional_dependencies: + - flake8-docstrings - repo: https://github.com/timothycrosley/isort rev: 5.6.4 hooks: - id: isort + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.790 + hooks: + - id: mypy + - repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + + # Ansible hooks - repo: https://github.com/ansible/ansible-lint.git rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml + + # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.43.0 hooks: @@ -89,15 +107,9 @@ repos: # above have been resolved, which we hope will be with the release of # Terraform 0.13. # - id: terraform_validate + + # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit rev: v2.0.0 hooks: - id: docker-compose-check - - repo: https://github.com/prettier/pre-commit - rev: v2.1.2 - hooks: - - id: prettier - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 - hooks: - - id: mypy From 9b62ff0789df5d2d5c65f0674f13a12438dcb2b1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 18 Nov 2020 16:43:19 -0500 Subject: [PATCH 198/417] Fix test job missing Python version in cache key --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d56e3eb..5666fb8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -241,17 +241,21 @@ jobs: needs: [build] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - id: setup-python + uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache pip test requirements + - name: Cache testing environments uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | + test-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}- test-${{ runner.os }}- - name: Install dependencies run: | From f4131e57d81557a751eec90ff0f07784809aa5d2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 20 Nov 2020 12:22:39 -0500 Subject: [PATCH 199/417] Clean up our actions/cache step Removed name because it was not more informative than the default. Swapped out a hardcoded job reference for the github.job context value. Switch the base cache key to a step environment value so we can set it once and reuse. Removed additional restore-key value that might have undesirable results. --- .github/workflows/build.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bae558f..5c65f71 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,22 +20,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache linting environments - uses: actions/cache@v2 + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: | - lint-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ + key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - lint-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- - lint-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | python -m pip install --upgrade pip From e3b0253bd16cd14efb69f732c028b5dbf5c4851c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Dec 2020 11:26:53 -0500 Subject: [PATCH 200/417] Update remaining actions/cache uses Make sure all actions/cache steps are in-line with the changes made to the lint job's step. --- .github/workflows/build.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 05f5110..ce70bf5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -179,11 +179,13 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Cache Docker layers uses: actions/cache@v2 + env: + BASE_CACHE_KEY: buildx-${{ runner.os }}- with: path: ${{ env.BUILDX_CACHE_DIR }} - key: buildx-${{ runner.os }}-${{ github.sha }} + key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} restore-keys: | - buildx-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Create dist directory run: mkdir -p dist - name: Build image @@ -244,16 +246,16 @@ jobs: python-version: 3.9 - name: Cache testing environments uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" with: path: ${{ env.PIP_CACHE_DIR }} - key: "test-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ + key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - test-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- - test-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | python -m pip install --upgrade pip @@ -294,11 +296,13 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Cache Docker layers uses: actions/cache@v2 + env: + BASE_CACHE_KEY: buildx-${{ runner.os }}- with: path: ${{ env.BUILDX_CACHE_DIR }} - key: buildx-${{ runner.os }}-${{ github.sha }} + key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} restore-keys: | - buildx-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Create cross-platform support Dockerfile-x run: ./buildx-dockerfile.sh - name: Build and push platform images to Docker Hub From b769825c41defa9365c108fbed03e79b753680c1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:03:13 -0500 Subject: [PATCH 201/417] Update prettier hook details Per https://github.com/prettier/prettier/pull/8937 the pre-commit hook has been moved to https://github.com/pre-commit/mirrors-prettier. I have also updated to the latest version in that repository. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..1d6845f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: markdownlint args: - --config=.mdl_config.json - - repo: https://github.com/prettier/pre-commit - rev: v2.1.2 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.2.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint From 2d6bf50eab010ed699f5ed38f91fd4f397c473fd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:13:04 -0500 Subject: [PATCH 202/417] Update black hook repo url At some point the python GitHub organization renamed to psf (Python Software Foundation). Although it redirects with no issue, I think it should be updated to reflect the current repository URL. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d6845f..af393d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,7 +56,7 @@ repos: - id: bandit args: - --config=.bandit.yml - - repo: https://github.com/python/black + - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black From 5115baee52b77770f86f22af8618c432c8f0c064 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:15:03 -0500 Subject: [PATCH 203/417] Remove trailing '.git's from repository URLs Two hooks had trailing '.git's in the URLs. Although this is not a problem, we should be consistent in how we format things. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af393d0..fc1ed74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,14 +80,14 @@ repos: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible/ansible-lint.git + - repo: https://github.com/ansible/ansible-lint rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - - repo: https://github.com/antonbabenko/pre-commit-terraform.git + - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.43.0 hooks: - id: terraform_fmt From a7f86ddad0b4a70d0ee06ce8f6316749cb83e3f9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:21:03 -0500 Subject: [PATCH 204/417] Enable two additional hooks from default pre-commit-hooks Enabled 'check-case-conflict' because of our mixed Linux and macOS development. Although APFS supports case-sensitive containers, it is not the default as far as I am aware. Linux filesystems are typically case-sensitive however. With the merge of https://github.com/cisagov/development-guide/pull/42 we now have a TOML file in a repository, so it does not hurt to add this hook in case more are added in the future. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc1ed74..3e2c07c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,9 +7,11 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: + - id: check-case-conflict - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-toml - id: check-xml - id: debug-statements - id: detect-aws-credentials From 06159cbb65e0f5bf76ffe487b82684f0fc632f59 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:30:17 -0500 Subject: [PATCH 205/417] Run pre-commit autoupdate --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e2c07c..cc668e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v3.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.24.0 + rev: v0.26.0 hooks: - id: markdownlint args: @@ -53,7 +53,7 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.6.2 + rev: 1.7.0 hooks: - id: bandit args: @@ -77,20 +77,20 @@ repos: hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.7.2 + rev: v2.7.4 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v4.3.5 + rev: v4.3.7 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.43.0 + rev: v1.45.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue From d836e91d323472fbdab0becb78bac7c79e3a2e73 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 17 Dec 2020 13:50:34 -0500 Subject: [PATCH 206/417] Update repo URL for the isort hook I missed that this repository was transfered from the creator, Timothy Crosley, to the PyCQA organization. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc668e1..e042de8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: - id: flake8 additional_dependencies: - flake8-docstrings - - repo: https://github.com/timothycrosley/isort + - repo: https://github.com/PyCQA/isort rev: 5.6.4 hooks: - id: isort From 46ec3e7002c98c6653b39595008456eff1cd7f01 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:06:18 -0500 Subject: [PATCH 207/417] Update ansible-lint hook URL --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e042de8..af4fed5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,7 +82,7 @@ repos: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible/ansible-lint + - repo: https://github.com/ansible-community/ansible-lint rev: v4.3.7 hooks: - id: ansible-lint From 362b054459b15b6dd3ff5d7eb587682fee82f72a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:08:07 -0500 Subject: [PATCH 208/417] Run pre-commit autoupdate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af4fed5..2b3a71e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.6.4 + rev: 5.7.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 8f7435a6fa7494c380b7b45831c7dec5ce3cd2bc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:09:53 -0500 Subject: [PATCH 209/417] Add the pre-commit-packer repo and hooks --- .pre-commit-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..28aa9bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -113,3 +113,10 @@ repos: rev: v2.0.0 hooks: - id: docker-compose-check + + # Packer hooks + - repo: https://github.com/cisagov/pre-commit-packer + rev: v0.0.2 + hooks: + - id: packer_validate + - id: packer_fmt From 2fb4e15cf6c743a3ce0ab434d3de7ae62ffc6fe2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 7 Jan 2021 04:07:38 -0500 Subject: [PATCH 210/417] Add the --strict flag to the yamllint pre-commot hook The --strict flag will cause the yamllint hook to exit with a non-zero exit code when warnings are found instead of only when errors are found. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..26c06a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,8 @@ repos: rev: v1.25.0 hooks: - id: yamllint + args: + - --strict # Shell script hooks - repo: https://github.com/detailyang/pre-commit-shell From ecdfc527d4338218f005684439214c2d126aa11b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 15 Jan 2021 10:34:34 -0500 Subject: [PATCH 211/417] Add the beautysh pre-commit hook This hook performs auto-formatting (beautifying) of Bash scripts. --- .pre-commit-config.yaml | 7 +++++++ setup-env | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a3c7eb..42a2c7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,13 @@ repos: - --strict # Shell script hooks + - repo: https://github.com/lovesegfault/beautysh + rev: 6.0.1 + hooks: + - id: beautysh + args: + - --indent-size + - '2' - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: diff --git a/setup-env b/setup-env index 4d822c4..1579e04 100755 --- a/setup-env +++ b/setup-env @@ -50,14 +50,14 @@ while (( "$#" )); do shift ;; -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 - exit 1 - ;; + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift - ;; - esac + PARAMS="$PARAMS $1" + shift + ;; + esac done # set positional arguments in their proper place From 6bd93ff6fbd1e9fe1a6e508d6a65225b63220a53 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 31 Jan 2021 19:24:54 -0500 Subject: [PATCH 212/417] Update pre-commit hooks with `pre-commit autoupdate` --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a3c7eb..e1da8e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.25.0 + rev: v1.26.0 hooks: - id: yamllint args: @@ -75,17 +75,17 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 + rev: v0.800 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.7.4 + rev: v2.10.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v4.3.7 + rev: v5.0.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From 6e33a4e5fb7e4e91e14971e33283902f11e3d9c1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 10:44:50 -0500 Subject: [PATCH 213/417] Add a requirements-dev.txt pip requirements file Our other skeletons have this file, even if they are not Python projects. The directions in CONTRIBUTING.md reference such a file, which can be misleading. Rather than amending the CONTRIBUTING documentation, I feel it's better to align with our other skeletons for consistency. --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..d84ee68 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +--requirement requirements-test.txt +ipython From c97a883ada7ae0cbd50f790d4deb4e79639736dd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 13:18:08 -0500 Subject: [PATCH 214/417] Revert ansible-lint version update The v5 release of ansible-lint introduces breaking changes, so we are holding off on updating until things are resolved. Conversation about this can be tracked in https://github.com/cisagov/skeleton-ansible-role/issues/69. --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1da8e2..4fdfdff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,7 +85,9 @@ repos: # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.0.0 + # This is intentionally being held back because of issues in v5 per + # https://github.com/cisagov/skeleton-ansible-role/issues/69 + rev: v4.3.7 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From ab1bcd5ddea599ad614da235f6b43e72630d8e77 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 17:47:35 -0500 Subject: [PATCH 215/417] Autoformat bump_version.sh with beautysh --- bump_version.sh | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index 7c23343..81a394f 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -14,34 +14,34 @@ old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) if [ $# -ne 1 ] then - echo "$HELP_INFORMATION" + echo "$HELP_INFORMATION" else - case $1 in - major|minor|patch|prerelease|build) - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bumping version from $old_version to $new_version" - git push - ;; - finalize) - new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bumping version from $old_version to $new_version" - git push - ;; - show) - echo "$old_version" - ;; - *) - echo "$HELP_INFORMATION" - ;; - esac + case $1 in + major|minor|patch|prerelease|build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + show) + echo "$old_version" + ;; + *) + echo "$HELP_INFORMATION" + ;; + esac fi From ae83ed86e32b3a70e98881efae218e400d789267 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 17:48:28 -0500 Subject: [PATCH 216/417] Autoformat push_readme.sh with beautysh --- push_readme.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 13155cd..12e2340 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -11,19 +11,19 @@ set -o pipefail echo "Logging in and requesting JWT..." token=$(curl --silent --request POST \ - --header "Content-Type: application/json" \ - --data \ - '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ + --header "Content-Type: application/json" \ + --data \ + '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq --raw-output .token) echo "Pushing README file..." code=$(jq --null-input --arg msg "$( Date: Tue, 16 Feb 2021 22:39:33 -0500 Subject: [PATCH 217/417] Fix the agency name in the vendor label --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8819053..1c48a45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ARG VERSION # https://github.com/opencontainers/image-spec/blob/master/annotations.md # Note: Additional labels are added by the build workflow. LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" -LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" +LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" From 9f3efb52537f19a41211c9346dfd03e898fee8fd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 26 Feb 2021 16:43:38 -0500 Subject: [PATCH 218/417] Update CodeQL workflow for GitHub Actions I noticed the following warning when looking at GHA workflow runs: 1 issue was detected with this workflow: git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results. After looking into it I updated the workflow per this page on GitHub Docs: https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-the-codeql-workflow#warning-git-checkout-head2-is-no-longer-necessary --- .github/workflows/codeql-analysis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 561e8c1..127ef08 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,15 +33,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL From 8e92286049b45da9eb219368ab3a6c29092665ff Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 25 Mar 2021 17:37:30 -0400 Subject: [PATCH 219/417] Enable container publication to GitHub Container Registry. * Duplicates DockerHub tags for ghcr.io * Adds login to GitHub Container Registry using the GITHUB_TOKEN secret. * Documentation update to reflect multi-registry publication. --- .github/workflows/build.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce70bf5..a45d7f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -156,6 +156,10 @@ jobs: if [ "${{ github.event_name }}" = "push" ]; then TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}" fi + for i in ${TAGS//,/ } + do + TAGS="${TAGS},ghcr.io/${i}" + done echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') echo ::set-output name=source_version::$(./bump_version.sh show) echo ::set-output name=tags::${TAGS} @@ -275,9 +279,10 @@ jobs: build-push-all: # Builds the final set of images for each of the platforms listed in # PLATFORMS environment variable. These images are tagged with the Docker - # tags calculated in the "prepare" job and pushed to DockerHub. The - # contents of README.md is pushed as the image's description. This job is - # skipped when the triggering event is a pull request. + # tags calculated in the "prepare" job and pushed to DockerHub and the + # GitHub Container Registry. The contents of README.md is pushed as the + # image's description to DockerHub. This job is skipped when the triggering + # event is a pull request. name: "Build and push all platforms" runs-on: ubuntu-latest needs: [lint, prepare, test] @@ -288,6 +293,12 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout uses: actions/checkout@v2 - name: Set up QEMU @@ -305,7 +316,7 @@ jobs: ${{ env.BASE_CACHE_KEY }} - name: Create cross-platform support Dockerfile-x run: ./buildx-dockerfile.sh - - name: Build and push platform images to Docker Hub + - name: Build and push platform images to registries id: docker_build uses: docker/build-push-action@v2 with: From 27c0451a4f18c6d0d3b87df4783b6b13795cff73 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Fri, 26 Mar 2021 12:38:26 -0400 Subject: [PATCH 220/417] Update .github/workflows/build.yml Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a45d7f8..9afe401 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -280,7 +280,7 @@ jobs: # Builds the final set of images for each of the platforms listed in # PLATFORMS environment variable. These images are tagged with the Docker # tags calculated in the "prepare" job and pushed to DockerHub and the - # GitHub Container Registry. The contents of README.md is pushed as the + # GitHub Container Registry. The contents of README.md are pushed as the # image's description to DockerHub. This job is skipped when the triggering # event is a pull request. name: "Build and push all platforms" From 6427dfaede0d45de6e425cb42936e4a3e9de37e3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 6 Apr 2021 16:17:57 -0400 Subject: [PATCH 221/417] Add better build instructions to the README. * Add instructions for cross-platform builds * Clean up shields * Clean up tables for each Docker configuration item * Add section about supported tags --- README.md | 210 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 183 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 99246a9..42faa08 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # skeleton-docker 💀🐳 # -[![GitHub Build Status](https://github.com/cisagov/skeleton-docker/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-docker/actions) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/) -[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/context:python) +[![GitHub Build Status](https://github.com/cisagov/skeleton-docker/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/build.yml) +[![CodeQL](https://github.com/cisagov/skeleton-docker/workflows/CodeQL/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/codeql-analysis.yml) +[![Known Vulnerabilities](https://snyk.io/test/github/cisagov/skeleton-docker/badge.svg)](https://snyk.io/test/github/cisagov/skeleton-docker) ## Docker Image ## @@ -17,49 +17,205 @@ information](LICENSE), as well as [pre-commit hooks](https://pre-commit.com) and [GitHub Actions](https://github.com/features/actions) configurations appropriate for docker containers and the major languages that we use. -## Usage ## +## Running ## -### Install ### +### Using Docker ### -Pull `cisagov/example` from the Docker repository: +```console +docker run cisagov/example:0.0.1 +``` - docker pull cisagov/example +## Using a Docker composition ### -Or build `cisagov/example` from source: +1. Create a `docker-compose.yml` file similar to the one below. - git clone https://github.com/cisagov/skeleton-docker.git - cd skeleton-docker - docker-compose build --build-arg VERSION=0.0.1 + ```yaml + version: "3.8" -### Run ### + services: + example: + image: cisagov/example:0.0.1 + volumes: + - type: bind + source: + target: /var/log + environment: + - ECHO_MESSAGE="Hello from docker-compose" + ports: + - target: "8080" + published: "8080" + protocol: tcp + ``` - docker-compose run --rm example +1. Start the container and detach: -## Ports ## + ```console + docker-compose up --detach + ``` -This container exposes the following ports: +## Using secrets ## -| Port | Protocol | Service | -|-------|----------|----------| -| 8080 | TCP | http | +This container also supports passing sensitive values via [Docker +secrets](https://docs.docker.com/engine/swarm/secrets/). Passing sensitive +values like your credentials can be more secure using secrets than using +environment variables. See the +[secrets](#secrets) section below for a table of all supported secret files. -## Environment Variables ## +1. To use secrets, create a `quote.txt` file containing the values you want set: -| Variable | Default Value | Purpose | -|---------------|-------------------------------|--------------| -| ECHO_MESSAGE | `Hello World from Dockerfile` | Text to echo | + ```text + Better lock it in your pocket. + ``` -## Secrets ## +1. Then add the secret to your `docker-compose.yml` file: + + ```yaml + version: "3.8" + + secrets: + quote_txt: + file: quote.txt + + services: + example: + image: cisagov/example:0.0.1 + volumes: + - type: bind + source: + target: /var/log + environment: + - ECHO_MESSAGE="Hello from docker-compose" + ports: + - target: "8080" + published: "8080" + protocol: tcp + secrets: + - source: quote_txt + target: quote.txt + ``` + +## Updating ## + +### Docker-compose ### + +1. Pull the new image from Docker hub: + + ```console + docker-compose pull + ``` + +1. Recreate the running container: + + ```console + docker-compose up --detach + ``` -| Filename | Purpose | -|---------------|----------------------| -| quote.txt | Secret text to echo | +### Docker ### + +1. Stop the running container: + + ```console + docker stop + ``` + +1. Pull the new image: + + ```console + docker pull cisagov/example:0.0.1 + ``` + +1. Follow the previous instructions for [running](#running) the container above. + +## Image tags ## + +The images of this container are tagged with [semantic +versions](https://semver.org) of the underlying example project that they +containerize. It is recommended that most users use a version tag. e.g., +`:0.0.1` + +| Image:tag | Description | +|-----------|-------------| +|`cisagov/example:1.2.3`| An exact release version. | +|`cisagov/example:1.2`| The most recent release matching the major and minor version numbers. | +|`cisagov/example:1`| The most recent release matching the major version number. | +|`cisagov/example:edge` | The most recent image built from a merge into the `develop` branch of this repository. | +|`cisagov/example:nightly` | A nightly build of the `develop` branch of this repository. | +|`cisagov/example:latest`| The most recent release image pushed to a container registry. Pulling an image using the `:latest` tag [should be avoided.](https://vsupalov.com/docker-latest-tag/) | + +See the [tags tab](https://hub.docker.com/r/cisagov/example/tags) on Docker +Hub for a list of all the supported tags. ## Volumes ## | Mount point | Purpose | |-------------|----------------| -| /var/log | logging output | +| `/var/log` | Log storage | + +## Environment variables ## + +### Required ### + +There are no required environment variables. + + + +### Optional ### + +| Name | Purpose | Default | +|-------|---------|---------| +| `ECHO_MESSAGE` | Sets the message echoed by this container. | `Hello World from Dockerfile` | + +## Secrets ## + +| Filename | Purpose | +|--------------|---------| +| `quote.txt` | Replaces the secret stored in the example library's package data. | + +## Building from source ## + +Build the image locally using this git repository as the [build context](https://docs.docker.com/engine/reference/commandline/build/#git-repositories): + +```console +docker build \ + --build-arg VERSION=0.0.1 \ + --tag cisagov/example:0.0.1 \ + https://github.com/cisagov/example.git#develop +``` + +## Cross-platform builds ## + +To create images that are compatible with other platforms you can use the +[`buildx`](https://docs.docker.com/buildx/working-with-buildx/) feature of +Docker: + +1. Copy the project to your machine using the `Code` button above + or the command line: + + ```console + git clone https://github.com/cisagov/example.git + cd example + ``` + +1. Create the `Dockerfile-x` file with `buildx` platform support: + + ```console + ./buildx-dockerfile.sh + ``` + +1. Build the image using `buildx`: + + ```console + docker buildx build \ + --file Dockerfile-x \ + --platform linux/amd64 \ + --build-arg VERSION=0.0.1 \ + --output type=docker \ + --tag cisagov/example:0.0.1 . + ``` ## New Repositories from a Skeleton ## From cd0d27203224b0a3edac3295435f7c26c15aa3cf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Apr 2021 17:02:24 -0400 Subject: [PATCH 222/417] Skip CodeQL workflow for Dependabot branches on push events This should resolve the following error: Error: Workflows triggered by Dependabot on the "push" event run with read-only access. Uploading Code Scanning results requires write access. To use Code Scanning with Dependabot, please ensure you are using the "pull_request" event for this workflow and avoid triggering on the "push" event for Dependabot branches. See https://docs.github.com/en/code-security/secure-coding/configuring-code-scanning#scanning-on-push for more information on how to configure these events. --- .github/workflows/codeql-analysis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 127ef08..33d1999 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -9,6 +9,9 @@ name: "CodeQL" on: push: + # Dependabot triggered push events have read-only access, but uploading code + # scanning requires write access. + branches-ignore: [dependabot/**] pull_request: # The branches below must be a subset of the branches above branches: [develop] From d354a54ede128286a4ac09287ac4e513e2a202b1 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 14:55:33 -0400 Subject: [PATCH 223/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42faa08..a2a30af 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Docker: --tag cisagov/example:0.0.1 . ``` -## New Repositories from a Skeleton ## +## New repositories from a skeleton ## Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) for step-by-step instructions on how to start a new repository from From 267e56bb80c2832c243a488d22e8cc8644b1cb78 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 14:55:46 -0400 Subject: [PATCH 224/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2a30af..754ff79 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ docker build \ ## Cross-platform builds ## -To create images that are compatible with other platforms you can use the +To create images that are compatible with other platforms, you can use the [`buildx`](https://docs.docker.com/buildx/working-with-buildx/) feature of Docker: From d0d474c1a6ef4b44cec5221cf5199a37bb57e8b9 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 14:56:12 -0400 Subject: [PATCH 225/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 754ff79..96a5768 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ environment variables. See the docker pull cisagov/example:0.0.1 ``` -1. Follow the previous instructions for [running](#running) the container above. +1. Recreate and run the container by following the [previous instructions](#running-with-docker). ## Image tags ## From d58e50c628abc2c5c6bc7dd33d93863447cbc2be Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 14:56:24 -0400 Subject: [PATCH 226/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96a5768..a83610f 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ appropriate for docker containers and the major languages that we use. ## Running ## -### Using Docker ### +### Running with Docker ### ```console docker run cisagov/example:0.0.1 From c8ea90222231a011fe1a4e1c078c8020b4a74c43 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 14:56:36 -0400 Subject: [PATCH 227/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a83610f..7c2792b 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ environment variables. See the docker-compose pull ``` -1. Recreate the running container: +1. Recreate the running container by following the [previous instructions](#running-with-docker-compose): ```console docker-compose up --detach From f014b4c36577b771fb3be7173241fec0fe494da1 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 15:04:48 -0400 Subject: [PATCH 228/417] Update README.md Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c2792b..4d21a3b 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ environment variables. See the ## Updating ## -### Docker-compose ### +### Docker Compose ### 1. Pull the new image from Docker hub: From 4df9da2402287a0c6db58a61e5aa48dfded8e2c5 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 15:07:41 -0400 Subject: [PATCH 229/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d21a3b..5778527 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ appropriate for docker containers and the major languages that we use. docker run cisagov/example:0.0.1 ``` -## Using a Docker composition ### +### Running with Docker Compose ### 1. Create a `docker-compose.yml` file similar to the one below. From 2c06a69890aa98b687db6de6c0c378a859aa76dd Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 15:07:54 -0400 Subject: [PATCH 230/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5778527..3fa2d03 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ docker run cisagov/example:0.0.1 ### Running with Docker Compose ### -1. Create a `docker-compose.yml` file similar to the one below. +1. Create a `docker-compose.yml` file similar to the one below to use [Docker Compose](https://docs.docker.com/compose/). ```yaml version: "3.8" From 6a11d943849b17cbcd2ee2e5ae22102bd85bba06 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Apr 2021 15:08:15 -0400 Subject: [PATCH 231/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fa2d03..2df2c8c 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ environment variables. See the target: quote.txt ``` -## Updating ## +## Updating your container ## ### Docker Compose ### From ba49077ab48d1ac7ad45b2bb05d1db346950284b Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 12 Apr 2021 10:37:49 -0400 Subject: [PATCH 232/417] Improve gitignore file comments and organization. * Add a comment describing what the files does. * Add sections to organize patterns. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 4b15d77..937e21d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# This file specifies intentionally untracked files that Git should ignore. +# Files already tracked by Git are not affected. +# See: https://git-scm.com/docs/gitignore + +## Python ## __pycache__ .mypy_cache .python-version From 2c4d7a2504a08f9ebbcc728b35bcd41bfc74ec63 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 15 Apr 2021 11:53:57 -0400 Subject: [PATCH 233/417] Remove user from CODEOWNERS Due to her departure this removes hillaryj from the default CODEOWNERS we use in our projects. --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9c3d21f..371258c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,8 +3,8 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jsf9k @mcdonnnj From 5fb5dbc9445abe5bb45de679f19e24d7bb44b985 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 25 Apr 2021 19:11:24 -0400 Subject: [PATCH 234/417] Add semver package to development requirements The semver package is needed by the bump_version.sh script, but it is not included in requirements-dev.txt. --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index d84ee68..cb51627 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ --requirement requirements-test.txt ipython +semver From 9938f67eecb591f7255cc4e4984b0799386803aa Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 26 Apr 2021 11:46:30 -0400 Subject: [PATCH 235/417] Unquote port integers as this is now a parse error in Docker. --- README.md | 8 ++++---- docker-compose.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2df2c8c..0becc17 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ docker run cisagov/example:0.0.1 environment: - ECHO_MESSAGE="Hello from docker-compose" ports: - - target: "8080" - published: "8080" + - target: 8080 + published: 8080 protocol: tcp ``` @@ -86,8 +86,8 @@ environment variables. See the environment: - ECHO_MESSAGE="Hello from docker-compose" ports: - - target: "8080" - published: "8080" + - target: 8080 + published: 8080 protocol: tcp secrets: - source: quote_txt diff --git a/docker-compose.yml b/docker-compose.yml index 6774387..0fc7ee5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,8 @@ services: environment: - ECHO_MESSAGE=Hello World from docker-compose! ports: - - target: "8080" - published: "8080" + - target: 8080 + published: 8080 protocol: tcp mode: host secrets: From 67f3a0cf6abe393d6bd5fc85e466d02d303fd9f8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 26 Apr 2021 12:32:39 -0400 Subject: [PATCH 236/417] Change compose specification version to match pyenv testing composition. The version attribute has been depricated and will be removed once our systems move to a more recent release. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0becc17..1215880 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ docker run cisagov/example:0.0.1 1. Create a `docker-compose.yml` file similar to the one below to use [Docker Compose](https://docs.docker.com/compose/). ```yaml - version: "3.8" + --- + version: "3.7" services: example: @@ -70,7 +71,8 @@ environment variables. See the 1. Then add the secret to your `docker-compose.yml` file: ```yaml - version: "3.8" + --- + version: "3.7" secrets: quote_txt: From 0d696361c5ac3e3b45874ec1dc6925552efd3fd2 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 26 Apr 2021 12:43:04 -0400 Subject: [PATCH 237/417] Add context around command-line instructions for non-technical folks. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1215880..f8289ca 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ appropriate for docker containers and the major languages that we use. ### Running with Docker ### +To run the `cisagov/example` image via Docker: + ```console docker run cisagov/example:0.0.1 ``` From 3b62397eed3421840f3c06b8a7a859855aca5ef9 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Mon, 26 Apr 2021 12:47:40 -0400 Subject: [PATCH 238/417] Update README.md Co-authored-by: Hillary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f8289ca..b43f663 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ docker run cisagov/example:0.0.1 docker-compose up --detach ``` -## Using secrets ## +## Using secrets with your container ## This container also supports passing sensitive values via [Docker secrets](https://docs.docker.com/engine/swarm/secrets/). Passing sensitive From bc46deb02b618a4efe15187abbdfecf67834d71b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 10 May 2021 17:04:00 -0400 Subject: [PATCH 239/417] Add the tag.sh script to this repository Since the GitHub Actions configuration builds images on tag pushes, we should use the standard tagging script we use for the same purpose in other repositories. This will make it easier to tag, build, and push Docker images of a specific version in a standardized manner. --- tag.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 tag.sh diff --git a/tag.sh b/tag.sh new file mode 100755 index 0000000..e1f7447 --- /dev/null +++ b/tag.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +version=$(./bump_version.sh show) + +git tag "v$version" && git push --tags From d405a7683240221871ecf492f12021d5e3c603c5 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Tue, 25 May 2021 17:12:10 -0400 Subject: [PATCH 240/417] Update README.md Co-authored-by: Shane Frasier --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b43f663..6066cbf 100644 --- a/README.md +++ b/README.md @@ -134,8 +134,8 @@ environment variables. See the The images of this container are tagged with [semantic versions](https://semver.org) of the underlying example project that they -containerize. It is recommended that most users use a version tag. e.g., -`:0.0.1` +containerize. It is recommended that most users use a version tag (e.g. +`:0.0.1`). | Image:tag | Description | |-----------|-------------| From 03494999d60d682a882a5191b16a41cdcdf96214 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 15 Apr 2021 11:36:21 -0400 Subject: [PATCH 241/417] Update pre-commit hooks Hooks updated with the `pre-commit autoupdate` command. Note: The `ansible-lint` hook is intentionally being held back on 4.3.7 because of ongoing issues with the 5.x version and how we use Ansible (standalone Galaxy roles). --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c851317..cf0330d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.26.0 + rev: v0.27.1 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.2.1 + rev: v2.3.0 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.0 + rev: v1.26.1 hooks: - id: yamllint args: @@ -49,7 +49,7 @@ repos: # Shell script hooks - repo: https://github.com/lovesegfault/beautysh - rev: 6.0.1 + rev: v6.1.0 hooks: - id: beautysh args: @@ -68,25 +68,25 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 21.5b2 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + rev: 3.9.2 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.7.0 + rev: 5.8.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.800 + rev: v0.812 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.10.0 + rev: v2.19.1 hooks: - id: pyupgrade @@ -101,7 +101,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.45.0 + rev: v1.50.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue From 23b324befc9cdc26ffcf1af6f333b63c545dfb7b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Jun 2021 11:14:31 -0400 Subject: [PATCH 242/417] Update configuration instructions for pyenv With the release of pyenv v2.0.0 there is a breaking change around startup logic that necessitates an update for our pyenv setup instructions. Also add a statement about how to get configuration instructions from pyenv itself. --- CONTRIBUTING.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6434d30..27bb9e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,9 @@ installation is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your profile: ```bash +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` @@ -80,14 +83,35 @@ On WSL you should treat your platform as whatever Linux distribution you've chosen to install. Once you have installed `pyenv` you will need to add the following -lines to your `.bashrc`: +lines to your `.bash_profile` (or `.profile`): + +```bash +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init --path)" +``` + +and then add the following lines to your `.bashrc`: ```bash -export PATH="$PATH:$HOME/.pyenv/bin" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` +If you want more information about setting up `pyenv` once installed, please run + +```console +pyenv init +``` + +and + +```console +pyenv virtualenv-init +``` + +for the current configuration instructions. + If you are using a shell other than `bash` you should follow the instructions that the `pyenv-installer` script outputs. From 8c517079fc308b3a7445edd252a306d196f13b1c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 2 Jun 2021 15:22:09 -0400 Subject: [PATCH 243/417] Add Docker section to the gitignore file Mirror changes in cisagov/skeleton-generic by adding a Docker section to the gitignore file and moving appropriate entries to that section. --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 157a18e..b1efc19 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,11 @@ # Files already tracked by Git are not affected. # See: https://git-scm.com/docs/gitignore +## Docker ## +Dockerfile-x + ## Python ## __pycache__ .mypy_cache .pytest_cache .python-version -Dockerfile-x From ca4306ac06cb5437be7f1bf6a1f799876ec413cc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 7 Jun 2021 08:26:02 -0400 Subject: [PATCH 244/417] Standardize the layout of the Lineage configuration file --- .github/lineage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/lineage.yml b/.github/lineage.yml index 8dfc20b..49f9c4f 100644 --- a/.github/lineage.yml +++ b/.github/lineage.yml @@ -1,6 +1,5 @@ --- -version: "1" - lineage: skeleton: remote-url: https://github.com/cisagov/skeleton-generic.git +version: '1' From 68948bb06154b3dd1b0d57139eec473b236bcd12 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 15 Jun 2021 09:54:13 -0400 Subject: [PATCH 245/417] Update the README to reference the latest version of the Docker container This change was discussed among @cisagov/team-ois here: https://github.com/cisagov/gatherer/pull/60#pullrequestreview-683086244 Also correct git comments to use the imperative mood. --- bump_version.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index 81a394f..ec9313d 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -7,6 +7,7 @@ set -o errexit set -o pipefail VERSION_FILE=src/version.txt +README_FILE=README.md HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" @@ -23,8 +24,10 @@ else tmp_file=/tmp/version.$$ sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bumping version from $old_version to $new_version" + sed "s/$old_version/$new_version/" $README_FILE > $tmp_file + mv $tmp_file $README_FILE + git add $VERSION_FILE $README_FILE + git commit -m"Bump version from $old_version to $new_version" git push ;; finalize) @@ -33,8 +36,10 @@ else tmp_file=/tmp/version.$$ sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bumping version from $old_version to $new_version" + sed "s/$old_version/$new_version/" $README_FILE > $tmp_file + mv $tmp_file $README_FILE + git add $VERSION_FILE $README_FILE + git commit -m"Bump version from $old_version to $new_version" git push ;; show) From d09151887a5e3dd4f6adbdc811ba6ff71a2a1b61 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 15 Jun 2021 09:42:02 -0400 Subject: [PATCH 246/417] Add a section to the README for ports Co-authored-by: dav3r --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 6066cbf..86a3873 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,20 @@ Hub for a list of all the supported tags. |-------------|----------------| | `/var/log` | Log storage | +## Ports ## + +There are no ports exposed by this container. + + + + + + + + + + + ## Environment variables ## ### Required ### From 1f6d0906a3fbfc19963d1bf7832315494c908329 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 16 Jun 2021 10:10:52 -0400 Subject: [PATCH 247/417] Document the single exposed (but non-functional) port Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- README.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 86a3873..446c7ab 100644 --- a/README.md +++ b/README.md @@ -157,17 +157,14 @@ Hub for a list of all the supported tags. ## Ports ## -There are no ports exposed by this container. +The following ports are exposed by this container: - +| Port | Purpose | +|------|----------------| +| 8080 | Example only; nothing is actually listening on the port | - - - - - - - +The sample [Docker composition](docker-compose.yml) publishes the +exposed port at 8080. ## Environment variables ## From 1e8f8223910f41294c16bfebea332c80fd83573c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:33:26 -0400 Subject: [PATCH 248/417] Add style enforcement rules Add rules to enforce ATX-closed headers, dashes for unordered list elements, and `1.` for ordered list elements. --- .mdl_config.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.mdl_config.json b/.mdl_config.json index 7a6f3f8..8266cdb 100644 --- a/.mdl_config.json +++ b/.mdl_config.json @@ -1,4 +1,10 @@ { + "MD003": { + "style": "atx_closed" + }, + "MD004": { + "style": "dash" + }, "MD013": { "code_blocks": false, "tables": false @@ -6,5 +12,8 @@ "MD024": { "allow_different_nesting": true }, + "MD029": { + "style": "one" + }, "default": true } From afc6bd6f6d754f8c2bc5675411535b0c61e0ec78 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:37:08 -0400 Subject: [PATCH 249/417] Add rule for image headers Add

and tags to the allowed list for MD033 (HTML elements) to support using an image as the first thing in a markdown file (header image). --- .mdl_config.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.mdl_config.json b/.mdl_config.json index 8266cdb..38bc045 100644 --- a/.mdl_config.json +++ b/.mdl_config.json @@ -15,5 +15,11 @@ "MD029": { "style": "one" }, + "MD033": { + "allowed_elements": [ + "h1", + "img" + ] + }, "default": true } From 33a28ee8f5b5dacd3a26e5479a50201164e14e85 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 22 Jun 2021 10:37:31 -0400 Subject: [PATCH 250/417] Bump actions/github-script pin from v3 to v4 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9afe401..4f94908 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -119,7 +119,7 @@ jobs: - uses: actions/checkout@v2 - name: Gather repository metadata id: repo - uses: actions/github-script@v3 + uses: actions/github-script@v4 with: script: | const repo = await github.repos.get(context.repo) From ce173f401d8eec4f2caf3cf8174a5b53f03222da Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:51:38 -0400 Subject: [PATCH 251/417] Switch to a YAML markdownlint configuration file This converts the existing `.mdl_config.json` file to an equivalent `.mdl_config.yaml` file. The reference in the markdownlint pre-commit hook configuration is updated to match. Co-authored-by: Shane Frasier --- .mdl_config.json | 25 ------------------------- .mdl_config.yaml | 24 ++++++++++++++++++++++++ .pre-commit-config.yaml | 2 +- 3 files changed, 25 insertions(+), 26 deletions(-) delete mode 100644 .mdl_config.json create mode 100644 .mdl_config.yaml diff --git a/.mdl_config.json b/.mdl_config.json deleted file mode 100644 index 38bc045..0000000 --- a/.mdl_config.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "MD003": { - "style": "atx_closed" - }, - "MD004": { - "style": "dash" - }, - "MD013": { - "code_blocks": false, - "tables": false - }, - "MD024": { - "allow_different_nesting": true - }, - "MD029": { - "style": "one" - }, - "MD033": { - "allowed_elements": [ - "h1", - "img" - ] - }, - "default": true -} diff --git a/.mdl_config.yaml b/.mdl_config.yaml new file mode 100644 index 0000000..a04720a --- /dev/null +++ b/.mdl_config.yaml @@ -0,0 +1,24 @@ +--- + +default: true + +MD003: + style: "atx_closed" + +MD004: + style: "dash" + +MD013: + code_blocks: false + tables: false + +MD024: + allow_different_nesting: true + +MD029: + style: "one" + +MD033: + allowed_elements: + - h1 + - img diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..6b87ab0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: markdownlint args: - - --config=.mdl_config.json + - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.3.0 hooks: From f2a423095efe2f1a96ce2621352ee7cb4dd0458a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:59:36 -0400 Subject: [PATCH 252/417] Add comments to markdownlint configuration Now that this is a YAML file we can add comments explaining the rule modifications we use. This will make it easier to edit or expand in the future. --- .mdl_config.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index a04720a..a2f08f3 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -1,24 +1,40 @@ --- +# Default state for all rules default: true +# MD003/heading-style/header-style - Heading style MD003: + # Enforce the ATX-closed style of header style: "atx_closed" +# MD004/ul-style - Unordered list style MD004: + # Enforce dashes for unordered lists style: "dash" +# MD013/line-length - Line length MD013: + # Do not enforce for code blocks code_blocks: false + # Do not enforce for tables tables: false +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the +# same content MD024: + # Allow headers with the same content as long as they are not in the same + # parent heading allow_different_nesting: true +# MD029/ol-prefix - Ordered list item prefix MD029: + # Enforce the `1.` style for ordered lists style: "one" +# MD033/no-inline-html - Inline HTML MD033: + # The h1 and img elements are allowed to permit header images allowed_elements: - h1 - img From d4781ee177698490b740c5dc0e3ae90282c70618 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 24 Jun 2021 01:38:44 -0400 Subject: [PATCH 253/417] Add the validate_manifest hook from pre-commit This hook will validate any pre-commit hook manifest files in the repository. --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..82a5e46 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,12 @@ repos: args: - --strict + # pre-commit hooks + - repo: https://github.com/pre-commit/pre-commit + rev: v2.13.0 + hooks: + - id: validate_manifest + # Shell script hooks - repo: https://github.com/lovesegfault/beautysh rev: v6.1.0 From 52fb946f51dad85a217b0ba88f298cd3c64f2045 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jul 2021 17:01:18 +0000 Subject: [PATCH 254/417] Bump python from 3.9-alpine to 3.9.6-alpine Bumps python from 3.9-alpine to 3.9.6-alpine. --- updated-dependencies: - dependency-name: python dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1c48a45..59ea4cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG VERSION=unspecified -FROM python:3.9-alpine +FROM python:3.9.6-alpine ARG VERSION From 106af21c04ae34d0402b9cfc59f386e2756776bd Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Jul 2021 13:34:16 -0400 Subject: [PATCH 255/417] Install terraform and packer for the linting job We should be doing this because the Packer and Terraform pre-commit hooks leverage the corresponding executables; therefore, it makes sense to go ahead and install the particular versions of those executables that we support. Also add support for optionally debugging via tmate. See also cisagov/skeleton-generic#74. --- .github/workflows/build.yml | 63 ++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c65f71..8fa1b2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,13 +8,16 @@ on: types: [apb] env: + CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: lint: runs-on: ubuntu-latest steps: + - uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 @@ -23,17 +26,72 @@ jobs: - uses: actions/cache@v2 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-" + py${{ steps.setup-python.outputs.python-version }}-\ + go${{ env.GO_VERSION }}-\ + packer${{ env.PACKER_VERSION }}-\ + tf${{ env.TERRAFORM_VERSION }}-" with: + # Note that the .terraform directory IS NOT included in the + # cache because if we were caching, then we would need to use + # the `-upgrade=true` option. This option blindly pulls down the + # latest modules and providers instead of checking to see if an + # update is required. That behavior defeats the benefits of caching. + # so there is no point in doing it for the .terraform directory. path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} + ${{ env.CURL_CACHE_DIR }} + ${{ steps.go-cache.outputs.dir }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + run: | + echo "GO_VERSION="\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ + >> $GITHUB_ENV + - name: Lookup go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" + - name: Install Packer + run: | + mkdir -p ${{ env.CURL_CACHE_DIR }} + PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" + curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ + --location \ + "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" + sudo unzip -o -d /usr/local/bin \ + ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" + - name: Install Terraform + run: | + mkdir -p ${{ env.CURL_CACHE_DIR }} + TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" + curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ + --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ + --location \ + "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}" + sudo unzip -d /opt/terraform \ + ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" + sudo ln -s /opt/terraform/terraform /usr/bin/terraform + sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default + sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform + - name: Install Terraform-docs + run: GO111MODULE=on go get github.com/terraform-docs/terraform-docs + - name: Find and initialize Terraform directories + run: | + for path in $(find . -not \( -type d -name ".terraform" -prune \) \ + -type f -iname "*.tf" -exec dirname "{}" \; | sort -u); do \ + echo "Initializing '$path'..."; \ + terraform init -input=false -backend=false "$path"; \ + done - name: Install dependencies run: | python -m pip install --upgrade pip @@ -42,3 +100,6 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files + - name: Setup tmate debug session + uses: mxschmitt/action-tmate@v3 + if: env.RUN_TMATE From c4810439814c3ebed6dd05f3b690b460670cd878 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Fri, 9 Jul 2021 22:43:09 -0400 Subject: [PATCH 256/417] Break out the curl cache creation into its own step Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8fa1b2f..bc632c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,9 +60,10 @@ jobs: id: go-cache run: | echo "::set-output name=dir::$(go env GOCACHE)" + - name: Setup curl cache + run: mkdir -p ${{ env.CURL_CACHE_DIR }} - name: Install Packer run: | - mkdir -p ${{ env.CURL_CACHE_DIR }} PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ @@ -72,7 +73,6 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - name: Install Terraform run: | - mkdir -p ${{ env.CURL_CACHE_DIR }} TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ From 70414cff28c661c3b76425edf5021f213f505413 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Jul 2021 22:46:47 -0400 Subject: [PATCH 257/417] Remove unnecessary line in tasks There is no reason to create /usr/bin/terraform. This is a vestige of an earlier age. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc632c3..3946d90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,6 @@ jobs: "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}" sudo unzip -d /opt/terraform \ ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" - sudo ln -s /opt/terraform/terraform /usr/bin/terraform sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform - name: Install Terraform-docs From b629f7f623490217fbd43d76fd77b4638cd4a4ec Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Jul 2021 22:48:21 -0400 Subject: [PATCH 258/417] Modify the Packer installation to model that of Terraform The Terraform installation does not destroy the existing system Terraform installation, and neither should the Packer installation. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3946d90..871bee7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,8 +69,10 @@ jobs: --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ --location \ "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -o -d /usr/local/bin \ + sudo unzip -d /opt/packer \ ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" + sudo mv /usr/local/bin/packer /usr/local/bin/packer-default + sudo ln -s /opt/packer/packer /usr/local/bin/packer - name: Install Terraform run: | TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" From 181d1b2fafa211fb7cae5b6023e1b5271b59bbda Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 10 Jul 2021 22:36:45 -0400 Subject: [PATCH 259/417] Install a specific version of terraform-docs Note that this change is dependent on the merging of cisagov/setup-env-github-action#31. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 871bee7..c8a1426 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,9 @@ jobs: sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform - name: Install Terraform-docs - run: GO111MODULE=on go get github.com/terraform-docs/terraform-docs + run: | + GO111MODULE=on go get \ + github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} - name: Find and initialize Terraform directories run: | for path in $(find . -not \( -type d -name ".terraform" -prune \) \ From bb6e566e3a8e1069ca2c6a1f441f67fc4c176685 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 11 Jul 2021 21:59:58 -0400 Subject: [PATCH 260/417] Move go installation so that it takes place before the cache task Some variables defined in the go installation are used in the cache task, so the go installation must happen first. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8a1426..04159c2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 + # GO_VERSION and GOCACHE are used by the cache task, so the go + # installation must happen before that. + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + run: | + echo "GO_VERSION="\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ + >> $GITHUB_ENV + - name: Lookup go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" - uses: actions/cache@v2 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ @@ -48,18 +62,6 @@ jobs: ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - - uses: actions/setup-go@v2 - with: - go-version: '1.16' - - name: Store installed Go version - run: | - echo "GO_VERSION="\ - "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ - >> $GITHUB_ENV - - name: Lookup go cache directory - id: go-cache - run: | - echo "::set-output name=dir::$(go env GOCACHE)" - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} - name: Install Packer From 337d1efb8f72c11cae6b83f3f5e63e8187599470 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 12 Jul 2021 09:06:41 -0400 Subject: [PATCH 261/417] Capitalize Go for consistency Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 04159c2..d84b7da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - # GO_VERSION and GOCACHE are used by the cache task, so the go + # GO_VERSION and GOCACHE are used by the cache task, so the Go # installation must happen before that. - uses: actions/setup-go@v2 with: @@ -33,7 +33,7 @@ jobs: echo "GO_VERSION="\ "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ >> $GITHUB_ENV - - name: Lookup go cache directory + - name: Lookup Go cache directory id: go-cache run: | echo "::set-output name=dir::$(go env GOCACHE)" From 8ee2116f428f1738540f0cdf261f1e4f1c15d092 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 13 Jul 2021 17:02:13 -0400 Subject: [PATCH 262/417] Prefer the newer "go install" syntax As of [Go 1.16](https://tip.golang.org/doc/go1.16#go-command) the `GO111MODULE` environment variable defaults to `on` and `go get` has been deprecated for module installation. Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d84b7da..73f345a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -88,7 +88,7 @@ jobs: sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform - name: Install Terraform-docs run: | - GO111MODULE=on go get \ + go install \ github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} - name: Find and initialize Terraform directories run: | From e2a729d0b11ab74207a3bb77367d8e9d8c577889 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 17:42:36 -0400 Subject: [PATCH 263/417] Install the shfmt tool for GHA The `shfmt` tool does not ship on the GitHub Actions runners so we must install it manually. --- .github/workflows/build.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c65f71..9dd5f7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,25 +15,44 @@ jobs: lint: runs-on: ubuntu-latest steps: + - uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 with: python-version: 3.9 + # GO_VERSION and GOCACHE are used by the cache task, so the Go + # installation must happen before that. + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - name: Store installed Go version + run: | + echo "GO_VERSION="\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ + >> $GITHUB_ENV + - name: Lookup Go cache directory + id: go-cache + run: | + echo "::set-output name=dir::$(go env GOCACHE)" - uses: actions/cache@v2 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-" + py${{ steps.setup-python.outputs.python-version }}-\ + go${{ env.GO_VERSION }}-" with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} + ${{ steps.go-cache.outputs.dir }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} + - name: Install shfmt + run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} - name: Install dependencies run: | python -m pip install --upgrade pip From 406b6880bd25a8592ad235102d4e832e05ab38e3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 17:53:55 -0400 Subject: [PATCH 264/417] Replace the beautysh hook with pre-commit-shfmt We have had a difficult time with how beautysh parses some shellscripts. I went in pursuit of an alternative and I believe shfmt to be a good alternative. Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..c915aa9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,13 +48,20 @@ repos: - --strict # Shell script hooks - - repo: https://github.com/lovesegfault/beautysh - rev: v6.1.0 + - repo: https://github.com/cisagov/pre-commit-shfmt + rev: v0.0.2 hooks: - - id: beautysh + - id: shfmt args: - - --indent-size + # Indent by two spaces + - -i - '2' + # Binary operators may start a line + - -bn + # Switch cases are indented + - -ci + # Redirect operators are followed by a space + - -sr - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: From 2b48e75b23cb80af9e97098da2dd6b9fb5eea2e4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 18:08:51 -0400 Subject: [PATCH 265/417] Apply changes from the shfmt pre-commit hook --- setup-env | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/setup-env b/setup-env index 1579e04..5d7f673 100755 --- a/setup-env +++ b/setup-env @@ -4,7 +4,8 @@ set -o nounset set -o errexit set -o pipefail -USAGE=$(cat << 'END_OF_LINE' +USAGE=$( + cat << 'END_OF_LINE' Configure a developement environment for this repository. It does the following: @@ -35,17 +36,17 @@ FORCE=0 PARAMS="" # Parse command line arguments -while (( "$#" )); do +while (("$#")); do case "$1" in - -f|--force) + -f | --force) FORCE=1 shift ;; - -h|--help) + -h | --help) echo "${USAGE}" exit 0 ;; - -i|--install-hooks) + -i | --install-hooks) INSTALL_HOOKS=1 shift ;; @@ -160,7 +161,8 @@ pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # This could fail if the remotes are already setup, but that is ok. set +o errexit -eval "$(python3 << 'END_OF_LINE' +eval "$( + python3 << 'END_OF_LINE' from pathlib import Path import yaml import sys From 4d37dfbaccb78f61c99be58bce79004808b3f295 Mon Sep 17 00:00:00 2001 From: dav3r Date: Thu, 15 Jul 2021 12:29:45 -0400 Subject: [PATCH 266/417] Use preferred capitalization for Docker Hub --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9afe401..b4e6c9d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,13 +79,13 @@ jobs: # The source version as reported by the `bump_version.sh show` command. # tags: # A comma separated list of Docker tags to be applied to the images on - # DockerHub. The tags will vary depending on: + # Docker Hub. The tags will vary depending on: # - The event that triggered the build. # - The branch the build is based upon. # - The git tag the build is based upon. # # When a build is based on a git tag of the form `v*.*.*` the image will - # be tagged on DockerHub with multiple levels of version specificity. + # be tagged on Docker Hub with multiple levels of version specificity. # For example, a git tag of `v1.2.3+a` will generate Docker tags of # `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`. # @@ -279,16 +279,16 @@ jobs: build-push-all: # Builds the final set of images for each of the platforms listed in # PLATFORMS environment variable. These images are tagged with the Docker - # tags calculated in the "prepare" job and pushed to DockerHub and the + # tags calculated in the "prepare" job and pushed to Docker Hub and the # GitHub Container Registry. The contents of README.md are pushed as the - # image's description to DockerHub. This job is skipped when the triggering + # image's description to Docker Hub. This job is skipped when the triggering # event is a pull request. name: "Build and push all platforms" runs-on: ubuntu-latest needs: [lint, prepare, test] if: github.event_name != 'pull_request' steps: - - name: Login to DockerHub + - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} From eede87c452fc0949962d504710feb834629badf7 Mon Sep 17 00:00:00 2001 From: dav3r Date: Thu, 15 Jul 2021 12:39:07 -0400 Subject: [PATCH 267/417] Satisfy linter by shortening a long line --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4e6c9d..f2cf5cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -281,8 +281,8 @@ jobs: # PLATFORMS environment variable. These images are tagged with the Docker # tags calculated in the "prepare" job and pushed to Docker Hub and the # GitHub Container Registry. The contents of README.md are pushed as the - # image's description to Docker Hub. This job is skipped when the triggering - # event is a pull request. + # image's description to Docker Hub. This job is skipped when the + # triggering event is a pull request. name: "Build and push all platforms" runs-on: ubuntu-latest needs: [lint, prepare, test] From 408969ac265d8197b942a67fae2d3cb200128f40 Mon Sep 17 00:00:00 2001 From: dav3r Date: Thu, 15 Jul 2021 14:24:51 -0400 Subject: [PATCH 268/417] Use preferred capitalization for Docker Hub --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 446c7ab..6e011be 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ environment variables. See the ### Docker Compose ### -1. Pull the new image from Docker hub: +1. Pull the new image from Docker Hub: ```console docker-compose pull From 51076ed3ddf39fc2304bfe9bdd49e85cb535b89e Mon Sep 17 00:00:00 2001 From: dav3r Date: Thu, 15 Jul 2021 14:30:21 -0400 Subject: [PATCH 269/417] Capitalize Docker in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6e011be..f92b3df 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,12 @@ [![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/cisagov/example)](https://hub.docker.com/r/cisagov/example) [![Platforms](https://img.shields.io/badge/platforms-amd64%20%7C%20arm%2Fv6%20%7C%20arm%2Fv7%20%7C%20arm64%20%7C%20ppc64le%20%7C%20s390x-blue)](https://hub.docker.com/r/cisagov/skeleton-docker/tags) -This is a docker skeleton project that can be used to quickly get a -new [cisagov](https://github.com/cisagov) GitHub docker project +This is a Docker skeleton project that can be used to quickly get a +new [cisagov](https://github.com/cisagov) GitHub Docker project started. This skeleton project contains [licensing information](LICENSE), as well as [pre-commit hooks](https://pre-commit.com) and [GitHub Actions](https://github.com/features/actions) configurations -appropriate for docker containers and the major languages that we use. +appropriate for Docker containers and the major languages that we use. ## Running ## From 440753c3eff55ede207f19d35018dea244552f4d Mon Sep 17 00:00:00 2001 From: David Redmin Date: Thu, 15 Jul 2021 14:36:24 -0400 Subject: [PATCH 270/417] Fix additional Docker and Docker Hub capitalizations --- push_readme.sh | 2 +- tests/conftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 12e2340..0d32ec8 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Push the README.md file to the docker hub repository +# Push the README.md file to the Docker Hub repository # Requires the following environment variables to be set: # DOCKER_PASSWORD, DOCKER_USERNAME, IMAGE_NAME diff --git a/tests/conftest.py b/tests/conftest.py index 28d6c42..90938e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,14 +11,14 @@ @pytest.fixture(scope="session") def main_container(dockerc): - """Return the main container from the docker composition.""" + """Return the main container from the Docker composition.""" # find the container by name even if it is stopped already return dockerc.containers(service_names=[MAIN_SERVICE_NAME], stopped=True)[0] @pytest.fixture(scope="session") def version_container(dockerc): - """Return the version container from the docker composition. + """Return the version container from the Docker composition. The version container should just output the version of its underlying contents. """ From 1708b5c45ac12f9acfb447e532c4fc5746a80d23 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Jul 2021 11:27:26 -0400 Subject: [PATCH 271/417] Update pre-commit hooks This is performed by running `pre-commit autoupdate`, but with the `ansible-lint` hook held back manually. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf0330d..cf319bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.3.0 + rev: v2.3.2 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint @@ -68,7 +68,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 21.5b2 + rev: 21.7b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -78,15 +78,15 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.8.0 + rev: 5.9.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.812 + rev: v0.910 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.19.1 + rev: v2.21.2 hooks: - id: pyupgrade @@ -123,7 +123,7 @@ repos: # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.0.0 + rev: v2.0.1 hooks: - id: docker-compose-check From 6ad71a80c77b942510b4c9a7dfafbd90b899171b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Jul 2021 16:46:56 -0400 Subject: [PATCH 272/417] Satisfy pre-commit hooks Update to reflect the new pre-commit configuration. --- bump_version.sh | 5 ++--- push_readme.sh | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index ec9313d..a6c8ed9 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -13,12 +13,11 @@ HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|final old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) -if [ $# -ne 1 ] -then +if [ $# -ne 1 ]; then echo "$HELP_INFORMATION" else case $1 in - major|minor|patch|prerelease|build) + major | minor | patch | prerelease | build) new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") echo Changing version from "$old_version" to "$new_version" tmp_file=/tmp/version.$$ diff --git a/push_readme.sh b/push_readme.sh index 12e2340..da6dc46 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -11,19 +11,19 @@ set -o pipefail echo "Logging in and requesting JWT..." token=$(curl --silent --request POST \ - --header "Content-Type: application/json" \ - --data \ - '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ + --header "Content-Type: application/json" \ + --data \ + '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq --raw-output .token) echo "Pushing README file..." -code=$(jq --null-input --arg msg "$( Date: Wed, 4 Aug 2021 09:30:18 -0400 Subject: [PATCH 273/417] Fix a typo --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5d7f673..f526cdb 100755 --- a/setup-env +++ b/setup-env @@ -6,7 +6,7 @@ set -o pipefail USAGE=$( cat << 'END_OF_LINE' -Configure a developement environment for this repository. +Configure a development environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. From 3ebf37da6ea14dd4bbdb607271d8faa9e1484dd2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 5 Aug 2021 11:02:01 -0400 Subject: [PATCH 274/417] Remove support for the linux/s390x platform Issue cisagov/skeleton-docker#69 has been created to remind us to re-add it once it is again supported by the official Python Docker image. --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2aa6319..ade2bdc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,8 +29,11 @@ env: CURL_CACHE_DIR: ~/.cache/curl IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip + # TODO: Add back in support for the linux/s390x platform once the + # official Python Docker image again supports it. See + # cisagov/skeleton-docker#69 for more details. PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ - linux/arm64,linux/ppc64le,linux/s390x" + linux/arm64,linux/ppc64le" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} From f7140d8116532d5d680e10109912fd2741c353ac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 9 Aug 2021 23:42:42 -0400 Subject: [PATCH 275/417] Use the hashicorp/setup-terraform Action Instead of manually installing a Terraform binary we can leverage the Action provided by Hashicorp to do the same thing. --- .github/workflows/build.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1160e61..6aa2aeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,17 +75,9 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - name: Install Terraform - run: | - TERRAFORM_ZIP="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" \ - --location \ - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_ZIP}" - sudo unzip -d /opt/terraform \ - ${{ env.CURL_CACHE_DIR }}/"${TERRAFORM_ZIP}" - sudo mv /usr/local/bin/terraform /usr/local/bin/terraform-default - sudo ln -s /opt/terraform/terraform /usr/local/bin/terraform + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} - name: Install shfmt run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} - name: Install Terraform-docs From 7f324b9807549b6b06acd3a43b3636743322e826 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 10 Aug 2021 11:36:33 -0400 Subject: [PATCH 276/417] Add a markdownlint rule for horizontal rules This sets a specific rule for MD035 (Horizontal rule style) instead of the default value of "consistent". --- .mdl_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index a2f08f3..f40ea77 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -38,3 +38,8 @@ MD033: allowed_elements: - h1 - img + +# MD035/hr-style - Horizontal rule style +MD035: + # Enforce dashes for horizontal rules + style: "---" From 9848ff69448b32b45e31013a4f8709963849659e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 10 Aug 2021 11:38:39 -0400 Subject: [PATCH 277/417] Add a markdownlint rule for code blocks This sets a specific rule for MD046 (code block style) instead of the default value of "consistent". --- .mdl_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index f40ea77..b36f943 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -43,3 +43,8 @@ MD033: MD035: # Enforce dashes for horizontal rules style: "---" + +# MD046/code-block-style Code block style +MD046: + # Enforce the fenced style for code blocks + style: "fenced" From 23f9cfd015d5b0de488823130ff0eee4bef2da67 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 9 Aug 2021 23:46:40 -0400 Subject: [PATCH 278/417] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back due to issues with switching to v5. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ed26ad..5f06453 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.27.1 + rev: v0.28.1 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.1 + rev: v1.26.2 hooks: - id: yamllint args: @@ -49,7 +49,7 @@ repos: # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.13.0 + rev: v2.14.0 hooks: - id: validate_manifest @@ -91,7 +91,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.9.2 + rev: 5.9.3 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy @@ -99,7 +99,7 @@ repos: hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.21.2 + rev: v2.23.3 hooks: - id: pyupgrade From a3825d8f6295049eacf1ba7f98009e896dc9019d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 23 Aug 2021 11:26:52 -0400 Subject: [PATCH 279/417] Revert "Remove support for the linux/s390x platform" This reverts commit 3ebf37da6ea14dd4bbdb607271d8faa9e1484dd2. There was finally a successful build of the official Python Docker image upstream, so we can add this back in. Resolves cisagov/skeleton-docker#69. --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ade2bdc..2aa6319 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,11 +29,8 @@ env: CURL_CACHE_DIR: ~/.cache/curl IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip - # TODO: Add back in support for the linux/s390x platform once the - # official Python Docker image again supports it. See - # cisagov/skeleton-docker#69 for more details. PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ - linux/arm64,linux/ppc64le" + linux/arm64,linux/ppc64le,linux/s390x" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} From 60b612ded599d0a24b99c9aa24e33a1f7ba8f598 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Thu, 26 Aug 2021 09:56:34 -0400 Subject: [PATCH 280/417] Enable terraform validate pre-commit hook This hook should now work as expected in most, if not all of our repos now that we are finally updating to terraform 0.13.x (on our way to 1.0.x). --- .pre-commit-config.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f06453..da27f36 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -117,22 +117,7 @@ repos: rev: v1.50.0 hooks: - id: terraform_fmt - # There are ongoing issues with how this command works. This issue - # documents the core issue: - # https://github.com/hashicorp/terraform/issues/21408 - # We have seen issues primarily with proxy providers and Terraform code - # that uses remote state. The PR - # https://github.com/hashicorp/terraform/pull/24887 - # has been approved and is part of the 0.13 release to resolve the issue - # with remote states. - # The PR - # https://github.com/hashicorp/terraform/pull/24896 - # is a proprosed fix to deal with `terraform validate` with proxy - # providers (among other configurations). - # We have decided to disable the terraform_validate hook until the issues - # above have been resolved, which we hope will be with the release of - # Terraform 0.13. - # - id: terraform_validate + - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit From 6a7fbf07bd371d0493c523ce24647e5c04c77c03 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Thu, 26 Aug 2021 15:33:59 -0400 Subject: [PATCH 281/417] Temporarily use cisagov/setup-env-github-action@improvement/support_tf_0.13 This change will be reverted when testing is completed. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6aa2aeb..b2b73fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: cisagov/setup-env-github-action@develop + - uses: cisagov/setup-env-github-action@improvement/support_tf_0.13 - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 From 895a692ad8dfc3df50b1c2c9a79bca7166b5f5f2 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Fri, 27 Aug 2021 13:02:00 -0400 Subject: [PATCH 282/417] Remove lint job step to initialize Terraform directories Initialization will now be done during the "terraform validate" step. --- .github/workflows/build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2b73fd..f276a36 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,13 +84,6 @@ jobs: run: | go install \ github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} - - name: Find and initialize Terraform directories - run: | - for path in $(find . -not \( -type d -name ".terraform" -prune \) \ - -type f -iname "*.tf" -exec dirname "{}" \; | sort -u); do \ - echo "Initializing '$path'..."; \ - terraform init -input=false -backend=false "$path"; \ - done - name: Install dependencies run: | python -m pip install --upgrade pip From b51dbb577e02baff361a6494e22f61aa517e28d4 Mon Sep 17 00:00:00 2001 From: David Redmin Date: Fri, 27 Aug 2021 14:59:29 -0400 Subject: [PATCH 283/417] Revert "Temporarily use cisagov/setup-env-github-action@improvement/support_tf_0.13" This reverts commit 6a7fbf07bd371d0493c523ce24647e5c04c77c03. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f276a36..090f039 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: cisagov/setup-env-github-action@improvement/support_tf_0.13 + - uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 From a52dbb0f0511e6d0249c6370b56928757297a460 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 17:01:43 +0000 Subject: [PATCH 284/417] Bump actions/github-script from 4 to 5 Bumps [actions/github-script](https://github.com/actions/github-script) from 4 to 5. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0c09c6..44fe6cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,7 +178,7 @@ jobs: - uses: actions/checkout@v2 - name: Gather repository metadata id: repo - uses: actions/github-script@v4 + uses: actions/github-script@v5 with: script: | const repo = await github.repos.get(context.repo) From aff2b1376c36a93da9deb6b034e21c9154d837d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 17:01:29 +0000 Subject: [PATCH 285/417] Bump python from 3.9.6-alpine to 3.10.0-alpine Bumps python from 3.9.6-alpine to 3.10.0-alpine. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 59ea4cd..cf1a5e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG VERSION=unspecified -FROM python:3.9.6-alpine +FROM python:3.10.0-alpine ARG VERSION From fc28df162b4264d1a59bed74a6f5ba175dadde57 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 17 Oct 2021 16:21:33 -0400 Subject: [PATCH 286/417] Update for actions/github-script API changes Update code to reflect breaking changes in V5 of cisagov/github-script as mentioned here: https://github.com/actions/github-script#breaking-changes-in-v5 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 44fe6cd..2acfc0c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -181,7 +181,7 @@ jobs: uses: actions/github-script@v5 with: script: | - const repo = await github.repos.get(context.repo) + const repo = await github.rest.repos.get(context.repo) return repo.data - name: Calculate output values id: prep From b66988de8c7547e456d570f718244b54a39e79e8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 29 Sep 2021 13:34:39 -0400 Subject: [PATCH 287/417] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back due to issues with switching to v5. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da27f36..3171404 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.28.1 + rev: v0.29.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.3.2 + rev: v2.4.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.2 + rev: v1.26.3 hooks: - id: yamllint args: @@ -49,7 +49,7 @@ repos: # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.14.0 + rev: v2.15.0 hooks: - id: validate_manifest @@ -81,7 +81,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 21.7b0 + rev: 21.9b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -95,11 +95,11 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.910 + rev: v0.910-1 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.23.3 + rev: v2.29.0 hooks: - id: pyupgrade @@ -114,7 +114,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.50.0 + rev: v1.52.0 hooks: - id: terraform_fmt - id: terraform_validate From b78b83226904126138ad1bdc1fc37e44ed7c2d41 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:01:56 -0400 Subject: [PATCH 288/417] Update the ansible-lint version for pre-commit --- .pre-commit-config.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3171404..d83fdbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -105,9 +105,7 @@ repos: # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - # This is intentionally being held back because of issues in v5 per - # https://github.com/cisagov/skeleton-ansible-role/issues/69 - rev: v4.3.7 + rev: v5.2.1 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From a3c5aa71cf2e8497bb0581b09bbf2978b6a96fc6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:05:45 -0400 Subject: [PATCH 289/417] Add an ansible-lint configuration file This file is copied from cisagov/skeleton-ansible-role#85 and tweaked to pass pre-commit. See here for for a list of the elements that can exist in this file: https://ansible-lint.readthedocs.io/en/latest/configuring.html Co-authored-by: Shane Frasier --- .ansible-lint | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .ansible-lint diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..bc8e0e5 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,22 @@ +--- +# See https://ansible-lint.readthedocs.io/en/latest/configuring.html +# for a list of the configuration elements that can exist in this +# file. +enable_list: + # Useful checks that one must opt-into. See here for more details: + # https://ansible-lint.readthedocs.io/en/latest/rules.html + - fcqn-builtins + - no-log-password + - no-same-owner +exclude_paths: + # This exclusion is implicit, unless exclude_paths is defined + - .cache + # Seems wise to ignore this too + - .github + # ansible-lint doesn't like the role name in this playbook, but it's + # what molecule requires + - molecule/default/converge.yml + # These two are Molecule configuration files, not Ansible playbooks + - molecule/default/molecule-no-systemd.yml + - molecule/default/molecule-with-systemd.yml +use_default_rules: true From 24df40abd141eb93c056df90a7809fd8573d5328 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 Oct 2021 12:19:14 -0400 Subject: [PATCH 290/417] Add dependabot configuration for Terraform This will configure `dependabot` to scan Terraform configurations if they exist in a repository. If a repository stores a Terraform configuration in a sub- directory this configuration will need to be modified or an additional configuration must be added if there still exists a Terraform configuration in the root directory. --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1fd2ed3..a3bcd94 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,8 @@ updates: directory: "/" schedule: interval: "weekly" + + - package-ecosystem: "terraform" + directory: "/" + schedule: + interval: "weekly" From 6cf78c02e11f78ebc35a151a38f4280bb0f8e53c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Sep 2021 01:00:47 -0400 Subject: [PATCH 291/417] Use an id when using cisagov/setup-env-github-action --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 090f039..18a327f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: cisagov/setup-env-github-action@develop + - id: setup-env + uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python uses: actions/setup-python@v2 From 1e16136272bfd8d55dcb554c4c1ee059223fafe7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Sep 2021 11:33:00 -0400 Subject: [PATCH 292/417] Use setup-env outputs for Terraform version Change from using an environment variable to the outputs of the cisagov/setup-env-github-action instead. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18a327f..eb0f504 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: py${{ steps.setup-python.outputs.python-version }}-\ go${{ env.GO_VERSION }}-\ packer${{ env.PACKER_VERSION }}-\ - tf${{ env.TERRAFORM_VERSION }}-" + tf${{ steps.setup-env.outputs.terraform-version }}-" with: # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use @@ -78,7 +78,7 @@ jobs: sudo ln -s /opt/packer/packer /usr/local/bin/packer - uses: hashicorp/setup-terraform@v1 with: - terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install shfmt run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} - name: Install Terraform-docs From 64b24714ac3f06a2fc2f5d9096eca29ecac41e47 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:03:19 -0400 Subject: [PATCH 293/417] Use setup-env outputs for Packer version We use a `PACKER_VERSION` environment variable for the "Install Packer" step that is populated from the `setup-env` outputs to get around `yamllint` lint length limits. --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb0f504..e6ef7ce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ go${{ env.GO_VERSION }}-\ - packer${{ env.PACKER_VERSION }}-\ + packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: # Note that the .terraform directory IS NOT included in the @@ -66,6 +66,8 @@ jobs: - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} - name: Install Packer + env: + PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} run: | PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ From 0851598edda3bf21d0976b4c2fa615e15cbec8e4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:08:15 -0400 Subject: [PATCH 294/417] Use setup-env outputs for shfmt version We change the "Install shfmt" step to use two local environment variables to provide the package's URL and version to install. This allows us to work around `yamllint` line length limits. --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6ef7ce..f0f39b2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -82,7 +82,10 @@ jobs: with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install shfmt - run: go install mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION} + env: + PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt + PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Terraform-docs run: | go install \ From 2699e09eb7e81ccd73d5f668345ad711dc162dc5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:11:11 -0400 Subject: [PATCH 295/417] Use setup-env outputs for Terraform-docs version We change the "Install Terraform-docs" step to use two local environment variables to provide the package's URL and version to install. This allows us to work around `yamllint` line length limits. --- .github/workflows/build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f0f39b2..c7141c1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,9 +87,10 @@ jobs: PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Terraform-docs - run: | - go install \ - github.com/terraform-docs/terraform-docs@${TERRAFORM_DOCS_VERSION} + env: + PACKAGE_URL: github.com/terraform-docs/terraform-docs + PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install dependencies run: | python -m pip install --upgrade pip From 8797e110d3c9cefbfe7e72a80496164bcdd7620a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:49:45 -0400 Subject: [PATCH 296/417] Store Go version as a step output This changes from using an environment variable to using a step output to store the Go version that is installed. This mirrors changes made to the other program versions and how they're stored. --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7141c1..7be2ce8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,16 +24,16 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - # GO_VERSION and GOCACHE are used by the cache task, so the Go - # installation must happen before that. + # We need the Go version and Go cache location for the actions/cache step, + # so the Go installation must happen before that. - uses: actions/setup-go@v2 with: go-version: '1.16' - name: Store installed Go version + id: go-version run: | - echo "GO_VERSION="\ - "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" \ - >> $GITHUB_ENV + echo "::set-output name=version::"\ + "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" - name: Lookup Go cache directory id: go-cache run: | @@ -42,7 +42,7 @@ jobs: env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ - go${{ env.GO_VERSION }}-\ + go${{ steps.go-version.outputs.version }}-\ packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: From 6a10135f8155b0128a0019d451bc9673a0c463ee Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 27 Oct 2021 21:50:51 -0400 Subject: [PATCH 297/417] Update ansible-lint configuration Remove the manual exclusion of the `converge.yml` file in the default molecule scenario. With the changes in cisagov/skeleton-ansible-role#88 this exclusion is no longer necessary. --- .ansible-lint | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index bc8e0e5..08c1ae7 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -13,9 +13,6 @@ exclude_paths: - .cache # Seems wise to ignore this too - .github - # ansible-lint doesn't like the role name in this playbook, but it's - # what molecule requires - - molecule/default/converge.yml # These two are Molecule configuration files, not Ansible playbooks - molecule/default/molecule-no-systemd.yml - molecule/default/molecule-with-systemd.yml From 4b912fe7caf317793606afc626d19a4b840d4f2e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 27 Oct 2021 21:53:10 -0400 Subject: [PATCH 298/417] Update ansible-lint configuration Instead of excluding molecule configurations for systemd individually we instead direct ansible-lint to process any of these configurations in scenario directories as plain yaml files. This mirrors how ansible-lint handles `molecule.yml` files in scenario directories by default. --- .ansible-lint | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index 08c1ae7..0e80b05 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -13,7 +13,10 @@ exclude_paths: - .cache # Seems wise to ignore this too - .github - # These two are Molecule configuration files, not Ansible playbooks - - molecule/default/molecule-no-systemd.yml - - molecule/default/molecule-with-systemd.yml +kinds: + # This will force our systemd specific molecule configurations to be treated + # as plain yaml files by ansible-lint. This mirrors the default kind + # configuration in ansible-lint for molecule configurations: + # yaml: "**/molecule/*/{base,molecule}.{yaml,yml}" + - yaml: "**/molecule/*/molecule-{no,with}-systemd.yml" use_default_rules: true From 18c99206e3c97c3f86639df0052a97165ecabcdc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 17:01:27 +0000 Subject: [PATCH 299/417] Bump python from 3.10.0-alpine to 3.10.1-alpine Bumps python from 3.10.0-alpine to 3.10.1-alpine. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cf1a5e9..248d6cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG VERSION=unspecified -FROM python:3.10.0-alpine +FROM python:3.10.1-alpine ARG VERSION From 3f370ebddeb80461a609e0ac1c51e4c1dfb04652 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:22:15 -0500 Subject: [PATCH 300/417] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d83fdbc..b8dcbe0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.1.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,13 +31,13 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.29.0 + rev: v0.30.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.4.1 + rev: v2.5.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint @@ -49,7 +49,7 @@ repos: # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.15.0 + rev: v2.16.0 hooks: - id: validate_manifest @@ -75,13 +75,13 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.0 + rev: 1.7.1 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 21.9b0 + rev: 21.12b0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -91,28 +91,28 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.9.3 + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.910-1 + rev: v0.931 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.29.0 + rev: v2.31.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.2.1 + rev: v5.3.2 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.52.0 + rev: v1.62.3 hooks: - id: terraform_fmt - id: terraform_validate From ad71ef369b933e709e051873c50298e2ddaf2345 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 23 Feb 2022 20:11:49 -0500 Subject: [PATCH 301/417] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b8dcbe0..e6dc7b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.30.0 + rev: v0.31.1 hooks: - id: markdownlint args: @@ -49,7 +49,7 @@ repos: # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.16.0 + rev: v2.17.0 hooks: - id: validate_manifest @@ -75,13 +75,13 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.1 + rev: 1.7.2 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 21.12b0 + rev: 22.1.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -105,14 +105,14 @@ repos: # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.3.2 + rev: v5.4.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.62.3 + rev: v1.64.0 hooks: - id: terraform_fmt - id: terraform_validate From a05b45adf9d817ee50357470363e212c4b6f81a0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 25 Feb 2022 08:26:08 -0500 Subject: [PATCH 302/417] Add comment to Dependabot configuration This comment explains that the configuration may have commented out ignore directives that should be uncommented in downstream projects. --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a3bcd94..5792ed9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,10 @@ --- +# Any ignore directives should be uncommented in downstream projects to disable +# Dependabot updates for the given dependency. Downstream projects will get +# these updates when the pull request(s) in the appropriate skeleton are merged +# and Lineage processes these changes. + version: 2 updates: - package-ecosystem: "github-actions" From 77b20ab3d7822b66715db168240b25e72d1c4ed0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 25 Feb 2022 08:27:32 -0500 Subject: [PATCH 303/417] Disable comments-indentation rule for yamllint yamllint does not like it when you comment out pieces of dictionaries in lists. Upcoming additions to the Dependabot configuration will run afoul of this so we are updating the yamllint configuration. --- .yamllint | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.yamllint b/.yamllint index 7ed00eb..76a1cce 100644 --- a/.yamllint +++ b/.yamllint @@ -2,6 +2,12 @@ extends: default rules: + # yamllint does not like it when you comment out different parts of + # dictionaries in a list. You can see + # https://github.com/adrienverge/yamllint/issues/384 for some examples of + # this behavior. + comments-indentation: disable + # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable From 706bdeda5a2adbf27919a217603aa127d63975f8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 25 Feb 2022 11:21:37 -0500 Subject: [PATCH 304/417] Use asterisks for emphasis- and strong-styles Given the inconsistent behavior of underscores used within words for style we should prefer asterisks now that these rules are available. --- .mdl_config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index b36f943..15a1a7c 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -48,3 +48,13 @@ MD035: MD046: # Enforce the fenced style for code blocks style: "fenced" + +# MD049/emphasis-style - Emphasis style should be consistent +MD049: + # Enforce asterisks as the style to use for emphasis + style: "asterisk" + +# MD050/strong-style - Strong style should be consistent +MD050: + # Enforce asterisks as the style to use for strong + style: "asterisk" From 7868a35e1097d291a73edb22fae3aa9017ec3768 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 25 Feb 2022 11:23:25 -0500 Subject: [PATCH 305/417] Add missing hyphen in markdownlint rule header --- .mdl_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mdl_config.yaml b/.mdl_config.yaml index 15a1a7c..4a650c1 100644 --- a/.mdl_config.yaml +++ b/.mdl_config.yaml @@ -44,7 +44,7 @@ MD035: # Enforce dashes for horizontal rules style: "---" -# MD046/code-block-style Code block style +# MD046/code-block-style - Code block style MD046: # Enforce the fenced style for code blocks style: "fenced" From 4ee1bfd96b83aa1d8fda363d0a8e93db9acc395a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 18:02:16 +0000 Subject: [PATCH 306/417] Bump actions/setup-python from 2 to 3 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 3. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7be2ce8..f15dacc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v2 - id: setup-python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 # We need the Go version and Go cache location for the actions/cache step, From 3406c2d4207140951e6442f797d699e0336da2e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 18:01:37 +0000 Subject: [PATCH 307/417] Bump actions/checkout from 2 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7be2ce8..a6e092e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: steps: - id: setup-env uses: cisagov/setup-env-github-action@develop - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - id: setup-python uses: actions/setup-python@v2 with: From 9a0380870ca93c2417956c1c959d9286588f4fec Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Mar 2022 16:19:18 -0400 Subject: [PATCH 308/417] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back due to issues with upgrading to v6. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e6dc7b7..d991216 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.5.1 + rev: v2.6.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint @@ -75,13 +75,13 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.2 + rev: 1.7.4 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 22.1.0 + rev: 22.3.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 @@ -95,11 +95,11 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.931 + rev: v0.942 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 + rev: v2.31.1 hooks: - id: pyupgrade @@ -119,7 +119,7 @@ repos: # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.0.1 + rev: v2.1.0 hooks: - id: docker-compose-check From 511a37cf1ecc9283f7a6793322067aa712d1abc4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Mar 2022 16:26:36 -0400 Subject: [PATCH 309/417] Add Dependabot ignore directives This adds commented out ignore directives for the following GitHub Actions: - action/cache - action/checkout - action/setup-python These should be uncommented downstream to ensure that updates to these dependencies are pushed from pull requests made in the skeleton. --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5792ed9..208659c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,6 +11,10 @@ updates: directory: "/" schedule: interval: "weekly" + # ignore: + # - dependency-name: actions/cache + # - dependency-name: actions/checkout + # - dependency-name: actions/setup-python - package-ecosystem: "pip" directory: "/" From 583992602c3815cd5c1b8338f12df83b1721883b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Mar 2022 16:29:30 -0400 Subject: [PATCH 310/417] Upgrade from Python 3.9 to 3.10 for the `lint` job in GHA --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7be2ce8..56de702 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: - id: setup-python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: "3.10" # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - uses: actions/setup-go@v2 From e22c12cbd467cac3e43298c67380b244da936782 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Mar 2022 16:35:59 -0400 Subject: [PATCH 311/417] Use consistent quoting for software versions --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 56de702..917af8a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: # so the Go installation must happen before that. - uses: actions/setup-go@v2 with: - go-version: '1.16' + go-version: "1.16" - name: Store installed Go version id: go-version run: | From c576ef56fbe70907da23a464b29eb243a6d5361b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Apr 2022 16:22:39 +0000 Subject: [PATCH 312/417] Bump actions/cache from 2 to 3 Bumps [actions/cache](https://github.com/actions/cache) from 2 to 3. - [Release notes](https://github.com/actions/cache/releases) - [Commits](https://github.com/actions/cache/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 264d488..ef95f3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: id: go-cache run: | echo "::set-output name=dir::$(go env GOCACHE)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ From 49d87d40e4f2bb6bfbba595c6a643e58b3ceccf9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Apr 2022 16:46:16 -0400 Subject: [PATCH 313/417] Add pre-commit hooks to validate GitHub Actions content Add hooks to validate GitHub Actions and GitHub Action workflows. These are validated using JSON Schemas. --- .pre-commit-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d991216..c5c2e10 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,13 @@ repos: args: - --strict + # GitHub Actions hooks + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.14.2 + hooks: + - id: check-github-actions + - id: check-github-workflows + # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit rev: v2.17.0 From 57b4abe8fd21bfd03d1010f82ff20d33aea353c4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 18:04:15 -0400 Subject: [PATCH 314/417] Update actions/setup-go from v2 to v3 Additionally as of v3.1.0 of actions/setup-go there is a go-version output value to retrieve the version of Go installed by the Action. This allows us to remove the step to manually retrieve this information from the Go executable. --- .github/workflows/build.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 27b2dcf..3216eec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,14 +26,10 @@ jobs: python-version: "3.10" # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - - uses: actions/setup-go@v2 + - id: setup-go + uses: actions/setup-go@v3 with: go-version: "1.16" - - name: Store installed Go version - id: go-version - run: | - echo "::set-output name=version::"\ - "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" - name: Lookup Go cache directory id: go-cache run: | @@ -42,7 +38,7 @@ jobs: env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ - go${{ steps.go-version.outputs.version }}-\ + go${{ steps.setup-go.outputs.go-version }}-\ packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: From 8f6849857bc2a66254cafa94db31526692fab209 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 18:07:10 -0400 Subject: [PATCH 315/417] Update the version of Go used from 1.16 to 1.18 Go 1.16 is no longer supported as of the release of 1.18 so it makes sense to update to the latest version available. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3216eec..d4c50c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: - id: setup-go uses: actions/setup-go@v3 with: - go-version: "1.16" + go-version: "1.18" - name: Lookup Go cache directory id: go-cache run: | From 72137712088be94f5ceddaba798d7902b93f2bf8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 29 May 2022 17:38:33 -0400 Subject: [PATCH 316/417] Enable Dependabot ignore directives Uncomment the `github-actions` Dependabot ignore directives because those dependencies are managed by cisagov/skeleton-generic upstream. --- .github/dependabot.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index af755e2..4d0dc17 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,10 +16,10 @@ updates: directory: "/" schedule: interval: "weekly" - # ignore: - # - dependency-name: actions/cache - # - dependency-name: actions/checkout - # - dependency-name: actions/setup-python + ignore: + - dependency-name: actions/cache + - dependency-name: actions/checkout + - dependency-name: actions/setup-python - package-ecosystem: "pip" directory: "/" From 534fb667ee07571a99fcf68dde8f8b196ee77988 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 29 May 2022 17:42:09 -0400 Subject: [PATCH 317/417] Fix issues with the `build.yml` file We must ensure that all `workflow_dispatch` input defaults are string values. This ensures expected behavior instead of relying on automatic type conversions. --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e8f5bdd..4642d49 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,12 +4,12 @@ name: build on: push: branches: - - '**' + - "**" tags: - - 'v*.*.*' + - "v*.*.*" pull_request: schedule: - - cron: '0 10 * * *' # everyday at 10am + - cron: "0 10 * * *" # everyday at 10am repository_dispatch: # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ types: [apb] @@ -18,11 +18,11 @@ on: remote-shell: description: "Debug with remote shell" required: true - default: false + default: "false" image-tag: description: "Tag to apply to pushed images" required: true - default: dispatch + default: "dispatch" env: BUILDX_CACHE_DIR: ~/.cache/buildx From d61a7cb7e3561314e6047a3059d16d5c3b609ddb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 May 2022 18:27:24 -0400 Subject: [PATCH 318/417] Add Dependabot dependency ownership for this repository This includes adding the versioned GitHub Actions that should be ignored in downstream projects and a comment stating where these dependencies are managed. --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4d0dc17..790a302 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,6 +20,14 @@ updates: - dependency-name: actions/cache - dependency-name: actions/checkout - dependency-name: actions/setup-python + # Managed by cisagov/skeleton-docker + # - dependency-name: actions/download-artifact + # - dependency-name: actions/github-script + # - dependency-name: actions/upload-artifact + # - dependency-name: docker/build-push-action + # - dependency-name: docker/login-action + # - dependency-name: docker/setup-buildx-action + # - dependency-name: docker/setup-qemu-action - package-ecosystem: "pip" directory: "/" From 499d1d5cebded88662944371a6c4a70fdddf7399 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 7 Jun 2022 15:38:59 -0400 Subject: [PATCH 319/417] Update code to use the "docker compose" syntax vice "docker-compose" The "docker compose" syntax is the preferred (and only correct) syntax after the changes in cisagov/ansible-role-docker#60. --- README.md | 10 +++++----- docker-compose.yml | 4 ++-- tests/container_test.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f92b3df..57f8c30 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ docker run cisagov/example:0.0.1 source: target: /var/log environment: - - ECHO_MESSAGE="Hello from docker-compose" + - ECHO_MESSAGE="Hello from docker compose" ports: - target: 8080 published: 8080 @@ -53,7 +53,7 @@ docker run cisagov/example:0.0.1 1. Start the container and detach: ```console - docker-compose up --detach + docker compose up --detach ``` ## Using secrets with your container ## @@ -88,7 +88,7 @@ environment variables. See the source: target: /var/log environment: - - ECHO_MESSAGE="Hello from docker-compose" + - ECHO_MESSAGE="Hello from docker compose" ports: - target: 8080 published: 8080 @@ -105,13 +105,13 @@ environment variables. See the 1. Pull the new image from Docker Hub: ```console - docker-compose pull + docker compose pull ``` 1. Recreate the running container by following the [previous instructions](#running-with-docker-compose): ```console - docker-compose up --detach + docker compose up --detach ``` ### Docker ### diff --git a/docker-compose.yml b/docker-compose.yml index 0fc7ee5..79b879d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ --- version: "3.7" -# This docker-compose file is used to build and test the container +# This docker compose file is used to build and test the container secrets: quote_txt: @@ -19,7 +19,7 @@ services: init: true restart: "no" environment: - - ECHO_MESSAGE=Hello World from docker-compose! + - ECHO_MESSAGE=Hello World from docker compose! ports: - target: 8080 published: 8080 diff --git a/tests/container_test.py b/tests/container_test.py index 6153028..b92ff86 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -9,7 +9,7 @@ import pytest ENV_VAR = "ECHO_MESSAGE" -ENV_VAR_VAL = "Hello World from docker-compose!" +ENV_VAR_VAL = "Hello World from docker compose!" READY_MESSAGE = "This is a debug message" SECRET_QUOTE = ( "There are no secrets better kept than the secrets everybody guesses." # nosec From 91ae75fc83b23a1ae435831efe5ac96f4873a694 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 May 2022 18:07:55 -0400 Subject: [PATCH 320/417] Bump docker/build-push-action from 2 to 3 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4642d49..5152603 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -252,7 +252,7 @@ jobs: run: mkdir -p dist - name: Build image id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} @@ -380,7 +380,7 @@ jobs: run: ./buildx-dockerfile.sh - name: Build and push platform images to registries id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} From 38ed0d47b00ee845fd55836b6723d0818ad8f1d6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 May 2022 18:13:16 -0400 Subject: [PATCH 321/417] Bump docker/login-action from 1 to 2 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5152603..a0ffa5f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -351,12 +351,12 @@ jobs: if: github.event_name != 'pull_request' steps: - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} From 2f93d69e2d2e392de47d2d7f9f18b6f615d8e565 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 May 2022 18:15:00 -0400 Subject: [PATCH 322/417] Bump docker/setup-buildx-action from 1 to 2 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0ffa5f..c17501f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -238,7 +238,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 env: @@ -366,7 +366,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Cache Docker layers uses: actions/cache@v3 env: From 1156d9642bf37fee05f89798d14387350183b98d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 May 2022 18:15:57 -0400 Subject: [PATCH 323/417] Bump docker/setup-qemu-action from 1 to 2 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c17501f..30f79de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -236,7 +236,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers @@ -364,7 +364,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers From 0aa0b2d6fb3f080124c829141f31e5444305af29 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 May 2022 18:18:02 -0400 Subject: [PATCH 324/417] Bump actions/download-artifact from 2 to 3 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30f79de..f965156 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -325,7 +325,7 @@ jobs: python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: dist path: dist From 3fd728de9bdcc92b769f188883bd846dafc858ec Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 May 2022 18:18:45 -0400 Subject: [PATCH 325/417] Bump actions/upload-artifact from 2 to 3 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f965156..cc56c8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -290,7 +290,7 @@ jobs: - name: Compress image run: gzip dist/image.tar - name: Upload artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: dist path: dist From cfb36d1376e2813f5d219a6aef2cd4036959c733 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 13 Jun 2022 11:05:46 -0400 Subject: [PATCH 326/417] Improve language in a comment Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 79b879d..e746ae9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ --- version: "3.7" -# This docker compose file is used to build and test the container +# This Docker composition file is used to build and test the container secrets: quote_txt: From cd366fd9a141a525bb5858d55cda8faafd47aa2f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 25 Aug 2022 17:50:08 -0400 Subject: [PATCH 327/417] Enable Dependabot ignore directives Enable the new ignore directives for dependencies managed by cisagov/skeleton-docker. --- .github/dependabot.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 790a302..df7b865 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,13 +21,13 @@ updates: - dependency-name: actions/checkout - dependency-name: actions/setup-python # Managed by cisagov/skeleton-docker - # - dependency-name: actions/download-artifact - # - dependency-name: actions/github-script - # - dependency-name: actions/upload-artifact - # - dependency-name: docker/build-push-action - # - dependency-name: docker/login-action - # - dependency-name: docker/setup-buildx-action - # - dependency-name: docker/setup-qemu-action + - dependency-name: actions/download-artifact + - dependency-name: actions/github-script + - dependency-name: actions/upload-artifact + - dependency-name: docker/build-push-action + - dependency-name: docker/login-action + - dependency-name: docker/setup-buildx-action + - dependency-name: docker/setup-qemu-action - package-ecosystem: "pip" directory: "/" From 0589d71e1e5c5aaafffeab52b2cd7bd545f37002 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 25 Aug 2022 17:55:51 -0400 Subject: [PATCH 328/417] Update pre-commit configuration Add a dependency for the mypy hook to successfully analyze the Python code used in this Docker image. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6cc81b..f10d504 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -115,6 +115,8 @@ repos: rev: v0.942 hooks: - id: mypy + additional_dependencies: + - types-PyYAML - repo: https://github.com/asottile/pyupgrade rev: v2.31.1 hooks: From d5c44e2b486069d3601e2761ec152da5d463075e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 25 Aug 2022 17:58:15 -0400 Subject: [PATCH 329/417] Resolve pre-commit check failures Resolve the errors reported when running `pre-commit run --all-files`. --- docker-compose.override.yml | 1 + src/email-update.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 7e92489..a21ed3e 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,3 +1,4 @@ +--- version: '3.2' services: diff --git a/src/email-update.py b/src/email-update.py index 0f5c286..bd422fc 100755 --- a/src/email-update.py +++ b/src/email-update.py @@ -20,6 +20,7 @@ """ +# Standard Python Libraries import csv from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart @@ -28,6 +29,7 @@ import io import logging +# Third-Party Libraries import boto3 import docopt from mongo_db_from_config import db_from_config From 8f873d35334c9f46607eb388c187cbdfb66f195c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 25 Aug 2022 18:06:13 -0400 Subject: [PATCH 330/417] Add example Docker secrets files Add examples of the two secrets file that are used in this configuration. --- src/secrets/aws_config | 4 ++++ src/secrets/scan_read_creds.yml | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 src/secrets/aws_config create mode 100644 src/secrets/scan_read_creds.yml diff --git a/src/secrets/aws_config b/src/secrets/aws_config new file mode 100644 index 0000000..8d6c84f --- /dev/null +++ b/src/secrets/aws_config @@ -0,0 +1,4 @@ +[default] +credential_source = Ec2InstanceMetadata +region = us-east-1 +role_arn = arn:aws:iam::123456789012:role/AssumeSesSendEmail-CodeGovUpdate diff --git a/src/secrets/scan_read_creds.yml b/src/secrets/scan_read_creds.yml new file mode 100644 index 0000000..0e068ad --- /dev/null +++ b/src/secrets/scan_read_creds.yml @@ -0,0 +1,4 @@ +--- +database: + uri: mongodb://username:password@localhost:27017/auth-db-name + name: db-name From eaaae3383953b1891afed755e683e18e083ffa17 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 26 Aug 2022 11:54:31 -0400 Subject: [PATCH 331/417] Use long options in the Dockerfile We prefer to use long options for clarity whenever possible. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 94c9bb0..e914c8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,16 +27,16 @@ RUN pip install --upgrade pip setuptools # Install client-cert-update python requirements ## COPY src/requirements.txt /tmp -RUN pip install -r /tmp/requirements.txt +RUN pip install --requirement /tmp/requirements.txt # Clean up aptitude cruft -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get clean && rm --recursive --force /var/lib/apt/lists/* # Put this just before we change users because the copy (and every # step after it) will often be rerun by docker, but we need to be root # for the chown command. COPY src/email-update.py src/body.txt src/body.html $CISA_HOME/ -RUN chown -R ${CISA_USER}:${CISA_USER} $CISA_HOME +RUN chown --recursive ${CISA_USER}:${CISA_USER} $CISA_HOME ### # Prepare to Run From b3d7f6c227cb115c8a50c5778eac96646a06ebc8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 26 Aug 2022 11:55:56 -0400 Subject: [PATCH 332/417] Do not cache pip packages You should generally avoid caching packages (system, pip, etc.) in a Docker image so we avoid that for pip here. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e914c8f..d3ba05a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,13 +21,13 @@ RUN useradd --system --uid ${CISA_UID} --gid ${CISA_GROUP} --comment "${CISA_USE ## # Make sure pip and setuptools are the latest versions ## -RUN pip install --upgrade pip setuptools +RUN pip install --no-cache-dir --upgrade pip setuptools ## # Install client-cert-update python requirements ## COPY src/requirements.txt /tmp -RUN pip install --requirement /tmp/requirements.txt +RUN pip install --no-cache-dir --requirement /tmp/requirements.txt # Clean up aptitude cruft RUN apt-get clean && rm --recursive --force /var/lib/apt/lists/* From 4484ae4aef0dde5ad3c140cb7d80d680cab9bcc8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 26 Aug 2022 11:58:36 -0400 Subject: [PATCH 333/417] Remove unnecessary Docker RUN command Since we are not installing anything with Aptitude we do not need to clean up any of its files. --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3ba05a..20c0fc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,9 +29,6 @@ RUN pip install --no-cache-dir --upgrade pip setuptools COPY src/requirements.txt /tmp RUN pip install --no-cache-dir --requirement /tmp/requirements.txt -# Clean up aptitude cruft -RUN apt-get clean && rm --recursive --force /var/lib/apt/lists/* - # Put this just before we change users because the copy (and every # step after it) will often be rerun by docker, but we need to be root # for the chown command. From 8f40494007799b751256ee7c8a08fe447a3e03c8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 26 Aug 2022 12:00:08 -0400 Subject: [PATCH 334/417] Add the `wheel` package Our standard list of Python packages to ensure are installed and up-to- date include the `wheel` package so it is added. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 20c0fc3..67eb1c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,9 @@ RUN groupadd --system --gid ${CISA_GID} ${CISA_GROUP} RUN useradd --system --uid ${CISA_UID} --gid ${CISA_GROUP} --comment "${CISA_USER} user" ${CISA_USER} ## -# Make sure pip and setuptools are the latest versions +# Make sure pip, setuptools, and wheel are the latest versions ## -RUN pip install --no-cache-dir --upgrade pip setuptools +RUN pip install --no-cache-dir --upgrade pip setuptools wheel ## # Install client-cert-update python requirements From fdc751010c2342028ec559aae21a1685b95cc06e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 1 Sep 2022 11:21:08 -0400 Subject: [PATCH 335/417] Update base Docker image tag Switch from python:3.7.3-slim-stretch to python:3.10.7-slim-bullseye. We switch from Debian Stretch to Debian Bullseye because there is no Debian Stretch version of a python:3.10.7 image available. If we already have to bump to a more recent Debian version it makes sense to jump to the latest release. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 67eb1c5..754581b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.3-slim-stretch +FROM python:3.10.7-slim-bullseye # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md From 7b6069896fe4fee45d5e67dbce8ffa67030a492d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 01:55:35 -0400 Subject: [PATCH 336/417] Switch to Alpine for the Docker image If possible it is preferable to use Alpine for our Docker images as its design is well-suited for Docker images. Since this is a straight- forward image that does not rely on packages with compiled elements we can safely switch from Debian to Alpine for the OS of the underlying Docker image. --- .github/workflows/build.yml | 2 +- Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bece403..9b6e681 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ env: CURL_CACHE_DIR: ~/.cache/curl IMAGE_NAME: cisagov/client-cert-update PIP_CACHE_DIR: ~/.cache/pip - PLATFORMS: "linux/386,linux/amd64,linux/arm/v5,linux/arm/v7,\ + PLATFORMS: "linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,\ linux/arm64/v8,linux/ppc64le,linux/s390x" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} diff --git a/Dockerfile b/Dockerfile index 754581b..f1a49f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.7-slim-bullseye +FROM python:3.10.7-alpine3.16 # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md @@ -15,8 +15,8 @@ ENV CISA_HOME="/home/cisa" ### # Create unprivileged user ### -RUN groupadd --system --gid ${CISA_GID} ${CISA_GROUP} -RUN useradd --system --uid ${CISA_UID} --gid ${CISA_GROUP} --comment "${CISA_USER} user" ${CISA_USER} +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} +RUN adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ## # Make sure pip, setuptools, and wheel are the latest versions From 7abcb690d62491a2df11aadb2ea33d16afd996fd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:04:33 -0400 Subject: [PATCH 337/417] Combine user creation commands under one RUN statement It's better to combine related commands when possible to both logically group them and to consolidate Docker image layers. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1a49f7..0110c79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,8 @@ ENV CISA_HOME="/home/cisa" ### # Create unprivileged user ### -RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} -RUN adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ + && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ## # Make sure pip, setuptools, and wheel are the latest versions From 62f5c459bd24fdd7bb427ccd941b2cfb6ed93738 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:07:40 -0400 Subject: [PATCH 338/417] Adjust the image entrypoint Rather than making the script executable and using it as an entrypoint we instead call python3 to invoke the script as the entrypoint. --- Dockerfile | 2 +- src/email-update.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) mode change 100755 => 100644 src/email-update.py diff --git a/Dockerfile b/Dockerfile index 0110c79..315ba3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,4 +40,4 @@ RUN chown --recursive ${CISA_USER}:${CISA_USER} $CISA_HOME ### # USER $USER WORKDIR $CISA_HOME -ENTRYPOINT ["./email-update.py"] +ENTRYPOINT ["python3", "email-update.py"] diff --git a/src/email-update.py b/src/email-update.py old mode 100755 new mode 100644 index bd422fc..b545d16 --- a/src/email-update.py +++ b/src/email-update.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """email-update.py sends a list of HTTP sites requiring client certs. Usage: From 552d78623f97d0d3c13f2f59c88c78628876fc4e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:11:42 -0400 Subject: [PATCH 339/417] Update user information in the Dockerfile Base default values on the user and uid instead of a mix of values. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 315ba3c..b175b3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,11 @@ FROM python:3.10.7-alpine3.16 LABEL org.opencontainers.image.authors="jeremy.frasier@trio.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" -ARG CISA_GID=421 -ARG CISA_UID=${CISA_GID} -ENV CISA_USER="cisa" +ARG CISA_UID=421 +ENV CISA_GID=${CISA_UID} +ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} -ENV CISA_HOME="/home/cisa" +ENV CISA_HOME="/home/${CISA_USER}" ### # Create unprivileged user From 7ff2d976d4cfc5471b5e39693f4ba87c8d37fd4a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:13:50 -0400 Subject: [PATCH 340/417] Ensure file ownership as part of the COPY command Instead of manually running a chown command to ensure ownership we instead assign ownership as part of copying the files into the image. --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b175b3b..c2571e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,8 +32,7 @@ RUN pip install --no-cache-dir --requirement /tmp/requirements.txt # Put this just before we change users because the copy (and every # step after it) will often be rerun by docker, but we need to be root # for the chown command. -COPY src/email-update.py src/body.txt src/body.html $CISA_HOME/ -RUN chown --recursive ${CISA_USER}:${CISA_USER} $CISA_HOME +COPY --chown=${CISA_USER}:${CISA_GROUP} src/email-update.py src/body.txt src/body.html $CISA_HOME/ ### # Prepare to Run From 2fd7d981362276958a83afe77c27444e9a1ad104 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:15:09 -0400 Subject: [PATCH 341/417] Use the unprivileged user that has been set up --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c2571e6..ed3f73e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,6 @@ COPY --chown=${CISA_USER}:${CISA_GROUP} src/email-update.py src/body.txt src/bod ### # Prepare to Run ### -# USER $USER WORKDIR $CISA_HOME +USER $CISA_USER ENTRYPOINT ["python3", "email-update.py"] From 5760ac86dfae657bb2b534b47696666b1207cfbf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:22:21 -0400 Subject: [PATCH 342/417] Add pinning for some Python packages Since they are managed in the Dockerfile we add version pins for the pip, setuptools, and wheel packages. --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ed3f73e..e4767a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,10 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ ## # Make sure pip, setuptools, and wheel are the latest versions ## -RUN pip install --no-cache-dir --upgrade pip setuptools wheel +RUN pip install --no-cache-dir --upgrade \ + pip==22.2.2 \ + setuptools==65.3.0 \ + wheel==0.37.1 ## # Install client-cert-update python requirements From 262fc950061f1938289d7d5dfd5ba99b04a18422 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:24:59 -0400 Subject: [PATCH 343/417] Update Dockerfile comments Cleanup, correct, and streamline the commenting in the Dockerfile. --- Dockerfile | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index e4767a1..9a37184 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,40 +6,32 @@ FROM python:3.10.7-alpine3.16 LABEL org.opencontainers.image.authors="jeremy.frasier@trio.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" +# Unprivileged user information ARG CISA_UID=421 ENV CISA_GID=${CISA_UID} ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" -### # Create unprivileged user -### RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -## -# Make sure pip, setuptools, and wheel are the latest versions -## +# Install core Python packages RUN pip install --no-cache-dir --upgrade \ pip==22.2.2 \ setuptools==65.3.0 \ wheel==0.37.1 -## -# Install client-cert-update python requirements -## +# Install client-cert-update Python requirements COPY src/requirements.txt /tmp RUN pip install --no-cache-dir --requirement /tmp/requirements.txt # Put this just before we change users because the copy (and every -# step after it) will often be rerun by docker, but we need to be root -# for the chown command. +# step after it) will often be rerun by Docker. COPY --chown=${CISA_USER}:${CISA_GROUP} src/email-update.py src/body.txt src/body.html $CISA_HOME/ -### -# Prepare to Run -### +# Prepare to run WORKDIR $CISA_HOME USER $CISA_USER ENTRYPOINT ["python3", "email-update.py"] From 2d59ea995dc9d5b83f8ae9c5afe638c9b175372f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 9 Sep 2022 13:25:52 -0400 Subject: [PATCH 344/417] Call pip as a Python module Instead of relying on the pip command we instead call it as a module from the python3 command. This ensures that pip is installing packages in the same Python version/environment as we use to run the `email-update.py` script. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a37184..c9119c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,14 +18,14 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} # Install core Python packages -RUN pip install --no-cache-dir --upgrade \ +RUN python3 -m pip install --no-cache-dir --upgrade \ pip==22.2.2 \ setuptools==65.3.0 \ wheel==0.37.1 # Install client-cert-update Python requirements COPY src/requirements.txt /tmp -RUN pip install --no-cache-dir --requirement /tmp/requirements.txt +RUN python3 -m pip install --no-cache-dir --requirement /tmp/requirements.txt # Put this just before we change users because the copy (and every # step after it) will often be rerun by Docker. From d271aa2c23014c87cff85aff8481dc02c2863a97 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Sep 2022 13:27:11 -0400 Subject: [PATCH 345/417] Use pipenv to manage Python dependencies Switch to using pipenv to manage the Python dependencies for this project. This ensures that repeat builds install the same package versions regardless of new package releases. This will help give us repeatable and consistent builds. --- Dockerfile | 42 +++++++-- requirements-dev.txt | 1 + src/Pipfile | 13 +++ src/Pipfile.lock | 212 +++++++++++++++++++++++++++++++++++++++++++ src/requirements.txt | 3 - 5 files changed, 258 insertions(+), 13 deletions(-) create mode 100644 src/Pipfile create mode 100644 src/Pipfile.lock delete mode 100644 src/requirements.txt diff --git a/Dockerfile b/Dockerfile index c9119c8..3d1a2e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.7-alpine3.16 +FROM python:3.10.7-alpine3.16 as compile-stage # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md @@ -6,26 +6,48 @@ FROM python:3.10.7-alpine3.16 LABEL org.opencontainers.image.authors="jeremy.frasier@trio.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" +# Unprivileged user information necessary for the Python virtual environment +ARG CISA_USER="cisa" +ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" + +# Install pipenv to manage installing the Python dependencies into a created +# Python virtual environment. This is done separately from the virtual +# environment so that pipenv and its dependencies are not installed in the +# Python virtual environment used in the final image. +RUN python3 -m pip install --no-cache-dir --upgrade pipenv==2022.9.8 \ + # Manually create Python virtual environment for the final image + && python3 -m venv ${VIRTUAL_ENV} \ + # Ensure the core Python packages are installed in the virtual environment + && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ + pip==22.2.2 \ + setuptools==65.3.0 \ + wheel==0.37.1 + +# Install client-cert-update Python requirements +WORKDIR /tmp +COPY src/Pipfile src/Pipfile.lock ./ +# pipenv will install packages into the virtual environment specified in the +# VIRTUAL_ENV environment variable if it is set. +RUN pipenv sync --clear --verbose + +FROM python:3.10.7-alpine3.16 as build-stage + # Unprivileged user information ARG CISA_UID=421 ENV CISA_GID=${CISA_UID} ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Create unprivileged user RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -# Install core Python packages -RUN python3 -m pip install --no-cache-dir --upgrade \ - pip==22.2.2 \ - setuptools==65.3.0 \ - wheel==0.37.1 - -# Install client-cert-update Python requirements -COPY src/requirements.txt /tmp -RUN python3 -m pip install --no-cache-dir --requirement /tmp/requirements.txt +# Copy in the Python virtual environment we created in the compile stage +COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # Put this just before we change users because the copy (and every # step after it) will often be rerun by Docker. diff --git a/requirements-dev.txt b/requirements-dev.txt index cb51627..bdc1615 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ --requirement requirements-test.txt ipython +pipenv semver diff --git a/src/Pipfile b/src/Pipfile new file mode 100644 index 0000000..072846e --- /dev/null +++ b/src/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +# Minimum version for IMDSv2 support +boto3 = ">=1.13.23" +docopt = ">=0.6.2" +mongo-db-from-config = {file = "https://github.com/cisagov/mongo-db-from-config/archive/v0.1.0.tar.gz"} + +[requires] +python_full_version = "3.10.7" diff --git a/src/Pipfile.lock b/src/Pipfile.lock new file mode 100644 index 0000000..e31728b --- /dev/null +++ b/src/Pipfile.lock @@ -0,0 +1,212 @@ +{ + "_meta": { + "hash": { + "sha256": "27ed994cc6a329349250d86b4d030e4e4f826050b328d3b3270ee108c445e025" + }, + "pipfile-spec": 6, + "requires": { + "python_full_version": "3.10.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:abac9c7e54212ee68c2a311812a498ece2c43f98d1302f099128a9fe45cf37c5", + "sha256:bb69ebe7c484305568b1aaed664db3d09716dc93235e41f23538f99ea4f3addc" + ], + "index": "pypi", + "version": "==1.24.70" + }, + "botocore": { + "hashes": [ + "sha256:2ad5be6ca322a549f858d245237358ed300f7347096caacbb29c013e10663d34", + "sha256:c0a014b4dfd7ffe739393034aa6e70e73f3a1b22c7667a59858259b341bf5082" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.70" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "index": "pypi", + "version": "==0.6.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "mongo-db-from-config": { + "file": "https://github.com/cisagov/mongo-db-from-config/archive/v0.1.0.tar.gz", + "hashes": [ + "sha256:0655651e5945e327e2b6baf17aafe6c603635278f8ac3c208c51f7a0360ab44c", + "sha256:f42787f5aba88ac63b20b70439a6fa09d60daeff9169991d340b49b4d777ce16" + ], + "version": "==0.1.0" + }, + "pymongo": { + "hashes": [ + "sha256:01721da74558f2f64a9f162ee063df403ed656b7d84229268d8e4ae99cfba59c", + "sha256:07564178ecc203a84f63e72972691af6c0c82d2dc0c9da66ba711695276089ba", + "sha256:0f53253f4777cbccc426e669a2af875f26c95bd090d88593287b9a0a8ac7fa25", + "sha256:10f09c4f09757c2e2a707ad7304f5d69cb8fdf7cbfb644dbacfe5bbe8afe311b", + "sha256:124d0e880b66f9b0778613198e89984984fdd37a3030a9007e5f459a42dfa2d3", + "sha256:147a23cd96feb67606ac957744d8d25b013426cdc3c7164a4f99bd8253f649e3", + "sha256:153b8f8705970756226dfeeb7bb9637e0ad54a4d79b480b4c8244e34e16e1662", + "sha256:193cc97d44b1e6d2253ea94e30c6f94f994efb7166e2452af4df55825266e88b", + "sha256:1a957cdc2b26eeed4d8f1889a40c6023dd1bd94672dd0f5ce327314f2caaefd4", + "sha256:1c81414b706627f15e921e29ae2403aab52e33e36ed92ed989c602888d7c3b90", + "sha256:21238b19243a42f9a34a6d39e7580ceebc6da6d2f3cf729c1cff9023cb61a5f1", + "sha256:2bfe6b59f431f40fa545547616f4acf0c0c4b64518b1f951083e3bad06eb368b", + "sha256:314b556afd72eb21a6a10bd1f45ef252509f014f80207db59c97372103c88237", + "sha256:31c50da4a080166bc29403aa91f4c76e0889b4f24928d1b60508a37c1bf87f9a", + "sha256:3be53e9888e759c49ae35d747ff77a04ff82b894dd64601e0f3a5a159b406245", + "sha256:44b36ccb90aac5ea50be23c1a6e8f24fbfc78afabdef114af16c6e0a80981364", + "sha256:4cadaaa5c19ad23fc84559e90284f2eb003c36958ebb2c06f286b678f441285f", + "sha256:60c470a58c5b62b1b12a5f5458f8e2f2f67b94e198d03dc5352f854d9230c394", + "sha256:6673ab3fbf3135cc1a8c0f70d480db5b2378c3a70af8d602f73f76b8338bdf97", + "sha256:68e1e49a5675748233f7b05330f092582cd52f2850b4244939fd75ba640593ed", + "sha256:69d0180bca594e81cdb4a2af328bdb4046f59e10aaeef7619496fe64f2ec918c", + "sha256:6bd5888997ea3eae9830c6cc7964b61dcfbc50eb3a5a6ce56ad5f86d5579b11c", + "sha256:701d331060dae72bf3ebdb82924405d14136a69282ccb00c89fc69dee21340b4", + "sha256:70216ec4c248213ae95ea499b6314c385ce01a5946c448fb22f6c8395806e740", + "sha256:72f338f6aabd37d343bd9d1fdd3de921104d395766bcc5cdc4039e4c2dd97766", + "sha256:764fc15418d94bce5c2f8ebdbf66544f96f42efb1364b61e715e5b33281b388d", + "sha256:766acb5b1a19eae0f7467bcd3398748f110ea5309cdfc59faa5185dcc7fd4dca", + "sha256:76892bbce743eb9f90360b3626ea92f13d338010a1004b4488e79e555b339921", + "sha256:773467d25c293f8e981b092361dab5fd800e1ba318403b7959d35004c67faedc", + "sha256:80cbf0b043061451660099fff9001a7faacb2c9c983842b4819526e2f944dc6c", + "sha256:83168126ae2457d1a19b2af665cafa7ef78c2dcff192d7d7b5dad6b36c73ae24", + "sha256:83cc3c35aeeceb67143914db67f685206e1aa37ea837d872f4bc28d7f80917c9", + "sha256:8a86e8c2ac2ec87141e1c6cb00bdb18a4560f06e5f96769abcd1dda24dc0e764", + "sha256:8a9bc4dcfc2bda69ee88cdb7a89b03f2b8eca668519b704384a264dea2db4209", + "sha256:8c223aea52c359cc8fdee5bd3475532590755c269ec4d4fe581acd47a44e9952", + "sha256:8cbb868e88c4eee1c53364bb343d226a3c0e959e791e6828030cb78f46cfcbe3", + "sha256:902e2c9030cb042c49750bc70d72d830d42c64ea0df5ff8630c171e065c93dd7", + "sha256:a25c0eb2d610b20e276e684be61c337396813b636b69373c17314283cb1a3b14", + "sha256:a3efdf154844244e0dabe902cf1827fdced55fa5b144adec2a86e5ce50a99b97", + "sha256:a6bf01b9237f794fa3bdad5089474067d28be7e199b356a18d3f247a45775f26", + "sha256:a7eb5b06744b911b6668b427c8abc71b6d624e72d3dfffed00988fa1b4340f97", + "sha256:b0be613d926c5dbb0d3fc6b58e4f2be4979f80ae76fda6e47309f011b388fe0c", + "sha256:b211e161b6cc2790e0d640ad38e0429d06c944e5da23410f4dc61809dba25095", + "sha256:b537dd282de1b53d9ae7cf9f3df36420c8618390f2da92100391f3ba8f3c141a", + "sha256:b9e4981a65f8500a3a46bb3a1e81b9feb45cf0b2115ad9c4f8d517326d026940", + "sha256:c549bb519456ee230e92f415c5b4d962094caac0fdbcc4ed22b576f66169764e", + "sha256:c69ef5906dcd6ec565d4d887ba97ceb2a84f3b614307ee3b4780cb1ea40b1867", + "sha256:c8b4a782aac43948308087b962c9ecb030ba98886ce6dee3ad7aafe8c5e1ce80", + "sha256:cc7ebc37b03956a070260665079665eae69e5e96007694214f3a2107af96816a", + "sha256:ccfdc7722df445c49dc6b5d514c3544cad99b53189165f7546793933050ac7fb", + "sha256:d8bb745321716e7a11220a67c88212ecedde4021e1de4802e563baef9df921d2", + "sha256:d94f535df9f539615bc3dbbef185ded3b609373bb44ca1afffcabac70202678a", + "sha256:d98d2a8283c9928a9e5adf2f3c0181e095579e9732e1613aaa55d386e2bcb6c5", + "sha256:dc24737d24ce0de762bee9c2a884639819485f679bbac8ab5be9c161ef6f9b2c", + "sha256:e08fe1731f5429435b8dea1db9663f9ed1812915ff803fc9991c7c4841ed62ad", + "sha256:e09cdf5aad507c8faa30d97884cc42932ed3a9c2b7f22cc3ccc607bae03981b3", + "sha256:e152c26ffc30331e9d57591fc4c05453c209aa20ba299d1deb7173f7d1958c22", + "sha256:e1b8f5e2f9637492b0da4d51f78ecb17786e61d6c461ead8542c944750faf4f9", + "sha256:e39cacee70a98758f9b2da53ee175378f07c60113b1fa4fae40cbaee5583181e", + "sha256:e64442aba81ed4df1ca494b87bf818569a1280acaa73071c68014f7a884e83f1", + "sha256:e7dcb73f683c155885a3488646fcead3a895765fed16e93c9b80000bc69e96cb", + "sha256:ecdcb0d4e9b08b739035f57a09330efc6f464bd7f942b63897395d996ca6ebd5", + "sha256:ed90a9de4431cbfb2f3b2ef0c5fd356e61c85117b2be4db3eae28cb409f6e2d5", + "sha256:f1c23527f8e13f526fededbb96f2e7888f179fe27c51d41c2724f7059b75b2fa", + "sha256:f47d5f10922cf7f7dfcd1406bd0926cef6d866a75953c3745502dffd7ac197dd", + "sha256:fe0820d169635e41c14a5d21514282e0b93347878666ec9d5d3bf0eed0649948", + "sha256:ff66014687598823b6b23751884b4aa67eb934445406d95894dfc60cb7bfcc18" + ], + "markers": "python_version >= '3.7'", + "version": "==4.2.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "pyyaml": { + "hashes": [ + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "markers": "python_version >= '3.6'", + "version": "==6.0" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "setuptools": { + "hashes": [ + "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82", + "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57" + ], + "markers": "python_version >= '3.7'", + "version": "==65.3.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", + "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.12" + } + }, + "develop": {} +} diff --git a/src/requirements.txt b/src/requirements.txt deleted file mode 100644 index 8e5e5bc..0000000 --- a/src/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -boto3 >= 1.9 -docopt >= 0.6.2 -https://api.github.com/repos/cisagov/mongo-db-from-config/tarball/develop From dcb947c47c0b4ee102da5a789b67af6437cf7b1a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Sep 2022 15:57:52 -0400 Subject: [PATCH 346/417] Consistently use the brace form of parameter expansion Using braces for parameter expansion helps ensure that the expected variable is dereferenecd and is a good safety practice. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d1a2e0..a6b8d91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,13 +47,13 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ # Copy in the Python virtual environment we created in the compile stage COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} -ENV PATH="${VIRTUAL_ENV}/bin:$PATH" +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" # Put this just before we change users because the copy (and every # step after it) will often be rerun by Docker. -COPY --chown=${CISA_USER}:${CISA_GROUP} src/email-update.py src/body.txt src/body.html $CISA_HOME/ +COPY --chown=${CISA_USER}:${CISA_GROUP} src/email-update.py src/body.txt src/body.html ${CISA_HOME}/ # Prepare to run -WORKDIR $CISA_HOME -USER $CISA_USER +WORKDIR ${CISA_HOME} +USER ${CISA_USER} ENTRYPOINT ["python3", "email-update.py"] From 8424c4163a87a37840fc9edc426ffd85969f2e47 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 21 Sep 2022 16:36:55 -0400 Subject: [PATCH 347/417] Remove the `docker-compose.override.yml` file The functionality this file provides is being moved into cisagov/ansible-role-client-cert-update since that is where that specific configuration is deployed. --- docker-compose.override.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 docker-compose.override.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml deleted file mode 100644 index a21ed3e..0000000 --- a/docker-compose.override.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -version: '3.2' - -services: - update: - command: - - "--db-creds-file=/run/secrets/scan_read_creds.yml" - - "--from=reports@cyber.dhs.gov" - - "--to=fnr.bod@hq.dhs.gov" - - "--cc=ncats-dev@beta.dhs.gov,ncats@hq.dhs.gov" - - "--reply=ncats-dev@beta.dhs.gov" - - >- - --subject="Latest list of web hosts that require authentication - via client certificates" - - "--text=body.txt" - - "--html=body.html" - - "--log-level=info" From fc55e84feba28e9652e2a21b976e3f8225b708af Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:27:47 -0500 Subject: [PATCH 348/417] Update the version of Go used from 1.18 to 1.19 Go 1.19 was released while this branch was in the wings and it makes sense to bump to the latest Go release. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4c50c5..66e0c95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: - id: setup-go uses: actions/setup-go@v3 with: - go-version: "1.18" + go-version: "1.19" - name: Lookup Go cache directory id: go-cache run: | From 2775dfa470b433cb9692b5b8c604ad55aa24ea2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:00:23 +0000 Subject: [PATCH 349/417] Bump hashicorp/setup-terraform from 1 to 2 Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 1 to 2. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v1...v2) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66e0c95..37df1a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,7 +74,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v1 + - uses: hashicorp/setup-terraform@v2 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install shfmt From 1f7c62e5f21f1ee0a034122b1e96e2e035ae4a5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:06:28 +0000 Subject: [PATCH 350/417] Bump actions/setup-python from 3 to 4 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 3 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37df1a3..a624fe2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v3 - id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" # We need the Go version and Go cache location for the actions/cache step, From 6a9c630f869dbd96001d9a00ea300e3195f2c05d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 17:30:33 -0400 Subject: [PATCH 351/417] Add an ownership comment to the Dependabot configuration Add a comment that states that the commented out ignore directives are managed by cisagov/skeleton-generic. --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 208659c..38ffdc0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,7 @@ updates: schedule: interval: "weekly" # ignore: + # # Managed by cisagov/skeleton-generic # - dependency-name: actions/cache # - dependency-name: actions/checkout # - dependency-name: actions/setup-python From dfd54e9ad294cedce783c61071f5dab151cf79e5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 17:38:06 -0400 Subject: [PATCH 352/417] Add additional Dependabot ignore directives This adds the other versioned Actions that should be managed by cisagov/skeleton-generic to the list of commented out dependencies to ignore. --- .github/dependabot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 38ffdc0..fef57f5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,7 +15,10 @@ updates: # # Managed by cisagov/skeleton-generic # - dependency-name: actions/cache # - dependency-name: actions/checkout + # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python + # - dependency-name: hashicorp/setup-terraform + # - dependency-name: mxschmitt/action-tmate - package-ecosystem: "pip" directory: "/" From 71a88a5694347f17115f90f69e8ab09108aae1ed Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 11 Oct 2022 10:57:16 -0400 Subject: [PATCH 353/417] Remove usage of set-output from our GitHub Actions workflow GitHub has deprecated the set-output command per: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ This updates the GitHub Actions workflow to use the newly preferred method to set the output for a job's step. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a624fe2..8d17421 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: - name: Lookup Go cache directory id: go-cache run: | - echo "::set-output name=dir::$(go env GOCACHE)" + echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ From 21a68f461cf04ee5009c22e6311afdfdeff6f0fe Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:35:44 -0400 Subject: [PATCH 354/417] Enable the allow-non-breakable-inline-mappings rule for yamllint --- .yamllint | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.yamllint b/.yamllint index 76a1cce..b5b302f 100644 --- a/.yamllint +++ b/.yamllint @@ -8,6 +8,14 @@ rules: # this behavior. comments-indentation: disable + # yamllint does not allow inline mappings that exceed the line length by + # default. There are many scenarios where the inline mapping may be a key, + # hash, or other long value that would exceed the line length but cannot + # reasonably be broken across lines. + line-length: + # This rule implies the allow-non-breakable-words rule + allow-non-breakable-inline-mappings: true + # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable From 3547c4f28d98a0e08582f5cf02e53d102a9e5dac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:37:21 -0400 Subject: [PATCH 355/417] Increase the maximum line length in the yamllint configuration Increase the line length maximum from the default of 80 to 88. This 10% increase mirrors the line length allowed in other linter configurations such as black. --- .yamllint | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.yamllint b/.yamllint index b5b302f..8f4e6bf 100644 --- a/.yamllint +++ b/.yamllint @@ -15,7 +15,8 @@ rules: line-length: # This rule implies the allow-non-breakable-words rule allow-non-breakable-inline-mappings: true - + # Allows a 10% overage from the default limit of 80 + max: 88 # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable From b1ea3d4239179f9dc15a3509593f921c77dedc6b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:48:04 -0400 Subject: [PATCH 356/417] Fix the repo of the flake8 pre-commit hook Per PyCQA/flake8#1290 this hook moved from GitLab to GitHub. The version we use is bumped to the latest tag on GitHub as well. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5c2e10..804fa6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,8 +91,8 @@ repos: rev: 22.3.0 hooks: - id: black - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 hooks: - id: flake8 additional_dependencies: From fb8d13f5baff79b0f0e8a231cdf2c68817695d4e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 18:45:45 -0400 Subject: [PATCH 357/417] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back due to issues with upgrading to v6. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 804fa6c..25f8d1d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.3.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.31.1 + rev: v0.32.2 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.6.1 + rev: v3.0.0-alpha.4 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.3 + rev: v1.28.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.14.2 + rev: 0.18.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.17.0 + rev: v2.20.0 hooks: - id: validate_manifest @@ -88,7 +88,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 22.10.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -102,11 +102,11 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.942 + rev: v0.990 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.31.1 + rev: v3.2.0 hooks: - id: pyupgrade @@ -119,14 +119,14 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.64.0 + rev: v1.76.0 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.1.0 + rev: v2.1.1 hooks: - id: docker-compose-check From e5fbd70b2035b6de74c5bcc804af929a19d9cb17 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 5 Oct 2022 04:44:13 -0400 Subject: [PATCH 358/417] Add a label configuration file This configuration file stores information about the labels expected in this repository. --- .github/labels.yml | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/labels.yml diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 0000000..7d7f7f3 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,67 @@ +--- +# Rather than breaking up descriptions into multiline strings we disable that +# specific rule in yamllint for this file. +# yamllint disable rule:line-length +- color: "#eb6420" + description: This issue or pull request is awaiting the outcome of another issue or pull request + name: blocked +- color: "#000000" + description: This issue or pull request involves changes to existing functionality + name: breaking change +- color: "#d73a4a" + description: This issue or pull request addresses broken functionality + name: bug +- color: "#07648d" + description: This issue will be advertised on code.gov's Open Tasks page (https://code.gov/open-tasks) + name: code.gov +- color: "#0366d6" + description: Pull requests that update a dependency file + name: dependencies +- color: "#5319e7" + description: This issue or pull request improves or adds to documentation + name: documentation +- color: "#5319e7" + description: This issue or pull request improves or adds to documentation + name: documentation +- color: "#cfd3d7" + description: This issue or pull request already exists or is covered in another issue or pull request + name: duplicate +- color: "#b005bc" + description: A high-level objective issue encompassing multiple issues instead of a specific unit of work + name: epic +- color: "#000000" + description: Pull requests that update GitHub Actions code + name: github-actions +- color: "#0e8a16" + description: This issue or pull request is well-defined and good for newcomers + name: good first issue +- color: "#ff7518" + description: Pull request that should count toward Hacktoberfest participation + name: hacktoberfest-accepted +- color: "#a2eeef" + description: This issue or pull request will add or improve functionality, maintainability, or ease of use + name: improvement +- color: "#fef2c0" + description: This issue or pull request is not applicable, incorrect, or obsolete + name: invalid +- color: "#ce099a" + description: This pull request is ready to merge during the next Lineage Kraken release + name: kraken 🐙 +- color: "#a4fc5d" + description: This issue or pull request requires further information + name: need info +- color: "#fcdb45" + description: This pull request is awaiting an action or decision to move forward + name: on hold +- color: "#ef476c" + description: This issue is a request for information or needs discussion + name: question +- color: "#1d76db" + description: This issue or pull request pulls in upstream updates + name: upstream update +- color: "#d4c5f9" + description: This issue or pull request increments the version number + name: version bump +- color: "#ffffff" + description: This issue will not be incorporated + name: wontfix From b9792a8d18d3690b8497e0cfe31ce855c85a7c2b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 5 Oct 2022 04:57:03 -0400 Subject: [PATCH 359/417] Add a workflow to synchronize repository labels This adds a workflow to ensure that the repository labels are updated to reflect changes to the label configuration file .github/labels.yml. --- .github/workflows/sync-labels.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/sync-labels.yml diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml new file mode 100644 index 0000000..ae7c091 --- /dev/null +++ b/.github/workflows/sync-labels.yml @@ -0,0 +1,29 @@ +--- +name: sync-labels + +on: + push: + paths: + - '.github/labels.yml' + - '.github/workflows/sync-labels.yml' + +permissions: + contents: read + +jobs: + labeler: + permissions: + # actions/checkout needs this to fetch code + contents: read + # crazy-max/ghaction-github-labeler needs this to manage repository labels + issues: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Sync repository labels + if: success() + uses: crazy-max/ghaction-github-labeler@v4 + with: + # This is a hideous ternary equivalent so we only do a dry run unless + # this workflow is triggered by the develop branch. + dry-run: ${{ github.ref_name == 'develop' && 'false' || 'true' }} From 161c4aebba002d4073a634749f3ae67f649a8a2a Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 21 Oct 2022 15:56:30 -0400 Subject: [PATCH 360/417] Remove duplicate label --- .github/labels.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 7d7f7f3..24bd511 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -20,9 +20,6 @@ - color: "#5319e7" description: This issue or pull request improves or adds to documentation name: documentation -- color: "#5319e7" - description: This issue or pull request improves or adds to documentation - name: documentation - color: "#cfd3d7" description: This issue or pull request already exists or is covered in another issue or pull request name: duplicate From 5993231c3a136c86fe2dad178ac47495771cccce Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 21 Oct 2022 15:56:49 -0400 Subject: [PATCH 361/417] Add a label for issues or PRs that involve test code --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 24bd511..b108748 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -53,6 +53,9 @@ - color: "#ef476c" description: This issue is a request for information or needs discussion name: question +- color: "#00008b" + description: This issue or pull request adds or otherwise modifies test code + name: test - color: "#1d76db" description: This issue or pull request pulls in upstream updates name: upstream update From 6cc53dc1f0bf5123465082100c57bb6a325024cf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:23:04 -0500 Subject: [PATCH 362/417] Fix GitHub label color values Update the configuration for repository labels to remove the leading `#` from color values. With a `#` leading the values they are seen as invalid by the GitHub API. --- .github/labels.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index b108748..500b000 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -2,66 +2,66 @@ # Rather than breaking up descriptions into multiline strings we disable that # specific rule in yamllint for this file. # yamllint disable rule:line-length -- color: "#eb6420" +- color: "eb6420" description: This issue or pull request is awaiting the outcome of another issue or pull request name: blocked -- color: "#000000" +- color: "000000" description: This issue or pull request involves changes to existing functionality name: breaking change -- color: "#d73a4a" +- color: "d73a4a" description: This issue or pull request addresses broken functionality name: bug -- color: "#07648d" +- color: "07648d" description: This issue will be advertised on code.gov's Open Tasks page (https://code.gov/open-tasks) name: code.gov -- color: "#0366d6" +- color: "0366d6" description: Pull requests that update a dependency file name: dependencies -- color: "#5319e7" +- color: "5319e7" description: This issue or pull request improves or adds to documentation name: documentation -- color: "#cfd3d7" +- color: "cfd3d7" description: This issue or pull request already exists or is covered in another issue or pull request name: duplicate -- color: "#b005bc" +- color: "b005bc" description: A high-level objective issue encompassing multiple issues instead of a specific unit of work name: epic -- color: "#000000" +- color: "000000" description: Pull requests that update GitHub Actions code name: github-actions -- color: "#0e8a16" +- color: "0e8a16" description: This issue or pull request is well-defined and good for newcomers name: good first issue -- color: "#ff7518" +- color: "ff7518" description: Pull request that should count toward Hacktoberfest participation name: hacktoberfest-accepted -- color: "#a2eeef" +- color: "a2eeef" description: This issue or pull request will add or improve functionality, maintainability, or ease of use name: improvement -- color: "#fef2c0" +- color: "fef2c0" description: This issue or pull request is not applicable, incorrect, or obsolete name: invalid -- color: "#ce099a" +- color: "ce099a" description: This pull request is ready to merge during the next Lineage Kraken release name: kraken 🐙 -- color: "#a4fc5d" +- color: "a4fc5d" description: This issue or pull request requires further information name: need info -- color: "#fcdb45" +- color: "fcdb45" description: This pull request is awaiting an action or decision to move forward name: on hold -- color: "#ef476c" +- color: "ef476c" description: This issue is a request for information or needs discussion name: question -- color: "#00008b" +- color: "00008b" description: This issue or pull request adds or otherwise modifies test code name: test -- color: "#1d76db" +- color: "1d76db" description: This issue or pull request pulls in upstream updates name: upstream update -- color: "#d4c5f9" +- color: "d4c5f9" description: This issue or pull request increments the version number name: version bump -- color: "#ffffff" +- color: "ffffff" description: This issue will not be incorporated name: wontfix From 8f1263549992de1edd4d4e30afae3fd7c8a8c546 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:24:25 -0500 Subject: [PATCH 363/417] Correct spacing in yamllint rules There was a missing empty line in the `.yamllint` file between two rule definitions. --- .yamllint | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamllint b/.yamllint index 8f4e6bf..2a119a6 100644 --- a/.yamllint +++ b/.yamllint @@ -17,6 +17,7 @@ rules: allow-non-breakable-inline-mappings: true # Allows a 10% overage from the default limit of 80 max: 88 + # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable From 982d414e47bffa952564d8764e900ba91de7a44a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 10 Nov 2022 13:10:25 -0500 Subject: [PATCH 364/417] Change the default uid/gid for the unprivileged user This aligns us with changes made in cisagov/cyhy_amis to set up the `cyhy` user/group with a uid of 2048. This should allow secrets that are set to read-only for the `cyhy` user to be usable by the Docker image's unprivileged user. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a6b8d91..4512047 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ RUN pipenv sync --clear --verbose FROM python:3.10.7-alpine3.16 as build-stage # Unprivileged user information -ARG CISA_UID=421 +ARG CISA_UID=2048 ENV CISA_GID=${CISA_UID} ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} From 663dd877b8b2849462deac74365b674d7cb858f0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:38:17 -0500 Subject: [PATCH 365/417] Add repository specific label --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 500b000..e8813ce 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -17,6 +17,9 @@ - color: "0366d6" description: Pull requests that update a dependency file name: dependencies +- color: "2497ed" + description: Pull requests that update Docker code + name: docker - color: "5319e7" description: This issue or pull request improves or adds to documentation name: documentation From 3aa675e339d684172e6525cabeb82786787af549 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:42:20 -0500 Subject: [PATCH 366/417] Update Python setup in the test job for GHA Bump actions/setup-python from v3 to v4 and use Python 3.10 to mirror the `lint` job. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35e2ee4..87b5d2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -301,9 +301,9 @@ jobs: steps: - uses: actions/checkout@v3 - id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.10" - name: Cache testing environments uses: actions/cache@v3 env: From 827ff50eff69295d4e1a0361b371116abbdaac97 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 23:38:07 -0500 Subject: [PATCH 367/417] Remove uses of the `set-output` command Remove the remaining uses of the `set-output` command since it has been deprecated per: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87b5d2b..da15890 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -215,9 +215,9 @@ jobs: do TAGS="${TAGS},ghcr.io/${i}" done - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - echo ::set-output name=source_version::$(./bump_version.sh show) - echo ::set-output name=tags::${TAGS} + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + echo "source_version=$(./bump_version.sh show)" >> $GITHUB_OUTPUT + echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo tags=${TAGS} - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 From a856426b6c321ffbef11d1b1a2503ae21193ebea Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 21 Nov 2022 13:51:53 -0500 Subject: [PATCH 368/417] Update the version of the second bandit hook The `bandit (everything else)` hook was not updated in sync with the `bandit (tests tree)` hook and is using an older version of bandit. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1ec95b..86c3d22 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,7 +92,7 @@ repos: - --config=.bandit.yml # Run bandit everything but tests directory - repo: https://github.com/PyCQA/bandit - rev: 1.7.0 + rev: 1.7.4 hooks: - id: bandit name: bandit (everything else) From 0ad31cc94043023ae72e92032362008623c13e98 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 21 Nov 2022 13:53:21 -0500 Subject: [PATCH 369/417] Correct poor English in comments for the bandit pre-commit hooks Co-authored-by: dav3r --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86c3d22..a119f7c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,6 +82,7 @@ repos: - id: shell-lint # Python hooks + # Run bandit on the "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit rev: 1.7.4 hooks: @@ -90,7 +91,7 @@ repos: files: tests args: - --config=.bandit.yml - # Run bandit everything but tests directory + # Run bandit on everything except the "tests" tree - repo: https://github.com/PyCQA/bandit rev: 1.7.4 hooks: From 0d24f9c08a6844a08d34adb616a7d49616ef1f6b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:57:51 -0500 Subject: [PATCH 370/417] Update the CodeQL configuration Pull in the CodeQL workflow from cisagov/skeleton-python-library to update the CodeQL configuration for this repository. We also add github/codeql-action to the dependencies managed by this repository in the dependabot configuration. --- .github/dependabot.yml | 1 + .github/workflows/codeql-analysis.yml | 42 +++++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9f71e38..f6eb2af 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,6 +32,7 @@ updates: # - dependency-name: docker/login-action # - dependency-name: docker/setup-buildx-action # - dependency-name: docker/setup-qemu-action + # - dependency-name: github/codeql-action - package-ecosystem: "pip" directory: "/" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 33d1999..bf0d148 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,5 +1,4 @@ --- - # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # @@ -11,10 +10,12 @@ on: push: # Dependabot triggered push events have read-only access, but uploading code # scanning requires write access. - branches-ignore: [dependabot/**] + branches-ignore: + - dependabot/** pull_request: # The branches below must be a subset of the branches above - branches: [develop] + branches: + - develop schedule: - cron: '0 21 * * 6' @@ -22,48 +23,45 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest - + permissions: + # required for all workflows + security-events: write strategy: fail-fast: false matrix: # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', - # 'python'] - language: ['python'] + # Supported options are go, javascript, csharp, python, cpp, and java + language: + - python # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a - # config file. By default, queries listed here will override any - # specified in a config file. Prefix the list here with "+" to use - # these queries and those in the config file. queries: - # ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Autobuild attempts to build any compiled languages (C/C++, C#, or # Java). If this step fails, then you should remove it and run the build - # manually (see below) + # manually (see below). - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏️ If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language # - run: | - # make bootstrap - # make release + # make bootstrap + # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From 87d0f97e200fe18d899e1d339fd0db6e36359535 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jun 2022 14:29:15 -0400 Subject: [PATCH 371/417] Remove dangling argument in the pre-commit configuration This argument references a file that is no longer found in the repository. The configuration also no longer uses any such files so it is safe to remove this as opposed to updating it instead. --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a119f7c..b5dce48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,6 @@ repos: args: - --allow-missing-credentials - id: detect-private-key - exclude: src/secrets/privkey.pem - id: end-of-file-fixer exclude: files/(issue|motd) - id: mixed-line-ending From eacc4763ab864fdc35ab78d8e07dc83cca525561 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jun 2022 14:33:47 -0400 Subject: [PATCH 372/417] Update files used for Python requirements by LGTM Add all requirements*.txt files in the configuration to this file. This ensures that all requirements in use are being used for scanning. --- .lgtm.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.lgtm.yml b/.lgtm.yml index 8950263..70385b2 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -4,5 +4,7 @@ extraction: python_setup: version: 3 requirements_files: + - requirements-dev.txt - requirements-test.txt + - requirements.txt setup_py: false From 42f3468e16963694c8e7d3663c80a3faf9d7b1c0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jun 2022 15:11:06 -0400 Subject: [PATCH 373/417] Restore the alerts badging for LGTM Until we completely remove LGTM from our repositories we should keep at least the minimum badging. This project does not directly use any Python, JavaScript, etc. so I am enabling the "Total alerts" badge. Other possibilities are commented out to make it easier for downstream repositories to enable what suits their configurations. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 57f8c30..a04d771 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,11 @@ [![GitHub Build Status](https://github.com/cisagov/skeleton-docker/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/build.yml) [![CodeQL](https://github.com/cisagov/skeleton-docker/workflows/CodeQL/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/codeql-analysis.yml) [![Known Vulnerabilities](https://snyk.io/test/github/cisagov/skeleton-docker/badge.svg)](https://snyk.io/test/github/cisagov/skeleton-docker) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/) + ## Docker Image ## From 5589f23d20ab012728eca11faab053d0e043189f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 22:59:02 +0000 Subject: [PATCH 374/417] Bump actions/github-script from 5 to 6 Bumps [actions/github-script](https://github.com/actions/github-script) from 5 to 6. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da15890..1f3a541 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -174,7 +174,7 @@ jobs: - uses: actions/checkout@v3 - name: Gather repository metadata id: repo - uses: actions/github-script@v5 + uses: actions/github-script@v6 with: script: | const repo = await github.rest.repos.get(context.repo) From 1a8ab89ff211813ff2f7e4c40d5aabdce2968b99 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 13 Sep 2022 09:59:08 -0400 Subject: [PATCH 375/417] Fix overly match-happy sed commands We saw in cisagov/postfix-docker#47 that the sed commands in the bump_version.sh script could inadvertently match the CC0 version in the README.md file. This change escapes the periods in the version before passing it on to sed so that they only match periods and not just any character. --- bump_version.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index a6c8ed9..963389f 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -12,6 +12,9 @@ README_FILE=README.md HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) +# Comment out periods so they are interpreted as periods and don't +# just match any character +old_version_regex=${old_version//\./\\\.} if [ $# -ne 1 ]; then echo "$HELP_INFORMATION" @@ -21,9 +24,9 @@ else new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") echo Changing version from "$old_version" to "$new_version" tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE - sed "s/$old_version/$new_version/" $README_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file mv $tmp_file $README_FILE git add $VERSION_FILE $README_FILE git commit -m"Bump version from $old_version to $new_version" @@ -33,12 +36,12 @@ else new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") echo Changing version from "$old_version" to "$new_version" tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE - sed "s/$old_version/$new_version/" $README_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file mv $tmp_file $README_FILE git add $VERSION_FILE $README_FILE - git commit -m"Bump version from $old_version to $new_version" + git commit -m"Finalize version from $old_version to $new_version" git push ;; show) From ba23619f6ab8a3d1a7ea65a43609024a4f9d75ca Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:01:01 -0400 Subject: [PATCH 376/417] Prettify a comment --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 248d6cf..6c50a0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,12 @@ FROM python:3.10.1-alpine ARG VERSION +### # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md +# # Note: Additional labels are added by the build workflow. +### LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" From 343c0cb236fbf288f2574e6b5544000200d97e0a Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:03:46 -0400 Subject: [PATCH 377/417] Rework creation of unprivileged user and installation of dependencies Also add a section to update pip and setuptools via pip. --- Dockerfile | 62 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c50a0f..54bd465 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,17 +13,65 @@ ARG VERSION LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" -ARG CISA_UID=421 +### +# Unprivileged user setup variables +### +ARG CISA_GID=421 +ARG CISA_UID=${CISA_GID} +ENV CISA_USER="cisa" +ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/cisa" ENV ECHO_MESSAGE="Hello World from Dockerfile" -RUN addgroup --system --gid ${CISA_UID} cisa \ - && adduser --system --uid ${CISA_UID} --ingroup cisa cisa +### +# Unprivileged user setup dependencies +# +# Install shadow, so we have adduser and addgroup. +# +# Note that we use apk --no-cache to avoid writing to a local cache. +# This results in a smaller final image, at the cost of slightly +# longer install times. +# +# Setup user dependencies are only needed for setting up the user and +# will be removed at the end of that process. +### +ENV SETUP_USER_DEPS \ + shadow +RUN apk --update --no-cache --quiet upgrade +RUN apk --no-cache --quiet add ${SETUP_USER_DEPS} + +### +# Create unprivileged user +### +RUN addgroup --system --gid ${CISA_UID} ${CISA_GROUP} \ + && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} + +### +# Remove build dependencies for unprivileged user +### +RUN apk --no-cache --quiet del ${SETUP_USER_DEPS} + +### +# Dependencies +# +# Note that we use apk --no-cache to avoid writing to a local cache. +# This results in a smaller final image, at the cost of slightly +# longer install times. +### +ENV DEPS \ + ca-certificates \ + openssl \ + py-pip +RUN apk --no-cache --quiet add ${DEPS} -RUN apk --update --no-cache add \ -ca-certificates \ -openssl \ -py-pip +### +# Make sure pip and setuptools are the latest versions +# +# Note that we use pip --no-cache-dir to avoid writing to a local +# cache. This results in a smaller final image, at the cost of +# slightly longer install times. +### +RUN pip install --no-cache-dir --upgrade pip setuptools WORKDIR ${CISA_HOME} From 717334c62cb1c972a16795d0e2bebd0ab724327f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:05:06 -0400 Subject: [PATCH 378/417] Add comments and prefer long CLI options --- Dockerfile | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 54bd465..de48f7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,14 +75,24 @@ RUN pip install --no-cache-dir --upgrade pip setuptools WORKDIR ${CISA_HOME} -RUN wget -O sourcecode.tgz https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ - tar xzf sourcecode.tgz --strip-components=1 && \ - pip install --requirement requirements.txt && \ - ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ - rm sourcecode.tgz +### +# Install Python dependencies +# +# Note that we use pip --no-cache-dir to avoid writing to a local +# cache. This results in a smaller final image, at the cost of +# slightly longer install times. +### +RUN wget --output-document sourcecode.tgz \ + https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ + tar --extract --gzip --file sourcecode.tgz --strip-components=1 && \ + pip install --no-cache-dir --requirement requirements.txt && \ + ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ + rm sourcecode.tgz +### +# Prepare to run +### USER cisa - EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] From 2a72bf612a152a0a4b32a481363cf394f5fbbf3b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:16:05 -0400 Subject: [PATCH 379/417] Move ECHO_MESSAGE declaration to the "Prepare to run" section --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index de48f7c..f4dddd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,6 @@ ARG CISA_UID=${CISA_GID} ENV CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/cisa" -ENV ECHO_MESSAGE="Hello World from Dockerfile" ### # Unprivileged user setup dependencies @@ -92,6 +91,7 @@ RUN wget --output-document sourcecode.tgz \ ### # Prepare to run ### +ENV ECHO_MESSAGE="Hello World from Dockerfile" USER cisa EXPOSE 8080/TCP VOLUME ["/var/log"] From 8cf8d234f67bde8648ca60e6caa4886fdf851872 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 21 Nov 2022 21:47:09 -0500 Subject: [PATCH 380/417] Remove LGTM from the repository LGTM.com is shutting down 2022-12-16 so we should remove all LGTM-related items to prepare for disabling the LGTM.com integration in the organization. --- .lgtm.yml | 10 ---------- README.md | 5 ----- 2 files changed, 15 deletions(-) delete mode 100644 .lgtm.yml diff --git a/.lgtm.yml b/.lgtm.yml deleted file mode 100644 index 70385b2..0000000 --- a/.lgtm.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -extraction: - python: - python_setup: - version: 3 - requirements_files: - - requirements-dev.txt - - requirements-test.txt - - requirements.txt - setup_py: false diff --git a/README.md b/README.md index a04d771..57f8c30 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,6 @@ [![GitHub Build Status](https://github.com/cisagov/skeleton-docker/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/build.yml) [![CodeQL](https://github.com/cisagov/skeleton-docker/workflows/CodeQL/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/codeql-analysis.yml) [![Known Vulnerabilities](https://snyk.io/test/github/cisagov/skeleton-docker/badge.svg)](https://snyk.io/test/github/cisagov/skeleton-docker) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/) - ## Docker Image ## From 8c73bf429d5b7e7b0403abdc789436fadb6840e1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 22 Nov 2022 10:25:48 -0500 Subject: [PATCH 381/417] Restore Dockerfile changes from review I accidentally clobbered these changes when I was updating cisagov/skeleton-docker#135 for merge. This restores the changes made during review as well as fixing a missed reference to the CISA_USER argument. Co-authored-by: Shane Frasier --- Dockerfile | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4dddd9..e3cf8aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,46 +10,39 @@ ARG VERSION # # Note: Additional labels are added by the build workflow. ### -LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" +# github@cisa.dhs.gov is a very generic email distribution, and it is +# unlikely that anyone on that distribution is familiar with the +# particulars of your repository. It is therefore *strongly* +# suggested that you use an email address here that is specific to the +# person or group that maintains this repository; for example: +# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" +LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" ### # Unprivileged user setup variables ### -ARG CISA_GID=421 -ARG CISA_UID=${CISA_GID} -ENV CISA_USER="cisa" +ARG CISA_UID=421 +ARG CISA_GID=${CISA_UID} +ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} -ENV CISA_HOME="/home/cisa" +ENV CISA_HOME="/home/${CISA_USER}" ### -# Unprivileged user setup dependencies -# -# Install shadow, so we have adduser and addgroup. +# Upgrade the system # # Note that we use apk --no-cache to avoid writing to a local cache. # This results in a smaller final image, at the cost of slightly # longer install times. -# -# Setup user dependencies are only needed for setting up the user and -# will be removed at the end of that process. ### -ENV SETUP_USER_DEPS \ - shadow RUN apk --update --no-cache --quiet upgrade -RUN apk --no-cache --quiet add ${SETUP_USER_DEPS} ### # Create unprivileged user ### -RUN addgroup --system --gid ${CISA_UID} ${CISA_GROUP} \ +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -### -# Remove build dependencies for unprivileged user -### -RUN apk --no-cache --quiet del ${SETUP_USER_DEPS} - ### # Dependencies # @@ -82,17 +75,17 @@ WORKDIR ${CISA_HOME} # slightly longer install times. ### RUN wget --output-document sourcecode.tgz \ - https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ - tar --extract --gzip --file sourcecode.tgz --strip-components=1 && \ - pip install --no-cache-dir --requirement requirements.txt && \ - ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ - rm sourcecode.tgz + https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz \ + && tar --extract --gzip --file sourcecode.tgz --strip-components=1 \ + && pip install --no-cache-dir --requirement requirements.txt \ + && ln -snf /run/secrets/quote.txt src/example/data/secret.txt \ + && rm sourcecode.tgz ### # Prepare to run ### ENV ECHO_MESSAGE="Hello World from Dockerfile" -USER cisa +USER ${CISA_USER} EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] From c40256f681b71dd8f4e78966951fa91b1751167c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 22 Nov 2022 11:46:20 -0500 Subject: [PATCH 382/417] Enable dependabot ignore directive Enable the dependabot ignore directive for the github/codeql-action GitHub Action. --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ca182c0..d560ad9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,7 +32,7 @@ updates: - dependency-name: docker/login-action - dependency-name: docker/setup-buildx-action - dependency-name: docker/setup-qemu-action - # - dependency-name: github/codeql-action + - dependency-name: github/codeql-action - package-ecosystem: "pip" directory: "/" From 36b87265de548be050edcf52983d97ddb4769d91 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 22 Nov 2022 11:47:08 -0500 Subject: [PATCH 383/417] Fix the `email-update.py` script Apply changes from the `pyupgrade` pre-commit hook to pass linting. --- src/email-update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/email-update.py b/src/email-update.py index b545d16..e9a197f 100644 --- a/src/email-update.py +++ b/src/email-update.py @@ -141,11 +141,11 @@ def main(): # Construct and attach the text body body = MIMEMultipart("alternative") - with open(text_filename, "r") as text: + with open(text_filename) as text: t = text.read() body.attach(MIMEText(t, "plain")) logging.debug(f"Message plain-text body is: {t}") - with open(html_filename, "r") as html: + with open(html_filename) as html: h = html.read() html_part = MIMEText(h, "html") # See https://en.wikipedia.org/wiki/MIME#Content-Disposition From 119fde2b22d1498e0c3c4fe3973705f18e65843f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Nov 2022 16:12:01 -0500 Subject: [PATCH 384/417] Add a `python` label to the repository labels --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index e8813ce..07339d5 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -53,6 +53,9 @@ - color: "fcdb45" description: This pull request is awaiting an action or decision to move forward name: on hold +- color: "3772a4" + description: Pull requests that update Python code + name: python - color: "ef476c" description: This issue is a request for information or needs discussion name: question From 4269ef02f3cd2e769f8b27a85f37ac3bfa4c7bca Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:51:22 -0500 Subject: [PATCH 385/417] Bump from python:3.10.7-alpine3.16 to python:3.10.8-alpine3.16 --- Dockerfile | 4 ++-- src/Pipfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a749018..462fdea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.7-alpine3.16 as compile-stage +FROM python:3.10.8-alpine3.16 as compile-stage ### # For a list of pre-defined annotation keys and value types see: @@ -36,7 +36,7 @@ COPY src/Pipfile src/Pipfile.lock ./ # VIRTUAL_ENV environment variable if it is set. RUN pipenv sync --clear --verbose -FROM python:3.10.7-alpine3.16 as build-stage +FROM python:3.10.8-alpine3.16 as build-stage ### # Unprivileged user setup variables diff --git a/src/Pipfile b/src/Pipfile index 072846e..e249b04 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -10,4 +10,4 @@ docopt = ">=0.6.2" mongo-db-from-config = {file = "https://github.com/cisagov/mongo-db-from-config/archive/v0.1.0.tar.gz"} [requires] -python_full_version = "3.10.7" +python_full_version = "3.10.8" From d01806c1b38b5530b04967d229911e6598b967d4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:53:14 -0500 Subject: [PATCH 386/417] Update pinned Python packages in the Dockerfile This updates the - pip - pipenv - setuptools - wheel packages to their latest released versions. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 462fdea..44b979f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,14 +20,14 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Python virtual environment. This is done separately from the virtual # environment so that pipenv and its dependencies are not installed in the # Python virtual environment used in the final image. -RUN python3 -m pip install --no-cache-dir --upgrade pipenv==2022.9.8 \ +RUN python3 -m pip install --no-cache-dir --upgrade pipenv==2022.11.25 \ # Manually create Python virtual environment for the final image && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ - pip==22.2.2 \ - setuptools==65.3.0 \ - wheel==0.37.1 + pip==22.3.1 \ + setuptools==65.6.3 \ + wheel==0.38.4 # Install client-cert-update Python requirements WORKDIR /tmp From f435480701d20bc10793a9e1bb16cd5dcc8872ac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:57:01 -0500 Subject: [PATCH 387/417] Update the pipenv lock file Update the pipenv lock file by running `pipenv lock` in the src/ sub-directory. --- src/Pipfile.lock | 188 ++++++++++++++++++++++++++--------------------- 1 file changed, 105 insertions(+), 83 deletions(-) diff --git a/src/Pipfile.lock b/src/Pipfile.lock index e31728b..d356e8e 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "27ed994cc6a329349250d86b4d030e4e4f826050b328d3b3270ee108c445e025" + "sha256": "1b23c744955205467715f172f3ac673592c9e22e25291e01f2324322b471bc4c" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.10.7" + "python_full_version": "3.10.8" }, "sources": [ { @@ -18,19 +18,27 @@ "default": { "boto3": { "hashes": [ - "sha256:abac9c7e54212ee68c2a311812a498ece2c43f98d1302f099128a9fe45cf37c5", - "sha256:bb69ebe7c484305568b1aaed664db3d09716dc93235e41f23538f99ea4f3addc" + "sha256:3c7315da16eb0b41823965e5ce55f99cb07e94680e0ed7830c581f505fb5bd15", + "sha256:933c88b189112a5fdd82d49ef00f95b9dd649d195e557a81aecb773a3e01c517" ], "index": "pypi", - "version": "==1.24.70" + "version": "==1.26.18" }, "botocore": { "hashes": [ - "sha256:2ad5be6ca322a549f858d245237358ed300f7347096caacbb29c013e10663d34", - "sha256:c0a014b4dfd7ffe739393034aa6e70e73f3a1b22c7667a59858259b341bf5082" + "sha256:26e86fce95049f6cc18b5611901549943c4c22522fa8a3b6b265404f673977b2", + "sha256:2aba44433b6eac6d3a12cf93f2985e2d7a843307c1a527042fc48dd09b273992" ], "markers": "python_version >= '3.7'", - "version": "==1.27.70" + "version": "==1.29.18" + }, + "dnspython": { + "hashes": [ + "sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e", + "sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f" + ], + "markers": "python_version >= '3.6' and python_version < '4.0'", + "version": "==2.2.1" }, "docopt": { "hashes": [ @@ -57,76 +65,83 @@ }, "pymongo": { "hashes": [ - "sha256:01721da74558f2f64a9f162ee063df403ed656b7d84229268d8e4ae99cfba59c", - "sha256:07564178ecc203a84f63e72972691af6c0c82d2dc0c9da66ba711695276089ba", - "sha256:0f53253f4777cbccc426e669a2af875f26c95bd090d88593287b9a0a8ac7fa25", - "sha256:10f09c4f09757c2e2a707ad7304f5d69cb8fdf7cbfb644dbacfe5bbe8afe311b", - "sha256:124d0e880b66f9b0778613198e89984984fdd37a3030a9007e5f459a42dfa2d3", - "sha256:147a23cd96feb67606ac957744d8d25b013426cdc3c7164a4f99bd8253f649e3", - "sha256:153b8f8705970756226dfeeb7bb9637e0ad54a4d79b480b4c8244e34e16e1662", - "sha256:193cc97d44b1e6d2253ea94e30c6f94f994efb7166e2452af4df55825266e88b", - "sha256:1a957cdc2b26eeed4d8f1889a40c6023dd1bd94672dd0f5ce327314f2caaefd4", - "sha256:1c81414b706627f15e921e29ae2403aab52e33e36ed92ed989c602888d7c3b90", - "sha256:21238b19243a42f9a34a6d39e7580ceebc6da6d2f3cf729c1cff9023cb61a5f1", - "sha256:2bfe6b59f431f40fa545547616f4acf0c0c4b64518b1f951083e3bad06eb368b", - "sha256:314b556afd72eb21a6a10bd1f45ef252509f014f80207db59c97372103c88237", - "sha256:31c50da4a080166bc29403aa91f4c76e0889b4f24928d1b60508a37c1bf87f9a", - "sha256:3be53e9888e759c49ae35d747ff77a04ff82b894dd64601e0f3a5a159b406245", - "sha256:44b36ccb90aac5ea50be23c1a6e8f24fbfc78afabdef114af16c6e0a80981364", - "sha256:4cadaaa5c19ad23fc84559e90284f2eb003c36958ebb2c06f286b678f441285f", - "sha256:60c470a58c5b62b1b12a5f5458f8e2f2f67b94e198d03dc5352f854d9230c394", - "sha256:6673ab3fbf3135cc1a8c0f70d480db5b2378c3a70af8d602f73f76b8338bdf97", - "sha256:68e1e49a5675748233f7b05330f092582cd52f2850b4244939fd75ba640593ed", - "sha256:69d0180bca594e81cdb4a2af328bdb4046f59e10aaeef7619496fe64f2ec918c", - "sha256:6bd5888997ea3eae9830c6cc7964b61dcfbc50eb3a5a6ce56ad5f86d5579b11c", - "sha256:701d331060dae72bf3ebdb82924405d14136a69282ccb00c89fc69dee21340b4", - "sha256:70216ec4c248213ae95ea499b6314c385ce01a5946c448fb22f6c8395806e740", - "sha256:72f338f6aabd37d343bd9d1fdd3de921104d395766bcc5cdc4039e4c2dd97766", - "sha256:764fc15418d94bce5c2f8ebdbf66544f96f42efb1364b61e715e5b33281b388d", - "sha256:766acb5b1a19eae0f7467bcd3398748f110ea5309cdfc59faa5185dcc7fd4dca", - "sha256:76892bbce743eb9f90360b3626ea92f13d338010a1004b4488e79e555b339921", - "sha256:773467d25c293f8e981b092361dab5fd800e1ba318403b7959d35004c67faedc", - "sha256:80cbf0b043061451660099fff9001a7faacb2c9c983842b4819526e2f944dc6c", - "sha256:83168126ae2457d1a19b2af665cafa7ef78c2dcff192d7d7b5dad6b36c73ae24", - "sha256:83cc3c35aeeceb67143914db67f685206e1aa37ea837d872f4bc28d7f80917c9", - "sha256:8a86e8c2ac2ec87141e1c6cb00bdb18a4560f06e5f96769abcd1dda24dc0e764", - "sha256:8a9bc4dcfc2bda69ee88cdb7a89b03f2b8eca668519b704384a264dea2db4209", - "sha256:8c223aea52c359cc8fdee5bd3475532590755c269ec4d4fe581acd47a44e9952", - "sha256:8cbb868e88c4eee1c53364bb343d226a3c0e959e791e6828030cb78f46cfcbe3", - "sha256:902e2c9030cb042c49750bc70d72d830d42c64ea0df5ff8630c171e065c93dd7", - "sha256:a25c0eb2d610b20e276e684be61c337396813b636b69373c17314283cb1a3b14", - "sha256:a3efdf154844244e0dabe902cf1827fdced55fa5b144adec2a86e5ce50a99b97", - "sha256:a6bf01b9237f794fa3bdad5089474067d28be7e199b356a18d3f247a45775f26", - "sha256:a7eb5b06744b911b6668b427c8abc71b6d624e72d3dfffed00988fa1b4340f97", - "sha256:b0be613d926c5dbb0d3fc6b58e4f2be4979f80ae76fda6e47309f011b388fe0c", - "sha256:b211e161b6cc2790e0d640ad38e0429d06c944e5da23410f4dc61809dba25095", - "sha256:b537dd282de1b53d9ae7cf9f3df36420c8618390f2da92100391f3ba8f3c141a", - "sha256:b9e4981a65f8500a3a46bb3a1e81b9feb45cf0b2115ad9c4f8d517326d026940", - "sha256:c549bb519456ee230e92f415c5b4d962094caac0fdbcc4ed22b576f66169764e", - "sha256:c69ef5906dcd6ec565d4d887ba97ceb2a84f3b614307ee3b4780cb1ea40b1867", - "sha256:c8b4a782aac43948308087b962c9ecb030ba98886ce6dee3ad7aafe8c5e1ce80", - "sha256:cc7ebc37b03956a070260665079665eae69e5e96007694214f3a2107af96816a", - "sha256:ccfdc7722df445c49dc6b5d514c3544cad99b53189165f7546793933050ac7fb", - "sha256:d8bb745321716e7a11220a67c88212ecedde4021e1de4802e563baef9df921d2", - "sha256:d94f535df9f539615bc3dbbef185ded3b609373bb44ca1afffcabac70202678a", - "sha256:d98d2a8283c9928a9e5adf2f3c0181e095579e9732e1613aaa55d386e2bcb6c5", - "sha256:dc24737d24ce0de762bee9c2a884639819485f679bbac8ab5be9c161ef6f9b2c", - "sha256:e08fe1731f5429435b8dea1db9663f9ed1812915ff803fc9991c7c4841ed62ad", - "sha256:e09cdf5aad507c8faa30d97884cc42932ed3a9c2b7f22cc3ccc607bae03981b3", - "sha256:e152c26ffc30331e9d57591fc4c05453c209aa20ba299d1deb7173f7d1958c22", - "sha256:e1b8f5e2f9637492b0da4d51f78ecb17786e61d6c461ead8542c944750faf4f9", - "sha256:e39cacee70a98758f9b2da53ee175378f07c60113b1fa4fae40cbaee5583181e", - "sha256:e64442aba81ed4df1ca494b87bf818569a1280acaa73071c68014f7a884e83f1", - "sha256:e7dcb73f683c155885a3488646fcead3a895765fed16e93c9b80000bc69e96cb", - "sha256:ecdcb0d4e9b08b739035f57a09330efc6f464bd7f942b63897395d996ca6ebd5", - "sha256:ed90a9de4431cbfb2f3b2ef0c5fd356e61c85117b2be4db3eae28cb409f6e2d5", - "sha256:f1c23527f8e13f526fededbb96f2e7888f179fe27c51d41c2724f7059b75b2fa", - "sha256:f47d5f10922cf7f7dfcd1406bd0926cef6d866a75953c3745502dffd7ac197dd", - "sha256:fe0820d169635e41c14a5d21514282e0b93347878666ec9d5d3bf0eed0649948", - "sha256:ff66014687598823b6b23751884b4aa67eb934445406d95894dfc60cb7bfcc18" + "sha256:016c412118e1c23fef3a1eada4f83ae6e8844fd91986b2e066fc1b0013cdd9ae", + "sha256:01f7cbe88d22440b6594c955e37312d932fd632ffed1a86d0c361503ca82cc9d", + "sha256:08fc250b5552ee97ceeae0f52d8b04f360291285fc7437f13daa516ce38fdbc6", + "sha256:0c466710871d0026c190fc4141e810cf9d9affbf4935e1d273fbdc7d7cda6143", + "sha256:1074f1a6f23e28b983c96142f2d45be03ec55d93035b471c26889a7ad2365db3", + "sha256:12f3621a46cdc7a9ba8080422262398a91762a581d27e0647746588d3f995c88", + "sha256:2c2fdc855149efe7cdcc2a01ca02bfa24761c640203ea94df467f3baf19078be", + "sha256:316498b642c00401370b2156b5233b256f9b33799e0a8d9d0b8a7da217a20fca", + "sha256:341221e2f2866a5960e6f8610f4cbac0bb13097f3b1a289aa55aba984fc0d969", + "sha256:34b040e095e1671df0c095ec0b04fc4ebb19c4c160f87c2b55c079b16b1a6b00", + "sha256:34e95ffb0a68bffbc3b437f2d1f25fc916fef3df5cdeed0992da5f42fae9b807", + "sha256:39b03045c71f761aee96a12ebfbc2f4be89e724ff6f5e31c2574c1a0e2add8bd", + "sha256:3b93043b14ba7eb08c57afca19751658ece1cfa2f0b7b1fb5c7a41452fbb8482", + "sha256:47f7aa217b25833cd6f0e72b0d224be55393c2692b4f5e0561cb3beeb10296e9", + "sha256:49210feb0be8051a64d71691f0acbfbedc33e149f0a5d6e271fddf6a12493fed", + "sha256:4d00b91c77ceb064c9b0459f0d6ea5bfdbc53ea9e17cf75731e151ef25a830c7", + "sha256:4ed00f96e147f40b565fe7530d1da0b0f3ab803d5dd5b683834500fa5d195ec4", + "sha256:5134d33286c045393c7beb51be29754647cec5ebc051cf82799c5ce9820a2ca2", + "sha256:524d78673518dcd352a91541ecd2839c65af92dc883321c2109ef6e5cd22ef23", + "sha256:52896e22115c97f1c829db32aa2760b0d61839cfe08b168c2b1d82f31dbc5f55", + "sha256:54c377893f2cbbffe39abcff5ff2e917b082c364521fa079305f6f064e1a24a9", + "sha256:55b6163dac53ef1e5d834297810c178050bd0548a4136cd4e0f56402185916ca", + "sha256:599d3f6fbef31933b96e2d906b0f169b3371ff79ea6aaf6ecd76c947a3508a3d", + "sha256:5effd87c7d363890259eac16c56a4e8da307286012c076223997f8cc4a8c435b", + "sha256:66413c50d510e5bcb0afc79880d1693a2185bcea003600ed898ada31338c004e", + "sha256:695939036a320f4329ccf1627edefbbb67cc7892b8222d297b0dd2313742bfee", + "sha256:6c2216d8b6a6d019c6f4b1ad55f890e5e77eb089309ffc05b6911c09349e7474", + "sha256:6dd1cf2995fdbd64fc0802313e8323f5fa18994d51af059b5b8862b73b5e53f0", + "sha256:6fcfbf435eebf8a1765c6d1f46821740ebe9f54f815a05c8fc30d789ef43cb12", + "sha256:704d939656e21b073bfcddd7228b29e0e8a93dd27b54240eaafc0b9a631629a6", + "sha256:711bc52cb98e7892c03e9b669bebd89c0a890a90dbc6d5bb2c47f30239bac6e9", + "sha256:74731c9e423c93cbe791f60c27030b6af6a948cef67deca079da6cd1bb583a8e", + "sha256:7761cacb8745093062695b11574effea69db636c2fd0a9269a1f0183712927b4", + "sha256:7b16250238de8dafca225647608dddc7bbb5dce3dd53b4d8e63c1cc287394c2f", + "sha256:7c051fe37c96b9878f37fa58906cb53ecd13dcb7341d3a85f1e2e2f6b10782d9", + "sha256:7d43ac9c7eeda5100fb0a7152fab7099c9cf9e5abd3bb36928eb98c7d7a339c6", + "sha256:81d1a7303bd02ca1c5be4aacd4db73593f573ba8e0c543c04c6da6275fd7a47e", + "sha256:8a06a0c02f5606330e8f2e2f3b7949877ca7e4024fa2bff5a4506bec66c49ec7", + "sha256:8fd6e191b92a10310f5a6cfe10d6f839d79d192fb02480bda325286bd1c7b385", + "sha256:943f208840777f34312c103a2d1caab02d780c4e9be26b3714acf6c4715ba7e1", + "sha256:9b87b23570565a6ddaa9244d87811c2ee9cffb02a753c8a2da9c077283d85845", + "sha256:a6cd6f1db75eb07332bd3710f58f5fce4967eadbf751bad653842750a61bda62", + "sha256:a966d5304b7d90c45c404914e06bbf02c5bf7e99685c6c12f0047ef2aa837142", + "sha256:a9c2885b4a8e6e39db5662d8b02ca6dcec796a45e48c2de12552841f061692ba", + "sha256:b0cfe925610f2fd59555bb7fc37bd739e4b197d33f2a8b2fae7b9c0c6640318c", + "sha256:b38a96b3eed8edc515b38257f03216f382c4389d022a8834667e2bc63c0c0c31", + "sha256:b8a03af1ce79b902a43f5f694c4ca8d92c2a4195db0966f08f266549e2fc49bc", + "sha256:bb869707d8e30645ed6766e44098600ca6cdf7989c22a3ea2b7966bb1d98d4b2", + "sha256:be1d2ce7e269215c3ee9a215e296b7a744aff4f39233486d2c4d77f5f0c561a6", + "sha256:c0640b4e9d008e13956b004d1971a23377b3d45491f87082161c92efb1e6c0d6", + "sha256:c09956606c08c4a7c6178a04ba2dd9388fcc5db32002ade9c9bc865ab156ab6d", + "sha256:c184ec5be465c0319440734491e1aa4709b5f3ba75fdfc9dbbc2ae715a7f6829", + "sha256:c1a70c51da9fa95bd75c167edb2eb3f3c4d27bc4ddd29e588f21649d014ec0b7", + "sha256:c29e758f0e734e1e90357ae01ec9c6daf19ff60a051192fe110d8fb25c62600e", + "sha256:c6258a3663780ae47ba73d43eb63c79c40ffddfb764e09b56df33be2f9479837", + "sha256:cafa52873ae12baa512a8721afc20de67a36886baae6a5f394ddef0ce9391f91", + "sha256:cd6a4afb20fb3c26a7bfd4611a0bbb24d93cbd746f5eb881f114b5e38fd55501", + "sha256:cdb87309de97c63cb9a69132e1cb16be470e58cffdfbad68fdd1dc292b22a840", + "sha256:d07d06dba5b5f7d80f9cc45501456e440f759fe79f9895922ed486237ac378a8", + "sha256:d3a51901066696c4af38c6c63a1f0aeffd5e282367ff475de8c191ec9609b56d", + "sha256:d5571b6978750601f783cea07fb6b666837010ca57e5cefa389c1d456f6222e2", + "sha256:d86c35d94b5499689354ccbc48438a79f449481ee6300f3e905748edceed78e7", + "sha256:dc0cff74cd36d7e1edba91baa09622c35a8a57025f2f2b7a41e3f83b1db73186", + "sha256:dc24d245026a72d9b4953729d31813edd4bd4e5c13622d96e27c284942d33f24", + "sha256:dca34367a4e77fcab0693e603a959878eaf2351585e7d752cac544bc6b2dee46", + "sha256:e2961b05f9c04a53da8bfc72f1910b6aec7205fcf3ac9c036d24619979bbee4b", + "sha256:e7fac06a539daef4fcf5d8288d0d21b412f9b750454cd5a3cf90484665db442a", + "sha256:eac0a143ef4f28f49670bf89cb15847eb80b375d55eba401ca2f777cd425f338", + "sha256:ef888f48eb9203ee1e04b9fb27429017b290fb916f1e7826c2f7808c88798394", + "sha256:f3055510fdfdb1775bc8baa359783022f70bb553f2d46e153c094dfcb08578ff", + "sha256:fa7e202feb683dad74f00dea066690448d0cfa310f8a277db06ec8eb466601b5", + "sha256:fc28e8d85d392a06434e9a934908d97e2cf453d69488d2bcd0bfb881497fd975", + "sha256:fd7bb378d82b88387dc10227cfd964f6273eb083e05299e9b97cbe075da12d11", + "sha256:ffcc8394123ea8d43fff8e5d000095fe7741ce3f8988366c5c919c4f5eb179d3" ], "markers": "python_version >= '3.7'", - "version": "==4.2.0" + "version": "==4.3.3" }, "python-dateutil": { "hashes": [ @@ -138,6 +153,7 @@ }, "pyyaml": { "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", @@ -149,26 +165,32 @@ "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" ], @@ -185,11 +207,11 @@ }, "setuptools": { "hashes": [ - "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82", - "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57" + "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54", + "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75" ], "markers": "python_version >= '3.7'", - "version": "==65.3.0" + "version": "==65.6.3" }, "six": { "hashes": [ @@ -201,11 +223,11 @@ }, "urllib3": { "hashes": [ - "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", - "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" + "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", + "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", - "version": "==1.26.12" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.13" } }, "develop": {} From f51f1bed210138ffcf81856c2f9ce512df19e90e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 5 Dec 2022 12:43:20 -0500 Subject: [PATCH 388/417] Bump from python:3.10.8-alpine3.16 to python:3.10.8-alpine3.17 This updates the Docker image from Alpine 3.16 to 3.17 to keep us on the latest supported release of Alpine Linux. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 44b979f..98b24ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.8-alpine3.16 as compile-stage +FROM python:3.10.8-alpine3.17 as compile-stage ### # For a list of pre-defined annotation keys and value types see: @@ -36,7 +36,7 @@ COPY src/Pipfile src/Pipfile.lock ./ # VIRTUAL_ENV environment variable if it is set. RUN pipenv sync --clear --verbose -FROM python:3.10.8-alpine3.16 as build-stage +FROM python:3.10.8-alpine3.17 as build-stage ### # Unprivileged user setup variables From fc2f011b13fc8a17cc16e52741767aa6631a831a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 8 Dec 2022 13:01:34 -0500 Subject: [PATCH 389/417] Update the Python Docker images used to Python 3.11.1 This entails bumping the tag from python:3.10.8-alpine3.17 to python:3.11.1-alpine3.17. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98b24ba..105e721 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.8-alpine3.17 as compile-stage +FROM python:3.11.1-alpine3.17 as compile-stage ### # For a list of pre-defined annotation keys and value types see: @@ -36,7 +36,7 @@ COPY src/Pipfile src/Pipfile.lock ./ # VIRTUAL_ENV environment variable if it is set. RUN pipenv sync --clear --verbose -FROM python:3.10.8-alpine3.17 as build-stage +FROM python:3.11.1-alpine3.17 as build-stage ### # Unprivileged user setup variables From e3fc00afa34d2aeb95ef5dc62e23aa91ffe39c01 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 8 Dec 2022 13:08:28 -0500 Subject: [PATCH 390/417] Install base Python packages into system Python Since the Docker images we are using do not use an operating system package version of Python we can install these packages with no ill effects. --- Dockerfile | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 105e721..d7bcdc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,11 +16,20 @@ ARG CISA_USER="cisa" ENV CISA_HOME="/home/${CISA_USER}" ENV VIRTUAL_ENV="${CISA_HOME}/.venv" -# Install pipenv to manage installing the Python dependencies into a created -# Python virtual environment. This is done separately from the virtual -# environment so that pipenv and its dependencies are not installed in the -# Python virtual environment used in the final image. -RUN python3 -m pip install --no-cache-dir --upgrade pipenv==2022.11.25 \ +# Install base Python requirements and then install pipenv to manage installing +# the Python dependencies into a created Python virtual environment. This is +# done separately from the virtual environment so that pipenv and its +# dependencies are not installed in the Python virtual environment used in the +# final image. +# +# Please note that we only install the base Python requirements (pip, +# setuptools, and wheel) pre-venv because this Docker image is using Python +# built from source and not a system Python package. +RUN python3 -m pip install --no-cache-dir --upgrade \ + pip==22.3.1 \ + setuptools==65.6.3 \ + wheel==0.38.4 \ + && python3 -m pip install --no-cache-dir --upgrade pipenv==2022.11.25 \ # Manually create Python virtual environment for the final image && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment From e830d4d2d4f612c8bb9aa7d07fa122641643cccf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:59:06 -0500 Subject: [PATCH 391/417] Bump core Python dependencies - Bump pipenv from 2022.11.25 to 2022.12.19 - Bump setuptools from 65.6.3 to 65.7.0 --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d7bcdc2..faca3a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,15 +27,15 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # built from source and not a system Python package. RUN python3 -m pip install --no-cache-dir --upgrade \ pip==22.3.1 \ - setuptools==65.6.3 \ + setuptools==65.7.0 \ wheel==0.38.4 \ - && python3 -m pip install --no-cache-dir --upgrade pipenv==2022.11.25 \ + && python3 -m pip install --no-cache-dir --upgrade pipenv==2022.12.19 \ # Manually create Python virtual environment for the final image && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ pip==22.3.1 \ - setuptools==65.6.3 \ + setuptools==65.7.0 \ wheel==0.38.4 # Install client-cert-update Python requirements From 4aceca40c7efee5cfa3209f06486f19e5341652f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 8 Dec 2022 16:02:10 -0500 Subject: [PATCH 392/417] Update Pipfile configuration Update the `python_full_version` to match our current Docker image tag. Update the Python dependencies by running `pipenv lock` in the `src/` directory. --- src/Pipfile | 2 +- src/Pipfile.lock | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Pipfile b/src/Pipfile index e249b04..5498437 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -10,4 +10,4 @@ docopt = ">=0.6.2" mongo-db-from-config = {file = "https://github.com/cisagov/mongo-db-from-config/archive/v0.1.0.tar.gz"} [requires] -python_full_version = "3.10.8" +python_full_version = "3.11.1" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index d356e8e..a86b50a 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "1b23c744955205467715f172f3ac673592c9e22e25291e01f2324322b471bc4c" + "sha256": "e6df03de8bcfb3076359431739c12f892df30c99b7f918ddca24c80db60847ad" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.10.8" + "python_full_version": "3.11.1" }, "sources": [ { @@ -18,19 +18,19 @@ "default": { "boto3": { "hashes": [ - "sha256:3c7315da16eb0b41823965e5ce55f99cb07e94680e0ed7830c581f505fb5bd15", - "sha256:933c88b189112a5fdd82d49ef00f95b9dd649d195e557a81aecb773a3e01c517" + "sha256:8aa3fd453a815240b85c01190ed0d9924622a1a061c93423b1443379eadd3c43", + "sha256:a23fe87e9bafca736620dd9d7e5977eeaaa27c3abf5800f7d8b7b7d08b4ad736" ], "index": "pypi", - "version": "==1.26.18" + "version": "==1.26.48" }, "botocore": { "hashes": [ - "sha256:26e86fce95049f6cc18b5611901549943c4c22522fa8a3b6b265404f673977b2", - "sha256:2aba44433b6eac6d3a12cf93f2985e2d7a843307c1a527042fc48dd09b273992" + "sha256:17eeb963b3ae4a3ea4a972bdf78c9925c3efd9e2232875bfc19043640d3dac14", + "sha256:c5f57c1f694e67c29c870426c0a7ebffce90099e794025e3d283db7f1ad7a65b" ], "markers": "python_version >= '3.7'", - "version": "==1.29.18" + "version": "==1.29.48" }, "dnspython": { "hashes": [ @@ -207,11 +207,11 @@ }, "setuptools": { "hashes": [ - "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54", - "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75" + "sha256:4d3c92fac8f1118bb77a22181355e29c239cabfe2b9effdaa665c66b711136d7", + "sha256:8ab4f1dbf2b4a65f7eec5ad0c620e84c34111a68d3349833494b9088212214dd" ], "markers": "python_version >= '3.7'", - "version": "==65.6.3" + "version": "==65.7.0" }, "six": { "hashes": [ @@ -223,11 +223,11 @@ }, "urllib3": { "hashes": [ - "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", - "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" + "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", + "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.13" + "version": "==1.26.14" } }, "develop": {} From 841216512af7950257cbbd2315285e4d5632531d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 13 Jan 2023 15:29:52 -0500 Subject: [PATCH 393/417] Use consistent formatting for chained RUN commands This ensures that any packages being installed are easily distinguishable from the rest of the command syntax. --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index faca3a3..95bfab4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,8 @@ RUN python3 -m pip install --no-cache-dir --upgrade \ pip==22.3.1 \ setuptools==65.7.0 \ wheel==0.38.4 \ - && python3 -m pip install --no-cache-dir --upgrade pipenv==2022.12.19 \ + && python3 -m pip install --no-cache-dir --upgrade \ + pipenv==2022.12.19 \ # Manually create Python virtual environment for the final image && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment From c4455dbe43532fcde283175440b3201616b0bd5c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 13 Jan 2023 15:34:26 -0500 Subject: [PATCH 394/417] Update comments for COPY commands that chown Explain why we are using the `--chown` flag for COPY commands. --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95bfab4..198fe90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,11 +64,13 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -# Copy in the Python virtual environment we created in the compile stage +# Copy in the Python virtual environment we created in the compile stage and +# ensure the unprivileged user owns all of the files contained within the venv. COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" -# Put this just before we change users because the copy (and every +# Copy in the core logic for the Docker image and ensure the unprivileged user +# owns the files. We put this just before we change users because the copy (and every # step after it) will often be rerun by Docker. COPY --chown=${CISA_USER}:${CISA_GROUP} src/email-update.py src/body.txt src/body.html ${CISA_HOME}/ From a086c2c1a83783de57121ac34e5ea91e8dd4ee69 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 13 Jan 2023 17:10:14 -0500 Subject: [PATCH 395/417] Improve the comments for copying the venv Mention why the PATH is being changed and improve the wording of the explanation for copying the virtual environment files. --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 198fe90..3412b66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,8 +65,9 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} # Copy in the Python virtual environment we created in the compile stage and -# ensure the unprivileged user owns all of the files contained within the venv. +# ensure the unprivileged user owns the files. COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# Update the PATH so that the virtual environment has priority. ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" # Copy in the core logic for the Docker image and ensure the unprivileged user From 3aaf67f3b69d11d957cda1111bac4054496307b9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:20:57 -0500 Subject: [PATCH 396/417] Bump version from 0.0.2 to 0.1.0 --- README.md | 16 ++++++++-------- src/version.txt | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ee3413a..5d4ee10 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ certificates. To run the `cisagov/client-cert-update` image via Docker: ```console -docker run cisagov/client-cert-update:0.0.2 +docker run cisagov/client-cert-update:0.1.0 ``` ### Running with Docker Compose ### @@ -34,7 +34,7 @@ docker run cisagov/client-cert-update:0.0.2 services: update: - image: cisagov/client-cert-update:0.0.2 + image: cisagov/client-cert-update:0.1.0 init: true environment: - AWS_CONFIG_FILE=path/to/aws_config @@ -82,7 +82,7 @@ environment variables. See the services: update: - image: cisagov/client-cert-update:0.0.2 + image: cisagov/client-cert-update:0.1.0 init: true environment: - AWS_CONFIG_FILE=/run/secrets/aws_config @@ -121,7 +121,7 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/client-cert-update:0.0.2 + docker pull cisagov/client-cert-update:0.1.0 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -130,11 +130,11 @@ environment variables. See the The images of this container are tagged with [semantic versions](https://semver.org). It is recommended that most users use a version -tag (e.g. `:0.0.2`). +tag (e.g. `:0.1.0`). | Image:tag | Description | |-----------|-------------| -|`cisagov/client-cert-update:0.0.2`| An exact release version. | +|`cisagov/client-cert-update:0.1.0`| An exact release version. | |`cisagov/client-cert-update:0.0`| The most recent release matching the major and minor version numbers. | |`cisagov/client-cert-update:0`| The most recent release matching the major version number. | |`cisagov/client-cert-update:edge` | The most recent image built from a merge into the `develop` branch of this repository. | @@ -196,7 +196,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/client-cert-update:0.0.2 \ + --tag cisagov/client-cert-update:0.1.0 \ https://github.com/cisagov/client-cert-update.git#develop ``` @@ -227,7 +227,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/client-cert-update:0.0.2 . + --tag cisagov/client-cert-update:0.1.0 . ``` ## Contributing ## diff --git a/src/version.txt b/src/version.txt index 3b93d0b..3dc1f76 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.2" +__version__ = "0.1.0" From 91e69a909abcc9fe10c1213d5c1bacc1835ecf85 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:22:16 -0500 Subject: [PATCH 397/417] Bump version from 0.1.0 to 0.1.0-rc.1 --- README.md | 16 ++++++++-------- src/version.txt | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5d4ee10..f2a1232 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ certificates. To run the `cisagov/client-cert-update` image via Docker: ```console -docker run cisagov/client-cert-update:0.1.0 +docker run cisagov/client-cert-update:0.1.0-rc.1 ``` ### Running with Docker Compose ### @@ -34,7 +34,7 @@ docker run cisagov/client-cert-update:0.1.0 services: update: - image: cisagov/client-cert-update:0.1.0 + image: cisagov/client-cert-update:0.1.0-rc.1 init: true environment: - AWS_CONFIG_FILE=path/to/aws_config @@ -82,7 +82,7 @@ environment variables. See the services: update: - image: cisagov/client-cert-update:0.1.0 + image: cisagov/client-cert-update:0.1.0-rc.1 init: true environment: - AWS_CONFIG_FILE=/run/secrets/aws_config @@ -121,7 +121,7 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/client-cert-update:0.1.0 + docker pull cisagov/client-cert-update:0.1.0-rc.1 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -130,11 +130,11 @@ environment variables. See the The images of this container are tagged with [semantic versions](https://semver.org). It is recommended that most users use a version -tag (e.g. `:0.1.0`). +tag (e.g. `:0.1.0-rc.1`). | Image:tag | Description | |-----------|-------------| -|`cisagov/client-cert-update:0.1.0`| An exact release version. | +|`cisagov/client-cert-update:0.1.0-rc.1`| An exact release version. | |`cisagov/client-cert-update:0.0`| The most recent release matching the major and minor version numbers. | |`cisagov/client-cert-update:0`| The most recent release matching the major version number. | |`cisagov/client-cert-update:edge` | The most recent image built from a merge into the `develop` branch of this repository. | @@ -196,7 +196,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/client-cert-update:0.1.0 \ + --tag cisagov/client-cert-update:0.1.0-rc.1 \ https://github.com/cisagov/client-cert-update.git#develop ``` @@ -227,7 +227,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/client-cert-update:0.1.0 . + --tag cisagov/client-cert-update:0.1.0-rc.1 . ``` ## Contributing ## diff --git a/src/version.txt b/src/version.txt index 3dc1f76..39bfdd9 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.1.0" +__version__ = "0.1.0-rc.1" From d488d4fb8198c6e81ab23a8718556170fa5d76fa Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:09:47 -0500 Subject: [PATCH 398/417] Update dependabot configuration Add another `pip` package ecosystem for the `src/` directory so the `pipenv` configuration is seen and maintained by dependabot. --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d560ad9..81bb96f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -39,6 +39,11 @@ updates: schedule: interval: "weekly" + - package-ecosystem: "pip" + directory: "/src" + schedule: + interval: "weekly" + - package-ecosystem: "terraform" directory: "/" schedule: From 0bb91c5654db04ad15182dc8c0e491ae0664394f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Jan 2023 13:06:42 -0500 Subject: [PATCH 399/417] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back to be updated independently to v6. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25f8d1d..2764bb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.32.2 + rev: v0.33.0 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.28.0 + rev: v1.29.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.18.4 + rev: 0.21.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.20.0 + rev: v3.0.2 hooks: - id: validate_manifest @@ -88,25 +88,25 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 22.12.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 6.0.0 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.990 + rev: v0.991 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.2.0 + rev: v3.3.1 hooks: - id: pyupgrade @@ -119,7 +119,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.76.0 + rev: v1.77.0 hooks: - id: terraform_fmt - id: terraform_validate From 313e79263e8b6a1fbd311e5b9543a2b9a99ab93b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 2 Feb 2023 13:56:59 -0500 Subject: [PATCH 400/417] Add a security label --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 500b000..5b16492 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -53,6 +53,9 @@ - color: "ef476c" description: This issue is a request for information or needs discussion name: question +- color: "d73a4a" + description: This issue or pull request addresses a security issue + name: security - color: "00008b" description: This issue or pull request adds or otherwise modifies test code name: test From da15398ef27c2022f4fd1e88215cc0a72c9751b7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 6 Dec 2022 11:24:07 -0500 Subject: [PATCH 401/417] Add package write permission to workflow When Dependabot creates a PR it requires this permission in order to push Docker images to ghcr.io. --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f3a541..af07e26 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -345,6 +345,10 @@ jobs: runs-on: ubuntu-latest needs: [lint, prepare, test] if: github.event_name != 'pull_request' + # When Dependabot creates a PR it requires this permission in + # order to push Docker images to ghcr.io. + permissions: + packages: write steps: - name: Login to Docker Hub uses: docker/login-action@v2 From 4e5287fe05d6ba47aedce58cc851a69fe495e769 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 3 Dec 2022 15:05:50 -0500 Subject: [PATCH 402/417] Update wheel along with pip and setuptools Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e3cf8aa..3635be0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,13 +57,16 @@ ENV DEPS \ RUN apk --no-cache --quiet add ${DEPS} ### -# Make sure pip and setuptools are the latest versions +# Make sure pip, setuptools, and wheel are the latest versions # # Note that we use pip --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip install --no-cache-dir --upgrade pip setuptools +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel WORKDIR ${CISA_HOME} From 83beae5295821391ffc88f7190514b0881520b95 Mon Sep 17 00:00:00 2001 From: Jeremy Morrow Date: Thu, 9 Feb 2023 21:24:50 -0500 Subject: [PATCH 403/417] Specify pip3 In this case it doesn't matter because we are starting from a Python3-specific base container, but other projects that use this skeleton may not be. Specifying pip3 ensures that the Python 2 version of pip is not called by mistake. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3635be0..9f4e9c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,7 @@ RUN apk --no-cache --quiet add ${DEPS} ### # Make sure pip, setuptools, and wheel are the latest versions # -# Note that we use pip --no-cache-dir to avoid writing to a local +# Note that we use pip3 --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### @@ -73,14 +73,14 @@ WORKDIR ${CISA_HOME} ### # Install Python dependencies # -# Note that we use pip --no-cache-dir to avoid writing to a local +# Note that we use pip3 --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### RUN wget --output-document sourcecode.tgz \ https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz \ && tar --extract --gzip --file sourcecode.tgz --strip-components=1 \ - && pip install --no-cache-dir --requirement requirements.txt \ + && pip3 install --no-cache-dir --requirement requirements.txt \ && ln -snf /run/secrets/quote.txt src/example/data/secret.txt \ && rm sourcecode.tgz From d6bf3c0e641a60def66cefea4d3185b540a7eae2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 22 Nov 2022 12:13:43 -0500 Subject: [PATCH 404/417] Specify the default group to run under in the Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f4e9c2..b32ee0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,7 +88,7 @@ RUN wget --output-document sourcecode.tgz \ # Prepare to run ### ENV ECHO_MESSAGE="Hello World from Dockerfile" -USER ${CISA_USER} +USER ${CISA_USER}:${CISA_GROUP} EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] From 73a526f57f0756ac2d622d0e714a76515344f5f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Feb 2023 04:55:35 +0000 Subject: [PATCH 405/417] Bump docker/build-push-action from 3 to 4 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 3 to 4. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af07e26..08d67c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -248,7 +248,7 @@ jobs: run: mkdir -p dist - name: Build image id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} @@ -380,7 +380,7 @@ jobs: run: ./buildx-dockerfile.sh - name: Build and push platform images to registries id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} From 7aab0747e550ee1b81c016f171f2fcb5f26f42e0 Mon Sep 17 00:00:00 2001 From: Jeremy Morrow Date: Fri, 10 Feb 2023 10:26:10 -0500 Subject: [PATCH 406/417] Specify pip3 in the file where it got blown away by a rebase --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b32ee0f..1516787 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,7 +63,7 @@ RUN apk --no-cache --quiet add ${DEPS} # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip install --no-cache-dir --upgrade \ +RUN pip3 install --no-cache-dir --upgrade \ pip \ setuptools \ wheel From 7140cbf28f0b586cbacf3c1c433338e719546f8a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:12:19 -0500 Subject: [PATCH 407/417] Update Python package versions in the Dockerfile - Bump pip from 22.3.1 to 23.0 - Bump setuptools from 65.7.0 to 67.3.1 - Bump pipenv from 2022.12.19 to 2023.2.4 --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3412b66..e49e448 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,17 +26,17 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # setuptools, and wheel) pre-venv because this Docker image is using Python # built from source and not a system Python package. RUN python3 -m pip install --no-cache-dir --upgrade \ - pip==22.3.1 \ - setuptools==65.7.0 \ + pip==23.0 \ + setuptools==67.3.1 \ wheel==0.38.4 \ && python3 -m pip install --no-cache-dir --upgrade \ - pipenv==2022.12.19 \ + pipenv==2023.2.4 \ # Manually create Python virtual environment for the final image && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ - pip==22.3.1 \ - setuptools==65.7.0 \ + pip==23.0 \ + setuptools==67.3.1 \ wheel==0.38.4 # Install client-cert-update Python requirements From b9e39bb395873abdab0d65fd6cdd021ea19f95f8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 13 Feb 2023 12:51:44 -0500 Subject: [PATCH 408/417] Bump cisagov/mongo-db-from-config from 0.1.0 to 0.2.0 --- src/Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Pipfile b/src/Pipfile index 5498437..ede2cf9 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -7,7 +7,7 @@ name = "pypi" # Minimum version for IMDSv2 support boto3 = ">=1.13.23" docopt = ">=0.6.2" -mongo-db-from-config = {file = "https://github.com/cisagov/mongo-db-from-config/archive/v0.1.0.tar.gz"} +mongo-db-from-config = {file = "https://github.com/cisagov/mongo-db-from-config/archive/v0.2.0.tar.gz"} [requires] python_full_version = "3.11.1" From ecad92ac01b73ceef95c4ece73c6e21645390eff Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:17:42 -0500 Subject: [PATCH 409/417] Update the `pipenv` lockfile This was done by running `pipenv lock` in the `src/` directory. --- src/Pipfile.lock | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/Pipfile.lock b/src/Pipfile.lock index a86b50a..3df0cf3 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e6df03de8bcfb3076359431739c12f892df30c99b7f918ddca24c80db60847ad" + "sha256": "171c8f53d21c69affa2e2f4de9c7a63b7070bd4ff09c1a71a225ada86e7cec55" }, "pipfile-spec": 6, "requires": { @@ -18,27 +18,27 @@ "default": { "boto3": { "hashes": [ - "sha256:8aa3fd453a815240b85c01190ed0d9924622a1a061c93423b1443379eadd3c43", - "sha256:a23fe87e9bafca736620dd9d7e5977eeaaa27c3abf5800f7d8b7b7d08b4ad736" + "sha256:5a9d19cdd8dcec679c483408f208027e01ab2087cbc66787790036087b6737de", + "sha256:6c4845243d1896019646d649f1f0ff4042cedcc5db3ecfba3dc2d611ea11cd08" ], "index": "pypi", - "version": "==1.26.48" + "version": "==1.26.71" }, "botocore": { "hashes": [ - "sha256:17eeb963b3ae4a3ea4a972bdf78c9925c3efd9e2232875bfc19043640d3dac14", - "sha256:c5f57c1f694e67c29c870426c0a7ebffce90099e794025e3d283db7f1ad7a65b" + "sha256:40406466f5c416b1f54bfbfc11aef90d783103f7ea77a1992dcaf1768ab04e12", + "sha256:783e7fa97bb5bf3759e4b333b8da2bcaffdb54828ea1d759b55329cc39003b98" ], "markers": "python_version >= '3.7'", - "version": "==1.29.48" + "version": "==1.29.71" }, "dnspython": { "hashes": [ - "sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e", - "sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f" + "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9", + "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46" ], - "markers": "python_version >= '3.6' and python_version < '4.0'", - "version": "==2.2.1" + "markers": "python_version >= '3.7' and python_version < '4.0'", + "version": "==2.3.0" }, "docopt": { "hashes": [ @@ -56,12 +56,11 @@ "version": "==1.0.1" }, "mongo-db-from-config": { - "file": "https://github.com/cisagov/mongo-db-from-config/archive/v0.1.0.tar.gz", + "file": "https://github.com/cisagov/mongo-db-from-config/archive/v0.2.0.tar.gz", "hashes": [ - "sha256:0655651e5945e327e2b6baf17aafe6c603635278f8ac3c208c51f7a0360ab44c", - "sha256:f42787f5aba88ac63b20b70439a6fa09d60daeff9169991d340b49b4d777ce16" + "sha256:51a633ea17cc1e9042fb908b0ea65cac75b188b1c750bf8a5dc0dc355d285892" ], - "version": "==0.1.0" + "version": "==0.2.0" }, "pymongo": { "hashes": [ @@ -207,11 +206,11 @@ }, "setuptools": { "hashes": [ - "sha256:4d3c92fac8f1118bb77a22181355e29c239cabfe2b9effdaa665c66b711136d7", - "sha256:8ab4f1dbf2b4a65f7eec5ad0c620e84c34111a68d3349833494b9088212214dd" + "sha256:23c86b4e44432bfd8899384afc08872ec166a24f48a3f99f293b0a557e6a6b5d", + "sha256:daec07fd848d80676694d6bf69c009d28910aeece68a38dbe88b7e1bb6dba12e" ], "markers": "python_version >= '3.7'", - "version": "==65.7.0" + "version": "==67.3.1" }, "six": { "hashes": [ From cfd5959d4ae93835495c3b6abc62faa438106fd7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Jan 2023 12:22:11 -0500 Subject: [PATCH 410/417] Correct the minor version tag in the README This was missed when the version was bumped. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2a1232..867f02a 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ tag (e.g. `:0.1.0-rc.1`). | Image:tag | Description | |-----------|-------------| |`cisagov/client-cert-update:0.1.0-rc.1`| An exact release version. | -|`cisagov/client-cert-update:0.0`| The most recent release matching the major and minor version numbers. | +|`cisagov/client-cert-update:0.1`| The most recent release matching the major and minor version numbers. | |`cisagov/client-cert-update:0`| The most recent release matching the major version number. | |`cisagov/client-cert-update:edge` | The most recent image built from a merge into the `develop` branch of this repository. | |`cisagov/client-cert-update:nightly` | A nightly build of the `develop` branch of this repository. | From bd63992e83f3935e17de167716318da5073eceb3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 13 Feb 2023 13:00:53 -0500 Subject: [PATCH 411/417] Bump python from 3.11.1 to 3.11.2 This includes the Docker images used as well as the version specified in the Pipfile. --- Dockerfile | 4 ++-- src/Pipfile | 2 +- src/Pipfile.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e49e448..3821bd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.1-alpine3.17 as compile-stage +FROM python:3.11.2-alpine3.17 as compile-stage ### # For a list of pre-defined annotation keys and value types see: @@ -46,7 +46,7 @@ COPY src/Pipfile src/Pipfile.lock ./ # VIRTUAL_ENV environment variable if it is set. RUN pipenv sync --clear --verbose -FROM python:3.11.1-alpine3.17 as build-stage +FROM python:3.11.2-alpine3.17 as build-stage ### # Unprivileged user setup variables diff --git a/src/Pipfile b/src/Pipfile index ede2cf9..8b1d1a1 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -10,4 +10,4 @@ docopt = ">=0.6.2" mongo-db-from-config = {file = "https://github.com/cisagov/mongo-db-from-config/archive/v0.2.0.tar.gz"} [requires] -python_full_version = "3.11.1" +python_full_version = "3.11.2" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 3df0cf3..3f5d628 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "171c8f53d21c69affa2e2f4de9c7a63b7070bd4ff09c1a71a225ada86e7cec55" + "sha256": "cb8e9066bc4b33bc4576334ae67dc503d7c2694a75ff3e1de66693d390af2ab8" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.11.1" + "python_full_version": "3.11.2" }, "sources": [ { From 861bd357fe1d0372d044e5329d4914c5382c7413 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 13 Feb 2023 13:28:30 -0500 Subject: [PATCH 412/417] Update the `mypy` pre-commit hook configuration Add some additional typing packges to improve analysis. Since this is a Docker project it doesn't make sense to install packages into the local Python environment for analysis. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54c3114..caee308 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -116,6 +116,8 @@ repos: hooks: - id: mypy additional_dependencies: + - boto3-stubs + - types-docopt - types-PyYAML - repo: https://github.com/asottile/pyupgrade rev: v3.3.1 From 6fb7de7f093af8f97144c75ac94df3b1caff1f77 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:28:55 -0500 Subject: [PATCH 413/417] Bump version from 0.1.0-rc.1 to 0.1.0-rc.2 --- README.md | 16 ++++++++-------- src/version.txt | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 867f02a..041bd3c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ certificates. To run the `cisagov/client-cert-update` image via Docker: ```console -docker run cisagov/client-cert-update:0.1.0-rc.1 +docker run cisagov/client-cert-update:0.1.0-rc.2 ``` ### Running with Docker Compose ### @@ -34,7 +34,7 @@ docker run cisagov/client-cert-update:0.1.0-rc.1 services: update: - image: cisagov/client-cert-update:0.1.0-rc.1 + image: cisagov/client-cert-update:0.1.0-rc.2 init: true environment: - AWS_CONFIG_FILE=path/to/aws_config @@ -82,7 +82,7 @@ environment variables. See the services: update: - image: cisagov/client-cert-update:0.1.0-rc.1 + image: cisagov/client-cert-update:0.1.0-rc.2 init: true environment: - AWS_CONFIG_FILE=/run/secrets/aws_config @@ -121,7 +121,7 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/client-cert-update:0.1.0-rc.1 + docker pull cisagov/client-cert-update:0.1.0-rc.2 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -130,11 +130,11 @@ environment variables. See the The images of this container are tagged with [semantic versions](https://semver.org). It is recommended that most users use a version -tag (e.g. `:0.1.0-rc.1`). +tag (e.g. `:0.1.0-rc.2`). | Image:tag | Description | |-----------|-------------| -|`cisagov/client-cert-update:0.1.0-rc.1`| An exact release version. | +|`cisagov/client-cert-update:0.1.0-rc.2`| An exact release version. | |`cisagov/client-cert-update:0.1`| The most recent release matching the major and minor version numbers. | |`cisagov/client-cert-update:0`| The most recent release matching the major version number. | |`cisagov/client-cert-update:edge` | The most recent image built from a merge into the `develop` branch of this repository. | @@ -196,7 +196,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/client-cert-update:0.1.0-rc.1 \ + --tag cisagov/client-cert-update:0.1.0-rc.2 \ https://github.com/cisagov/client-cert-update.git#develop ``` @@ -227,7 +227,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/client-cert-update:0.1.0-rc.1 . + --tag cisagov/client-cert-update:0.1.0-rc.2 . ``` ## Contributing ## diff --git a/src/version.txt b/src/version.txt index 39bfdd9..2c5baa2 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.1.0-rc.1" +__version__ = "0.1.0-rc.2" From 685be832be4309a6f2ebd43faba87dac2a0ee68e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 15 Feb 2023 13:45:41 -0500 Subject: [PATCH 414/417] Store package versions as Docker environment variables Update the Docker configuration to use environment variables to house the versions of pip, pipenv, setuptools, and wheel that are installed directly in the Dockerfile. This will DRY out the configuration a bit as well as ensuring that the same versions are used for packages that are installed multiple times. The environment variable name format is cribbed from the official Python Docker image configuration: https://github.com/docker-library/python/blob/0a56f04a07495e0ed61663c73bc0c2e175773ddc/3.11/alpine3.17/Dockerfile#L132-L135 I tested without the leading `PYTHON_` in the names and only observed a failure for usage of `PIPENV_VERSION`. Following the above mentioned standard seemed like a reasonable approach as a result. Co-authored-by: Shane Frasier --- Dockerfile | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3821bd5..b1e826c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,13 @@ ARG CISA_USER="cisa" ENV CISA_HOME="/home/${CISA_USER}" ENV VIRTUAL_ENV="${CISA_HOME}/.venv" + +# Versions of the Python packages installed directly +ENV PYTHON_PIP_VERSION=23.0 +ENV PYTHON_SETUPTOOLS_VERSION=67.3.1 +ENV PYTHON_WHEEL_VERSION=0.38.4 +ENV PYTHON_PIPENV_VERSION=2023.2.4 + # Install base Python requirements and then install pipenv to manage installing # the Python dependencies into a created Python virtual environment. This is # done separately from the virtual environment so that pipenv and its @@ -26,18 +33,18 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # setuptools, and wheel) pre-venv because this Docker image is using Python # built from source and not a system Python package. RUN python3 -m pip install --no-cache-dir --upgrade \ - pip==23.0 \ - setuptools==67.3.1 \ - wheel==0.38.4 \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} \ && python3 -m pip install --no-cache-dir --upgrade \ - pipenv==2023.2.4 \ + pipenv==${PYTHON_PIPENV_VERSION} \ # Manually create Python virtual environment for the final image && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ - pip==23.0 \ - setuptools==67.3.1 \ - wheel==0.38.4 + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} # Install client-cert-update Python requirements WORKDIR /tmp From 019bd4a289faefe98b1e2eae6e10f77330556d85 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 15 Feb 2023 14:19:43 -0500 Subject: [PATCH 415/417] Sort Python package versions in the Dockerfile Since we regularly beat the drum about alphabetizing such things unless there are technical constraints we should definitely be alphabetizing these variables. Co-authored-by: dav3r --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b1e826c..7ee3db2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,9 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=23.0 +ENV PYTHON_PIPENV_VERSION=2023.2.4 ENV PYTHON_SETUPTOOLS_VERSION=67.3.1 ENV PYTHON_WHEEL_VERSION=0.38.4 -ENV PYTHON_PIPENV_VERSION=2023.2.4 # Install base Python requirements and then install pipenv to manage installing # the Python dependencies into a created Python virtual environment. This is From 778deef2af010d9d5e96adf04a55896b9df4bd32 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 15 Feb 2023 15:18:42 -0500 Subject: [PATCH 416/417] Bump version from 0.1.0-rc.2 to 0.1.0-rc.3 --- README.md | 16 ++++++++-------- src/version.txt | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 041bd3c..36e00a8 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ certificates. To run the `cisagov/client-cert-update` image via Docker: ```console -docker run cisagov/client-cert-update:0.1.0-rc.2 +docker run cisagov/client-cert-update:0.1.0-rc.3 ``` ### Running with Docker Compose ### @@ -34,7 +34,7 @@ docker run cisagov/client-cert-update:0.1.0-rc.2 services: update: - image: cisagov/client-cert-update:0.1.0-rc.2 + image: cisagov/client-cert-update:0.1.0-rc.3 init: true environment: - AWS_CONFIG_FILE=path/to/aws_config @@ -82,7 +82,7 @@ environment variables. See the services: update: - image: cisagov/client-cert-update:0.1.0-rc.2 + image: cisagov/client-cert-update:0.1.0-rc.3 init: true environment: - AWS_CONFIG_FILE=/run/secrets/aws_config @@ -121,7 +121,7 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/client-cert-update:0.1.0-rc.2 + docker pull cisagov/client-cert-update:0.1.0-rc.3 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -130,11 +130,11 @@ environment variables. See the The images of this container are tagged with [semantic versions](https://semver.org). It is recommended that most users use a version -tag (e.g. `:0.1.0-rc.2`). +tag (e.g. `:0.1.0-rc.3`). | Image:tag | Description | |-----------|-------------| -|`cisagov/client-cert-update:0.1.0-rc.2`| An exact release version. | +|`cisagov/client-cert-update:0.1.0-rc.3`| An exact release version. | |`cisagov/client-cert-update:0.1`| The most recent release matching the major and minor version numbers. | |`cisagov/client-cert-update:0`| The most recent release matching the major version number. | |`cisagov/client-cert-update:edge` | The most recent image built from a merge into the `develop` branch of this repository. | @@ -196,7 +196,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/client-cert-update:0.1.0-rc.2 \ + --tag cisagov/client-cert-update:0.1.0-rc.3 \ https://github.com/cisagov/client-cert-update.git#develop ``` @@ -227,7 +227,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/client-cert-update:0.1.0-rc.2 . + --tag cisagov/client-cert-update:0.1.0-rc.3 . ``` ## Contributing ## diff --git a/src/version.txt b/src/version.txt index 2c5baa2..015ce95 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.1.0-rc.2" +__version__ = "0.1.0-rc.3" From 840b8c1166e9e6f6c47072b06f67ba68382b5583 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 17 Feb 2023 18:04:10 -0500 Subject: [PATCH 417/417] Finalize version from 0.1.0-rc.3 to 0.1.0 --- README.md | 16 ++++++++-------- src/version.txt | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 36e00a8..5527232 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ certificates. To run the `cisagov/client-cert-update` image via Docker: ```console -docker run cisagov/client-cert-update:0.1.0-rc.3 +docker run cisagov/client-cert-update:0.1.0 ``` ### Running with Docker Compose ### @@ -34,7 +34,7 @@ docker run cisagov/client-cert-update:0.1.0-rc.3 services: update: - image: cisagov/client-cert-update:0.1.0-rc.3 + image: cisagov/client-cert-update:0.1.0 init: true environment: - AWS_CONFIG_FILE=path/to/aws_config @@ -82,7 +82,7 @@ environment variables. See the services: update: - image: cisagov/client-cert-update:0.1.0-rc.3 + image: cisagov/client-cert-update:0.1.0 init: true environment: - AWS_CONFIG_FILE=/run/secrets/aws_config @@ -121,7 +121,7 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/client-cert-update:0.1.0-rc.3 + docker pull cisagov/client-cert-update:0.1.0 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -130,11 +130,11 @@ environment variables. See the The images of this container are tagged with [semantic versions](https://semver.org). It is recommended that most users use a version -tag (e.g. `:0.1.0-rc.3`). +tag (e.g. `:0.1.0`). | Image:tag | Description | |-----------|-------------| -|`cisagov/client-cert-update:0.1.0-rc.3`| An exact release version. | +|`cisagov/client-cert-update:0.1.0`| An exact release version. | |`cisagov/client-cert-update:0.1`| The most recent release matching the major and minor version numbers. | |`cisagov/client-cert-update:0`| The most recent release matching the major version number. | |`cisagov/client-cert-update:edge` | The most recent image built from a merge into the `develop` branch of this repository. | @@ -196,7 +196,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/client-cert-update:0.1.0-rc.3 \ + --tag cisagov/client-cert-update:0.1.0 \ https://github.com/cisagov/client-cert-update.git#develop ``` @@ -227,7 +227,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/client-cert-update:0.1.0-rc.3 . + --tag cisagov/client-cert-update:0.1.0 . ``` ## Contributing ## diff --git a/src/version.txt b/src/version.txt index 015ce95..3dc1f76 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.1.0-rc.3" +__version__ = "0.1.0"