Merge pull request #453 from cloud-gov/update-resources

Update resource types to use hardened images

Dockerfile

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1.2
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # Install general dependencies
 RUN apt-get update \
@@ -10,13 +10,10 @@ RUN apt-get update \
   autoconf automake libgdbm-dev libncurses5-dev \
   libsqlite3-dev libtool libyaml-dev pkg-config libgmp-dev \
   libpq-dev libxi6 libjpeg-dev libpng-dev libtiff-dev libgif-dev \
-  libwebp-dev wget \
+  libwebp-dev wget python3 python3-dev python3-pip\
   # Ruby deps
   gawk bison sqlite3
 
-# Uses python3.8 by default
-RUN apt install -y python3 python3-pip
-
 # Deps for container hardening
 RUN ln -sf "/usr/share/zoneinfo/$SYSTEM_TIMEZONE" /etc/localtime
 COPY docker/ua-attach-config.sh .

ci/pipeline-dev.yml

@@ -12,17 +12,19 @@ env-cf: &env-cf
 python-image: &python-image
   platform: linux
   image_resource:
-    type: docker-image
+    type: registry-image
     source:
-      repository: python
-      tag: 3.8
+      aws_access_key_id: ((ecr-aws-key))
+      aws_secret_access_key: ((ecr-aws-secret))
+      repository: pages-python-v3.11
+      aws_region: us-gov-west-1
+      tag: latest
 
 cf-image: &cf-image
   platform: linux
   image_resource:
     type: registry-image
     source:
-      repository: 18fgsa/concourse-task
       aws_access_key_id: ((ecr-aws-key))
       aws_secret_access_key: ((ecr-aws-secret))
       repository: harden-concourse-task
@@ -38,8 +40,6 @@ test: &test
       run:
         dir: src
         path: ci/tasks/test.sh
-    params:
-      CC_TEST_REPORTER_ID: 449ef357d9e81688685bde0dcdd348bc764d717fa6a4c9c01562bd8037ceb665
 
 ############################
 #  JOBS
@@ -64,10 +64,35 @@ jobs:
         resource: pr-((git-branch))
         trigger: true
         passed: [set-pipeline]
+
+      - put: src
+        resource: pr-((git-branch))
+        params:
+          path: src
+          status: pending
+          base_context: concourse
+          context: test-pages-build-container-((deploy-env))
+
       - do: *test
 
+    on_success:
+      put: src
+      resource: pr-((git-branch))
+      params:
+        path: src
+        status: success
+        base_context: concourse
+        context: test-pages-build-container-((deploy-env))
+
     on_failure:
       in_parallel:
+        - put: src
+          resource: pr-((git-branch))
+          params:
+            path: src
+            status: failure
+            base_context: concourse
+            context: test-pages-build-container-((deploy-env))
         - put: slack
           params:
             text: |
@@ -227,11 +252,28 @@ resources:
 resource_types:
 
   - name: slack-notification
-    type: docker-image
+    type: registry-image
     source:
-      repository: cfcommunity/slack-notification-resource
+      aws_access_key_id: ((ecr_aws_key))
+      aws_secret_access_key: ((ecr_aws_secret))
+      repository: slack-notification-resource
+      aws_region: us-gov-west-1
+      tag: latest
 
   - name: pull-request
-    type: docker-image
+    type: registry-image
     source:
-      repository: teliaoss/github-pr-resource
+      aws_access_key_id: ((ecr_aws_key))
+      aws_secret_access_key: ((ecr_aws_secret))
+      repository: github-pr-resource
+      aws_region: us-gov-west-1
+      tag: latest
+
+  - name: time
+    type: registry-image
+    source:
+      aws_access_key_id: ((ecr_aws_key))
+      aws_secret_access_key: ((ecr_aws_secret))
+      repository: time-resource
+      aws_region: us-gov-west-1
+      tag: latest

ci/pipeline.yml

@@ -12,17 +12,19 @@ env-cf: &env-cf
 python-image: &python-image
   platform: linux
   image_resource:
-    type: docker-image
+    type: registry-image
     source:
-      repository: python
-      tag: 3.8
+      aws_access_key_id: ((ecr-aws-key))
+      aws_secret_access_key: ((ecr-aws-secret))
+      repository: pages-python-v3.11
+      aws_region: us-gov-west-1
+      tag: latest
 
 cf-image: &cf-image
   platform: linux
   image_resource:
     type: registry-image
     source:
-      repository: 18fgsa/concourse-task
       aws_access_key_id: ((ecr-aws-key))
       aws_secret_access_key: ((ecr-aws-secret))
       repository: harden-concourse-task
@@ -38,8 +40,6 @@ test: &test
       run:
         dir: src
         path: ci/tasks/test.sh
-    params:
-      CC_TEST_REPORTER_ID: 449ef357d9e81688685bde0dcdd348bc764d717fa6a4c9c01562bd8037ceb665
 
 ############################
 #  JOBS
@@ -100,21 +100,27 @@ jobs:
         resource: src-((deploy-env))
         trigger: true
         params: {depth: 1}
-      - put: gh-status
-        inputs: [src]
-        params: {state: pending}
+      - put: pr-((git-branch))
+        params:
+          path: pull-request
+          status: pending
+          context: concourse
       - do: *test
 
     on_success:
-      put: gh-status
-      inputs: [src]
-      params: {state: success}
+      put: pr-((git-branch))
+      params:
+        path: pull-request
+        status: success
+        context: concourse
 
     on_failure:
       in_parallel:
-        - put: gh-status
-          inputs: [src]
-          params: {state: failure}
+        - put: pr-((git-branch))
+          params:
+            path: pull-request
+            status: failure
+            context: concourse
         - put: slack
           params:
             text: |
@@ -172,9 +178,11 @@ jobs:
 
     on_failure:
       in_parallel:
-        - put: gh-status
-          inputs: [src]
-          params: {state: failure}
+        - put: pr-((git-branch))
+          params:
+            path: pull-request
+            status: failure
+            context: concourse
         - put: slack
           params:
             text: |
@@ -230,9 +238,11 @@ jobs:
         passed: [deploy-((deploy-env))]
     on_success:
       in_parallel:
-        - put: gh-status
-          inputs: [src]
-          params: {state: success}
+        - put: pr-((git-branch))
+          params:
+            path: pull-request
+            status: success
+            context: concourse
         - put: slack
           params:
             text: |
@@ -277,15 +287,6 @@ resources:
     source:
       url: ((slack-webhook-url))
 
-  - name: gh-status
-    type: cogito
-    check_every: 1h
-    source:
-      owner: cloud-gov
-      repo: pages-build-container
-      access_token: ((gh-access-token))
-      context_prefix: concourse
-
   - name: image-repository-((deploy-env))
     type: registry-image
     source:
@@ -300,18 +301,29 @@ resources:
 
 resource_types:
 
-  - name: cogito
-    type: docker-image
-    check_every: 24h
-    source:
-      repository: pix4d/cogito
-
   - name: slack-notification
-    type: docker-image
+    type: registry-image
     source:
-      repository: cfcommunity/slack-notification-resource
+      aws_access_key_id: ((ecr_aws_key))
+      aws_secret_access_key: ((ecr_aws_secret))
+      repository: slack-notification-resource
+      aws_region: us-gov-west-1
+      tag: latest
 
   - name: pull-request
-    type: docker-image
+    type: registry-image
+    source:
+      aws_access_key_id: ((ecr_aws_key))
+      aws_secret_access_key: ((ecr_aws_secret))
+      repository: github-pr-resource
+      aws_region: us-gov-west-1
+      tag: latest
+
+  - name: time
+    type: registry-image
     source:
-      repository: teliaoss/github-pr-resource
+      aws_access_key_id: ((ecr_aws_key))
+      aws_secret_access_key: ((ecr_aws_secret))
+      repository: time-resource
+      aws_region: us-gov-west-1
+      tag: latest

ci/tasks/test.sh

@@ -9,13 +9,6 @@ pip install -r requirements-dev.txt
 flake8
 bandit -r src
 
-curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-chmod +x ./cc-test-reporter
-./cc-test-reporter before-build
-
 pytest --cov-report xml:./coverage/coverage.xml --cov-report html:./coverage --cov-report term --cov=src; status=$?
 
-./cc-test-reporter format-coverage -t coverage.py ./coverage/coverage.xml
-./cc-test-reporter upload-coverage || true
-
-exit $status
+exit $status

requirements-dev.txt

@@ -4,10 +4,10 @@
 # Testing and development dependencies
 bandit>=1.0,<2.0
 flake8==3.8.3
-moto==1.3.14
+moto==5.0.1
 pyfakefs==4.0.2
 pyflakes==2.2.0
 pylint==2.5.3
-pytest-cov==2.10.0
-pytest==5.4.3
+pytest-cov==4.1.0
+pytest==7.4.4
 requests-mock==1.8.0

requirements.txt

@@ -1,8 +1,8 @@
-requests==2.24.0
+requests==2.31.0
 boto3==1.14.20
 humanize==4.4.0
 stopit==1.1.2
-psycopg2==2.8.5
-cryptography==3.3.2
-pyyaml==5.4
-psutil==5.9.4
+psycopg2==2.9.9
+cryptography==42.0.2
+pyyaml==6.0.1
+psutil==5.9.4

test/publishing/test_models.py

@@ -5,6 +5,7 @@
 import pytest
 
 from publishing.models import SiteObject, SiteFile, SiteRedirect
+from ..support import generate_file_hash
 
 
 class TestSiteObject():
@@ -123,8 +124,7 @@ def test_compressible_file(self, tmpdir):
         assert model.is_compressible is True
         assert model.is_compressed is True
 
-        # hardcoded md5 hash of compressed 'content'
-        assert model.md5 == 'f3900f9f80fac3c6ee8e077d6b172568'
+        assert model.md5 == generate_file_hash(test_file)
         assert model.s3_key == '/site/test_file.html'
         assert model.dir_prefix == str(test_dir)
         assert model.content_encoding == 'gzip'

test/publishing/test_s3publisher.py

@@ -2,7 +2,7 @@
 import pytest
 import requests_mock
 
-from moto import mock_s3
+from moto import mock_aws
 
 from publishing.s3publisher import list_remote_objects, publish_to_s3
 from publishing.models import SiteObject
@@ -20,7 +20,7 @@ def s3_client(monkeypatch):
     monkeypatch.setenv('AWS_ACCESS_KEY_ID', TEST_ACCESS_KEY)
     monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', TEST_SECRET_KEY)
 
-    with mock_s3():
+    with mock_aws():
         conn = boto3.resource('s3', region_name=TEST_REGION)
 
         conn.create_bucket(

test/support.py

@@ -1,4 +1,5 @@
 import tempfile
+import hashlib
 
 from pathlib import Path
 
@@ -13,3 +14,13 @@ def patch_dir(monkeypatch, module, dir_constant):
 def create_file(file_path, contents='', mode='w'):
     with file_path.open(mode) as f:
         f.write(contents)
+
+
+def generate_file_hash(filename):
+    hash_md5 = hashlib.md5()  # nosec
+
+    with open(filename, 'rb') as file:
+        for chunk in iter(lambda: file.read(4096), b""):
+            hash_md5.update(chunk)
+
+    return hash_md5.hexdigest()

test/test_cache.py

@@ -5,7 +5,7 @@
 import filecmp
 import shutil
 
-from moto import mock_s3
+from moto import mock_aws
 
 from steps.cache import CacheFolder, get_checksum
 from log_utils import get_logger
@@ -21,7 +21,7 @@ def aws_credentials():
 
 @pytest.fixture
 def s3_client(aws_credentials):
-    with mock_s3():
+    with mock_aws():
         conn = boto3.client("s3")
         yield conn