diff --git a/Dockerfile b/Dockerfile index 5f6083a7..58373e73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax = docker/dockerfile:1.2 -FROM ubuntu:20.04 +FROM ubuntu:22.04 # Install general dependencies RUN apt-get update \ @@ -10,13 +10,10 @@ RUN apt-get update \ autoconf automake libgdbm-dev libncurses5-dev \ libsqlite3-dev libtool libyaml-dev pkg-config libgmp-dev \ libpq-dev libxi6 libjpeg-dev libpng-dev libtiff-dev libgif-dev \ - libwebp-dev wget \ + libwebp-dev wget python3 python3-dev python3-pip\ # Ruby deps gawk bison sqlite3 -# Uses python3.8 by default -RUN apt install -y python3 python3-pip - # Deps for container hardening RUN ln -sf "/usr/share/zoneinfo/$SYSTEM_TIMEZONE" /etc/localtime COPY docker/ua-attach-config.sh . diff --git a/ci/pipeline-dev.yml b/ci/pipeline-dev.yml index 6f9b5d6e..c500e98e 100644 --- a/ci/pipeline-dev.yml +++ b/ci/pipeline-dev.yml @@ -12,17 +12,19 @@ env-cf: &env-cf python-image: &python-image platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: python - tag: 3.8 + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-python-v3.11 + aws_region: us-gov-west-1 + tag: latest cf-image: &cf-image platform: linux image_resource: type: registry-image source: - repository: 18fgsa/concourse-task aws_access_key_id: ((ecr-aws-key)) aws_secret_access_key: ((ecr-aws-secret)) repository: harden-concourse-task @@ -38,8 +40,6 @@ test: &test run: dir: src path: ci/tasks/test.sh - params: - CC_TEST_REPORTER_ID: 449ef357d9e81688685bde0dcdd348bc764d717fa6a4c9c01562bd8037ceb665 ############################ # JOBS @@ -64,10 +64,35 @@ jobs: resource: pr-((git-branch)) trigger: true passed: [set-pipeline] + + - put: src + resource: pr-((git-branch)) + params: + path: src + status: pending + base_context: concourse + context: test-pages-build-container-((deploy-env)) + - do: *test + on_success: + put: src + resource: pr-((git-branch)) + params: + path: src + status: success + base_context: concourse + context: test-pages-build-container-((deploy-env)) + on_failure: in_parallel: + - put: src + resource: pr-((git-branch)) + params: + path: src + status: failure + base_context: concourse + context: test-pages-build-container-((deploy-env)) - put: slack params: text: | @@ -227,11 +252,28 @@ resources: resource_types: - name: slack-notification - type: docker-image + type: registry-image source: - repository: cfcommunity/slack-notification-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: slack-notification-resource + aws_region: us-gov-west-1 + tag: latest - name: pull-request - type: docker-image + type: registry-image source: - repository: teliaoss/github-pr-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: github-pr-resource + aws_region: us-gov-west-1 + tag: latest + + - name: time + type: registry-image + source: + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: time-resource + aws_region: us-gov-west-1 + tag: latest diff --git a/ci/pipeline.yml b/ci/pipeline.yml index f563be80..112cf50e 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -12,17 +12,19 @@ env-cf: &env-cf python-image: &python-image platform: linux image_resource: - type: docker-image + type: registry-image source: - repository: python - tag: 3.8 + aws_access_key_id: ((ecr-aws-key)) + aws_secret_access_key: ((ecr-aws-secret)) + repository: pages-python-v3.11 + aws_region: us-gov-west-1 + tag: latest cf-image: &cf-image platform: linux image_resource: type: registry-image source: - repository: 18fgsa/concourse-task aws_access_key_id: ((ecr-aws-key)) aws_secret_access_key: ((ecr-aws-secret)) repository: harden-concourse-task @@ -38,8 +40,6 @@ test: &test run: dir: src path: ci/tasks/test.sh - params: - CC_TEST_REPORTER_ID: 449ef357d9e81688685bde0dcdd348bc764d717fa6a4c9c01562bd8037ceb665 ############################ # JOBS @@ -100,21 +100,27 @@ jobs: resource: src-((deploy-env)) trigger: true params: {depth: 1} - - put: gh-status - inputs: [src] - params: {state: pending} + - put: pr-((git-branch)) + params: + path: pull-request + status: pending + context: concourse - do: *test on_success: - put: gh-status - inputs: [src] - params: {state: success} + put: pr-((git-branch)) + params: + path: pull-request + status: success + context: concourse on_failure: in_parallel: - - put: gh-status - inputs: [src] - params: {state: failure} + - put: pr-((git-branch)) + params: + path: pull-request + status: failure + context: concourse - put: slack params: text: | @@ -172,9 +178,11 @@ jobs: on_failure: in_parallel: - - put: gh-status - inputs: [src] - params: {state: failure} + - put: pr-((git-branch)) + params: + path: pull-request + status: failure + context: concourse - put: slack params: text: | @@ -230,9 +238,11 @@ jobs: passed: [deploy-((deploy-env))] on_success: in_parallel: - - put: gh-status - inputs: [src] - params: {state: success} + - put: pr-((git-branch)) + params: + path: pull-request + status: success + context: concourse - put: slack params: text: | @@ -277,15 +287,6 @@ resources: source: url: ((slack-webhook-url)) - - name: gh-status - type: cogito - check_every: 1h - source: - owner: cloud-gov - repo: pages-build-container - access_token: ((gh-access-token)) - context_prefix: concourse - - name: image-repository-((deploy-env)) type: registry-image source: @@ -300,18 +301,29 @@ resources: resource_types: - - name: cogito - type: docker-image - check_every: 24h - source: - repository: pix4d/cogito - - name: slack-notification - type: docker-image + type: registry-image source: - repository: cfcommunity/slack-notification-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: slack-notification-resource + aws_region: us-gov-west-1 + tag: latest - name: pull-request - type: docker-image + type: registry-image + source: + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: github-pr-resource + aws_region: us-gov-west-1 + tag: latest + + - name: time + type: registry-image source: - repository: teliaoss/github-pr-resource + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: time-resource + aws_region: us-gov-west-1 + tag: latest diff --git a/ci/tasks/test.sh b/ci/tasks/test.sh index 3dd7f8f1..1b9cb922 100755 --- a/ci/tasks/test.sh +++ b/ci/tasks/test.sh @@ -9,13 +9,6 @@ pip install -r requirements-dev.txt flake8 bandit -r src -curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter -chmod +x ./cc-test-reporter -./cc-test-reporter before-build - pytest --cov-report xml:./coverage/coverage.xml --cov-report html:./coverage --cov-report term --cov=src; status=$? -./cc-test-reporter format-coverage -t coverage.py ./coverage/coverage.xml -./cc-test-reporter upload-coverage || true - -exit $status \ No newline at end of file +exit $status diff --git a/requirements-dev.txt b/requirements-dev.txt index 0c601ace..aee064c0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,10 +4,10 @@ # Testing and development dependencies bandit>=1.0,<2.0 flake8==3.8.3 -moto==1.3.14 +moto==5.0.1 pyfakefs==4.0.2 pyflakes==2.2.0 pylint==2.5.3 -pytest-cov==2.10.0 -pytest==5.4.3 +pytest-cov==4.1.0 +pytest==7.4.4 requests-mock==1.8.0 diff --git a/requirements.txt b/requirements.txt index 0a326406..ba04984f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -requests==2.24.0 +requests==2.31.0 boto3==1.14.20 humanize==4.4.0 stopit==1.1.2 -psycopg2==2.8.5 -cryptography==3.3.2 -pyyaml==5.4 -psutil==5.9.4 \ No newline at end of file +psycopg2==2.9.9 +cryptography==42.0.2 +pyyaml==6.0.1 +psutil==5.9.4 diff --git a/test/publishing/test_models.py b/test/publishing/test_models.py index af77f0ba..fb06acc9 100644 --- a/test/publishing/test_models.py +++ b/test/publishing/test_models.py @@ -5,6 +5,7 @@ import pytest from publishing.models import SiteObject, SiteFile, SiteRedirect +from ..support import generate_file_hash class TestSiteObject(): @@ -123,8 +124,7 @@ def test_compressible_file(self, tmpdir): assert model.is_compressible is True assert model.is_compressed is True - # hardcoded md5 hash of compressed 'content' - assert model.md5 == 'f3900f9f80fac3c6ee8e077d6b172568' + assert model.md5 == generate_file_hash(test_file) assert model.s3_key == '/site/test_file.html' assert model.dir_prefix == str(test_dir) assert model.content_encoding == 'gzip' diff --git a/test/publishing/test_s3publisher.py b/test/publishing/test_s3publisher.py index a7aca9b0..edfee9fb 100644 --- a/test/publishing/test_s3publisher.py +++ b/test/publishing/test_s3publisher.py @@ -2,7 +2,7 @@ import pytest import requests_mock -from moto import mock_s3 +from moto import mock_aws from publishing.s3publisher import list_remote_objects, publish_to_s3 from publishing.models import SiteObject @@ -20,7 +20,7 @@ def s3_client(monkeypatch): monkeypatch.setenv('AWS_ACCESS_KEY_ID', TEST_ACCESS_KEY) monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', TEST_SECRET_KEY) - with mock_s3(): + with mock_aws(): conn = boto3.resource('s3', region_name=TEST_REGION) conn.create_bucket( diff --git a/test/support.py b/test/support.py index 918a20b7..862daa3e 100644 --- a/test/support.py +++ b/test/support.py @@ -1,4 +1,5 @@ import tempfile +import hashlib from pathlib import Path @@ -13,3 +14,13 @@ def patch_dir(monkeypatch, module, dir_constant): def create_file(file_path, contents='', mode='w'): with file_path.open(mode) as f: f.write(contents) + + +def generate_file_hash(filename): + hash_md5 = hashlib.md5() # nosec + + with open(filename, 'rb') as file: + for chunk in iter(lambda: file.read(4096), b""): + hash_md5.update(chunk) + + return hash_md5.hexdigest() diff --git a/test/test_cache.py b/test/test_cache.py index 1f4a8152..9a67440c 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -5,7 +5,7 @@ import filecmp import shutil -from moto import mock_s3 +from moto import mock_aws from steps.cache import CacheFolder, get_checksum from log_utils import get_logger @@ -21,7 +21,7 @@ def aws_credentials(): @pytest.fixture def s3_client(aws_credentials): - with mock_s3(): + with mock_aws(): conn = boto3.client("s3") yield conn