From 7e92fa072c03c98f5fd6960e8435cf0ed181b14a Mon Sep 17 00:00:00 2001 From: Sean Fern Date: Thu, 7 Mar 2024 14:49:28 -0500 Subject: [PATCH] PLT-339 Add export workflows (#87) --- .../workflows/opt-out-export-dev-deploy.yml | 38 +++++++++++++ .../workflows/opt-out-export-prod-deploy.yml | 24 ++++++++ .../workflows/opt-out-export-test-deploy.yml | 39 +++++++++++++ .../opt-out-export-test-integration.yml | 40 +++++++++++++ .github/workflows/opt-out-export-unit.yml | 33 +++++++++++ .../workflows/opt-out-import-deploy-sbx.yml | 21 ------- ...-dev.yml => opt-out-import-dev-deploy.yml} | 8 ++- ...rod.yml => opt-out-import-prod-deploy.yml} | 9 ++- ...est.yml => opt-out-import-test-deploy.yml} | 14 ++--- .../opt-out-import-test-integration.yml | 56 +++++++++++++++++++ ...-test-unit.yml => opt-out-import-unit.yml} | 2 +- 11 files changed, 248 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/opt-out-export-dev-deploy.yml create mode 100644 .github/workflows/opt-out-export-prod-deploy.yml create mode 100644 .github/workflows/opt-out-export-test-deploy.yml create mode 100644 .github/workflows/opt-out-export-test-integration.yml create mode 100644 .github/workflows/opt-out-export-unit.yml delete mode 100644 .github/workflows/opt-out-import-deploy-sbx.yml rename .github/workflows/{opt-out-import-deploy-dev.yml => opt-out-import-dev-deploy.yml} (76%) rename .github/workflows/{opt-out-import-deploy-prod.yml => opt-out-import-prod-deploy.yml} (50%) rename .github/workflows/{opt-out-import-deploy-test.yml => opt-out-import-test-deploy.yml} (73%) create mode 100644 .github/workflows/opt-out-import-test-integration.yml rename .github/workflows/{opt-out-import-test-unit.yml => opt-out-import-unit.yml} (93%) diff --git a/.github/workflows/opt-out-export-dev-deploy.yml b/.github/workflows/opt-out-export-dev-deploy.yml new file mode 100644 index 0000000..73bba8e --- /dev/null +++ b/.github/workflows/opt-out-export-dev-deploy.yml @@ -0,0 +1,38 @@ +name: opt-out-export dev deploy + +on: + workflow_dispatch: + +jobs: + deploy: + runs-on: self-hosted + environment: dev + defaults: + run: + working-directory: ./attribution-data-file-share + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + - uses: cmsgov/ab2d-bcda-dpc-platform/actions/aws-params-env-action@main + env: + AWS_REGION: ${{ vars.AWS_REGION }} + with: + params: | + ARTIFACTORY_URL=/artifactory/url + ARTIFACTORY_USER=/artifactory/user + ARTIFACTORY_PASSWORD=/artifactory/password + - name: Build opt-out-export zip file + run: ../gradlew buildZip + - uses: aws-actions/configure-aws-credentials@v3 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-dev-github-actions + - name: Upload and reload + run: | + aws s3 cp --no-progress build/distributions/attributionDataShare.zip \ + s3://ab2d-dev-opt-out-export-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name ab2d-dev-opt-out-export \ + --s3-bucket ab2d-dev-opt-out-export-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-export-prod-deploy.yml b/.github/workflows/opt-out-export-prod-deploy.yml new file mode 100644 index 0000000..333fde5 --- /dev/null +++ b/.github/workflows/opt-out-export-prod-deploy.yml @@ -0,0 +1,24 @@ +name: opt-out-export prod deploy + +on: + workflow_dispatch: + +jobs: + deploy: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + environment: prod + steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-prod-github-actions + - name: Promote lambda code from test to prod + run: | + aws s3 cp --no-progress \ + s3://ab2d-test-opt-out-export-function/function-${{ github.sha }}.zip \ + s3://ab2d-prod-opt-out-export-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name ab2d-prod-opt-out-export \ + --s3-bucket ab2d-prod-opt-out-export-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-export-test-deploy.yml b/.github/workflows/opt-out-export-test-deploy.yml new file mode 100644 index 0000000..0ed8a0e --- /dev/null +++ b/.github/workflows/opt-out-export-test-deploy.yml @@ -0,0 +1,39 @@ +name: opt-out-export test (impl) deploy + +on: + workflow_call: # Allow for call from integration test + workflow_dispatch: + +jobs: + deploy: + runs-on: self-hosted + environment: test + defaults: + run: + working-directory: ./attribution-data-file-share + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + - uses: cmsgov/ab2d-bcda-dpc-platform/actions/aws-params-env-action@main + env: + AWS_REGION: ${{ vars.AWS_REGION }} + with: + params: | + ARTIFACTORY_URL=/artifactory/url + ARTIFACTORY_USER=/artifactory/user + ARTIFACTORY_PASSWORD=/artifactory/password + - name: Build opt-out-export zip file + run: ../gradlew buildZip + - uses: aws-actions/configure-aws-credentials@v3 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-test-github-actions + - name: Upload and reload + run: | + aws s3 cp --no-progress build/distributions/attributionDataShare.zip \ + s3://ab2d-test-opt-out-export-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name ab2d-test-opt-out-export \ + --s3-bucket ab2d-test-opt-out-export-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-export-test-integration.yml b/.github/workflows/opt-out-export-test-integration.yml new file mode 100644 index 0000000..03d271c --- /dev/null +++ b/.github/workflows/opt-out-export-test-integration.yml @@ -0,0 +1,40 @@ +name: opt-out-export integration tests + +on: + push: + paths: + - .github/workflows/opt-out-export-test-integration.yml + - attribution-data-file-share/** + workflow_dispatch: + +# Ensure we only have one integration test running at a time +concurrency: + group: opt-out-export-test-integration + +jobs: + # Deploy first if triggered by push + deploy: + if: ${{ github.event_name == 'push' }} + uses: ./.github/workflows/opt-out-export-test-deploy.yml + secrets: inherit + + trigger: + if: ${{ always() }} + needs: deploy + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./attribution-data-file-share + steps: + - uses: aws-actions/configure-aws-credentials@v3 + with: + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.TEST_ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-test-github-actions + - name: Send event to trigger export lambda function + run: | + aws events put-events --entries file://test/events.json # TODO Add events.json matching rule for lambda trigger + + # TODO Check bucket for export file diff --git a/.github/workflows/opt-out-export-unit.yml b/.github/workflows/opt-out-export-unit.yml new file mode 100644 index 0000000..a1a997a --- /dev/null +++ b/.github/workflows/opt-out-export-unit.yml @@ -0,0 +1,33 @@ +name: opt-out-export unit tests + +on: + pull_request: + paths: + - .github/workflows/opt-out-export-unit.yml + - attribution-data-file-share/** + workflow_dispatch: + +jobs: + test: + runs-on: self-hosted + defaults: + run: + working-directory: ./attribution-data-file-share + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + - uses: cmsgov/ab2d-bcda-dpc-platform/actions/aws-params-env-action@main + env: + AWS_REGION: ${{ vars.AWS_REGION }} + with: + params: | + ARTIFACTORY_URL=/artifactory/url + ARTIFACTORY_USER=/artifactory/user + ARTIFACTORY_PASSWORD=/artifactory/password + SONAR_HOST_URL=/sonarqube/url + SONAR_TOKEN=/sonarqube/token + - name: Run unit tests for opt-out-export lambda + run: ../gradlew test diff --git a/.github/workflows/opt-out-import-deploy-sbx.yml b/.github/workflows/opt-out-import-deploy-sbx.yml deleted file mode 100644 index 71973ed..0000000 --- a/.github/workflows/opt-out-import-deploy-sbx.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: opt-out-import sbx (sandbox) deploy - -on: - workflow_dispatch: - -jobs: - deploy: - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - environment: sbx - steps: - - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ vars.ACCOUNT_ROLE }} - - name: Promote lambda code from test to sbx - run: | - aws s3 cp --no-progress s3://${{ vars.OPT_OUT_IMPORT_TEST_ZIP_BUCKET }}/function.zip s3://${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }}/function.zip - aws lambda update-function-code --function-name ab2d-sbx-opt-out-import --s3-bucket ${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }} --s3-key function.zip diff --git a/.github/workflows/opt-out-import-deploy-dev.yml b/.github/workflows/opt-out-import-dev-deploy.yml similarity index 76% rename from .github/workflows/opt-out-import-deploy-dev.yml rename to .github/workflows/opt-out-import-dev-deploy.yml index cef4d0f..a811fd2 100644 --- a/.github/workflows/opt-out-import-deploy-dev.yml +++ b/.github/workflows/opt-out-import-dev-deploy.yml @@ -29,8 +29,10 @@ jobs: - uses: aws-actions/configure-aws-credentials@v3 with: aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ vars.ACCOUNT_ROLE }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-dev-github-actions - name: Upload and reload run: | - aws s3 cp --no-progress build/distributions/optout.zip s3://${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }}/function.zip - aws lambda update-function-code --function-name ab2d-dev-opt-out-import --s3-bucket ${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }} --s3-key function.zip + aws s3 cp --no-progress build/distributions/optout.zip \ + s3://ab2d-dev-opt-out-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name ab2d-dev-opt-out-import \ + --s3-bucket ab2d-dev-opt-out-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-import-deploy-prod.yml b/.github/workflows/opt-out-import-prod-deploy.yml similarity index 50% rename from .github/workflows/opt-out-import-deploy-prod.yml rename to .github/workflows/opt-out-import-prod-deploy.yml index ad48335..df983d1 100644 --- a/.github/workflows/opt-out-import-deploy-prod.yml +++ b/.github/workflows/opt-out-import-prod-deploy.yml @@ -14,8 +14,11 @@ jobs: - uses: aws-actions/configure-aws-credentials@v3 with: aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ vars.ACCOUNT_ROLE }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-prod-github-actions - name: Promote lambda code from test to prod run: | - aws s3 cp --no-progress s3://${{ vars.OPT_OUT_IMPORT_TEST_ZIP_BUCKET }}/function.zip s3://${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }}/function.zip - aws lambda update-function-code --function-name ab2d-prod-opt-out-import --s3-bucket ${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }} --s3-key function.zip + aws s3 cp --no-progress \ + s3://ab2d-test-opt-out-import-function/function-${{ github.sha }}.zip \ + s3://ab2d-prod-opt-out-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name ab2d-prod-opt-out-import \ + --s3-bucket ab2d-prod-opt-out-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-import-deploy-test.yml b/.github/workflows/opt-out-import-test-deploy.yml similarity index 73% rename from .github/workflows/opt-out-import-deploy-test.yml rename to .github/workflows/opt-out-import-test-deploy.yml index fe5b01f..f753929 100644 --- a/.github/workflows/opt-out-import-deploy-test.yml +++ b/.github/workflows/opt-out-import-test-deploy.yml @@ -1,11 +1,7 @@ name: opt-out-import test (impl) deploy on: - push: - branches: - - main - paths: - - optout/** + workflow_call: # Allow for call from integration test workflow_dispatch: jobs: @@ -34,8 +30,10 @@ jobs: - uses: aws-actions/configure-aws-credentials@v3 with: aws-region: ${{ vars.AWS_REGION }} - role-to-assume: ${{ vars.ACCOUNT_ROLE }} + role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-test-github-actions - name: Upload and reload run: | - aws s3 cp --no-progress build/distributions/optout.zip s3://${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }}/function.zip - aws lambda update-function-code --function-name ab2d-test-opt-out-import --s3-bucket ${{ vars.OPT_OUT_IMPORT_ZIP_BUCKET }} --s3-key function.zip + aws s3 cp --no-progress build/distributions/optout.zip \ + s3://ab2d-test-opt-out-import-function/function-${{ github.sha }}.zip + aws lambda update-function-code --function-name ab2d-test-opt-out-import \ + --s3-bucket ab2d-test-opt-out-import-function --s3-key function-${{ github.sha }}.zip diff --git a/.github/workflows/opt-out-import-test-integration.yml b/.github/workflows/opt-out-import-test-integration.yml new file mode 100644 index 0000000..05d3bf4 --- /dev/null +++ b/.github/workflows/opt-out-import-test-integration.yml @@ -0,0 +1,56 @@ +name: opt-out-import test integration + +on: + push: + paths: + - .github/workflows/opt-out-import-test-integration.yml + - optout/** + workflow_dispatch: + +# Ensure we have only one integration test running at a time +concurrency: + group: opt-out-import-test-integration + +jobs: + # Deploy first if triggered by push + deploy: + if: ${{ github.event_name == 'push' }} + uses: ./.github/workflows/opt-out-import-test-deploy.yml + secrets: inherit + + trigger: + if: ${{ always() }} + needs: deploy + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./optout + steps: + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + # Note that we assume the lambda role + role-to-assume: arn:aws:iam::${{ secrets.TEST_ACCOUNT_ID }}:role/delegatedadmin/developer/ab2d-test-opt-out-import-function + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + # Now assume the BFD bucket role + role-to-assume: arn:aws:iam::${{ secrets.BFD_ACCOUNT_ID }}:role/bfd-test-eft-ab2d-bucket-role + role-chaining: true + role-skip-session-tagging: true + - name: Upload test file to the BFD bucket to trigger lambda function via SNS message + run: | + aws s3 cp --no-progress src/test/resources/optOutDummy.txt \ + s3://bfd-test-eft/bfdeft01/ab2d/in/test-${{ github.run_id }}.txt + - name: Check bucket for response file + run: | + sleep 30 + aws s3 cp --no-progress s3://bfd-test-eft/bfdeft01/ab2d/out/test-${{ github.run_id }}.txt \ + test-response.txt + cat test-response.txt + + # TODO Run another job to check database for update diff --git a/.github/workflows/opt-out-import-test-unit.yml b/.github/workflows/opt-out-import-unit.yml similarity index 93% rename from .github/workflows/opt-out-import-test-unit.yml rename to .github/workflows/opt-out-import-unit.yml index 475cf0b..74520c3 100644 --- a/.github/workflows/opt-out-import-test-unit.yml +++ b/.github/workflows/opt-out-import-unit.yml @@ -3,7 +3,7 @@ name: opt-out-import unit tests on: pull_request: paths: - - .github/workflows/opt-out-import-test-unit.yml + - .github/workflows/opt-out-import-unit.yml - optout/** workflow_dispatch: