[OSDEV-862] Use the Test environment for creating anomymized dump (#164)

This PR fixes issue with lack storage on GitHub runner to contain 2+GB dump * it fixes command in action * it uses self-hosted runner to have more space
opensupplyhub · Apr 22, 2024 · 38ee776 · 38ee776
1 parent 6fe3556
commit 38ee776
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 16 deletions.
diff --git a/.github/workflows/db_apply_anonimized.yml b/.github/workflows/db_apply_anonimized.yml
@@ -13,8 +13,8 @@ on:
         default: Test
 
 jobs:
-  save-anonymized-db:
-    runs-on: ubuntu-latest
+  apply-anonymized-db:
+    runs-on: self-hosted
     environment: ${{ inputs.deploy-env || 'Test' }}
 
     steps:
@@ -39,4 +39,23 @@ jobs:
               -e DATABASE_NAME=opensupplyhub \
               -e DATABASE_USERNAME=opensupplyhub \
               -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
-              restore
+              restore
+  post_deploy:
+    needs: apply-anonymized-db
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.deploy-env || 'Test' }}
+    steps:
+      - name: Get Environment Name for ${{ vars.ENV_NAME }}
+        id: get_env_name
+        uses: Entepotenz/change-string-case-action-min-dependencies@v1
+        with:
+          string: ${{ vars.ENV_NAME }}
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Run migrations for ${{ vars.ENV_NAME }}
+        run: |
+          ./deployment/run_cli_task ${{ vars.ENV_NAME }} "migrate,api"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: "eu-west-1"
diff --git a/.github/workflows/db_save_anonymized.yml b/.github/workflows/db_save_anonymized.yml
@@ -5,8 +5,8 @@ on:
 
 jobs:
   save-anonymized-db:
-    runs-on: ubuntu-latest
-    environment: Production
+    runs-on: self-hosted
+    environment: Test
     steps:
       - name: Get Environment Name for ${{ vars.ENV_NAME }}
         id: get_env_name
@@ -15,20 +15,20 @@ jobs:
           string: ${{ vars.ENV_NAME }}
       - name: Checkout repo
         uses: actions/checkout@v4
-      - name: Restore database for ${{ vars.ENV_NAME }}
+      - name: Dump database for ${{ vars.ENV_NAME }}
         run: |
           cd ./src/anon-tools
           mkdir -p ./keys
-          echo "${{ secrets.KEY_FILE }}" > ./keys/key
-          docker build -t restore -f Dockerfile.restore .
+          echo "${{ secrets.KEY_FILE_PROD }}" > ./keys/key
+          docker build -t dump_image -f Dockerfile.dump .
           docker run -v ./keys/key:/keys/key --shm-size=2gb --rm \
-              -e AWS_ACCESS_KEY_ID_PROD=${{ secrets.AWS_ACCESS_KEY_ID }} \
-              -e AWS_SECRET_ACCESS_KEY_PROD=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+              -e AWS_ACCESS_KEY_ID_PROD=${{ secrets.AWS_ACCESS_KEY_ID_PROD }} \
+              -e AWS_SECRET_ACCESS_KEY_PROD=${{ secrets.AWS_SECRET_ACCESS_KEY_RPOD }} \
               -e AWS_DEFAULT_REGION_PROD=eu-west-1 \
-              -e AWS_ACCESS_KEY_ID_TEST=${{ secrets.AWS_ACCESS_KEY_ID_TEST }} \
-              -e AWS_SECRET_ACCESS_KEY_TEST=${{ secrets.AWS_SECRET_ACCESS_KEY_TEST }} \
+              -e AWS_ACCESS_KEY_ID_TEST=${{ secrets.AWS_ACCESS_KEY_ID }} \
+              -e AWS_SECRET_ACCESS_KEY_TEST=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
               -e AWS_DEFAULT_REGION_TEST=eu-west-1 \
               -e DATABASE_NAME=opensupplyhub \
               -e DATABASE_USERNAME=opensupplyhub \
               -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
-              restore
+              dump_image
diff --git a/doc/release/RELEASE-NOTES.md b/doc/release/RELEASE-NOTES.md
@@ -17,6 +17,9 @@ This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html
 ### Code/API changes
 
 ### Architecture/Environment changes
+* [OSDEV-862](https://opensupplyhub.atlassian.net/browse/OSDEV-862) Fix `DB - Save Anonymized DB` / `DB - Apply Anonymized DB` workflows:
+  - run actions on self-hosted runners to eliminate `lack of storage` issue that hapopen on github's runners.
+  - use the `Test` environment for  `DB - Save Anonymized DB` action
 * [OSDEV-989](https://opensupplyhub.atlassian.net/browse/OSDEV-989) - The Strategy pattern was utilized to consolidate the processing of new facilities received from both API requests and list uploads. The code responsible for executing this processing was refactored, and new classes were implemented:
     * ProcessingFacility - abstract class for facility processing
     * ProcessingFacilityList - class to process a facility list

diff --git a/src/anon-tools/Dockerfile.restore b/src/anon-tools/Dockerfile.restore
@@ -26,7 +26,7 @@ RUN chmod 644 ~/.ssh/known_hosts
 
 COPY ./do_restore.sh ./
 
-VOLUME /keys/key
+VOLUME /keys
 
 CMD ["sh", "do_restore.sh"]
 
diff --git a/src/anon-tools/do_dump.sh b/src/anon-tools/do_dump.sh
@@ -1,7 +1,5 @@
 #!/bin/bash
 
-echo "$AWS_DEFAULT_REGION_PROD"
-
 bastion="$(AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID_PROD \
            AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY_PROD \
            AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION_PROD \
@@ -18,6 +16,7 @@ chmod 600 /keys/key
 ssh -f -i /keys/key -L 5433:database.service.osh.internal:5432 -N ec2-user@$bastion
 
 pg_dump --clean --no-owner --no-privileges -Fc -h localhost  -d $DATABASE_NAME -U $DATABASE_USERNAME -p 5433 -f /dumps/osh_prod_large.dump -w --verbose
+ls -la /dumps
 
 echo "Start anonymization"