ROX-20058: Use GCP secret manager + Helm for infra deployments (#1015)

.github/workflows/PR.yaml

@@ -10,6 +10,8 @@ defaults:
   run:
     working-directory: go/src/github.com/stackrox/infra
 
+concurrency: pr-${{ github.ref }}
+
 env:
   CLUSTER_NAME: infra-pr-${{ github.event.pull_request.number }}
   GH_TOKEN: ${{ secrets.ROBOT_ROX_GITHUB_TOKEN }}
@@ -97,7 +99,7 @@ jobs:
 
       - name: Deploy infra to dev cluster
         run: |
-          make install-local
+          ENVIRONMENT=development TEST_MODE=true make install-argo helm-deploy
           sleep 10 # wait for old pods to disappear so the svc port-forward doesn't connect to them
           kubectl -n infra port-forward svc/infra-server-service 8443:8443 &
           sleep 10
@@ -148,7 +150,7 @@ jobs:
         run: |
           make argo-workflow-lint
 
-      - name: Run e2e tests
+      - name: Run BATS e2e tests
         env:
           INFRA_TOKEN: ${{ secrets.INFRA_TOKEN_DEV }}
         run: |
@@ -158,7 +160,7 @@ jobs:
           $INFRACTL whoami || true
           $INFRACTL version || true
 
-          make e2e-tests || touch FAIL
+          make bats-e2e-tests || touch FAIL
           kubectl -n infra logs -l app=infra-server --tail=-1
           kill %1
 

.github/workflows/deploy.yaml

@@ -1,3 +1,4 @@
+name: Deploy infra
 run-name: >-
   ${{
     format('Deploy infra version {0} to {1}',
@@ -53,14 +54,14 @@ jobs:
           gcloud container clusters get-credentials infra-${{ inputs.environment }} \
             --project stackrox-infra \
             --region us-west2
-          make install-${{ inputs.environment }}
+          ENVIRONMENT=${{ inputs.environment }} make install-argo helm-deploy
 
       - name: Notify infra channel about new version
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         uses: slackapi/slack-github-action@v1.23.0
         with:
-          channel-id: C01H4DC33K3 #acs-infra
+          channel-id: CVANK5K5W #acs-infra
           payload: >-
             {
                 "blocks": [

CHANGELOG.md

@@ -8,6 +8,8 @@ Please avoid adding duplicate information across this changelog and JIRA/doc inp
 
 ## [NEXT RELEASE]
 
+- Deploying infra-server with Helm and GCP Secret Manager
+
 ## [0.8.2]
 
 - Hackathon '23:

DEPLOYMENT.md

@@ -49,27 +49,16 @@ infra-address-production   35.227.207.252  EXTERNAL
 
 ## Configuration
 
-Service configuration is [stored in a GCS bucket](https://console.cloud.google.com/storage/browser/infra-configuration?organizationId=847401270788&project=stackrox-infra).
+Service configuration and secrets are stored in [GCP Secret Manager](https://console.cloud.google.com/security/secret-manager?project=stackrox-infra).
 
-You will need to download this configuration if you plan to make a change to infra. Configuration changes are baked in to the `infra-server` image at build time.
+To view these, run:
 
-To download the configuration locally to `chart/infra-server/configuration`, run:
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest, 1,2,3,...> make secrets-download`.
 
-`make configuration-download`
+This will download the secrets to `chart/infra-server/configuration/`.
 
-After you change configuration run:
-
-`make create-consolidated-values`
-
-Which will e.g.:
-```
-INFO: Wed Jan 11 06:47:53 PM UTC 2023: Creating a combined values file for chart/infra-server/configuration/development files
-INFO: Wed Jan 11 06:47:53 PM UTC 2023: Creating a combined values file for chart/infra-server/configuration/production files
-```
-
-To upload the local configuration which includes the consolidated values back to the bucket, run:
-
-`make configuration-upload`
+- `<ENVIRONMENT>-values.yaml`: To show or edit a value, do it directly in this file, and use `ENVIRONMENT=<development,production> make secrets-upload` to upload the changes.
+- `<ENVIRONMENT>-values-from-files.yaml`: To show or edit a value, use `ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3> make secrets-<show, edit>` and follow the instructions. NOTE: This will download a fresh copy of the requested secret version and upload a new version after your changes. That ensures that your local secrets do not go stale.
 
 ## Regenerating the localhost certificates for the gRPC gateway
 
@@ -121,35 +110,35 @@ correct tooling installed with:
 
 Use the `deploy` Github action to update development or production environments with a new release.
 
-### Staging/(dev.infra.rox.systems)
+### Argo Deployment
 
-To render a copy of the charts (for inspection), run:
+To install Argo workflow server, run:
 
-`make render-development`
+`ENVIRONMENT=<development,production> make install-argo`
 
-To then apply that chart to the development cluster, run:
+NOTE: This is a separate step and not a dependant chart for example to avoid too frequent Argo deployments.
 
-`make install-development-with-rendered`
+### Manual deployment
 
-To do everything in one command, run:
+To render a copy of the charts (for inspection), run:
 
-`make install-development`
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3, ...> make helm-template`
 
-Note: This will always bounce the infra server pods.
+To show the diff between the current Helm release and the charts, run:
 
-### Production
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3, ...> make helm-diff`
 
-To render a copy of the charts (for inspection), run:
+To then apply that chart to the cluster, run:
 
-`make render-production`
+`ENVIRONMENT=<development,production> SECRET_VERSION=<latest,1,2,3, ...> make helm-deploy`
 
-To then apply that chart to the development cluster, run:
+#### Test Mode
 
-`make install-production-with-rendered`
+Use the environment variable `TEST_MODE` to disable certain infra service behavior, like:
 
-To do everything in one command, run:
+`TEST_MODE=true ENVIRONMENT=development SECRET_VERSION=latest make helm-deploy`
 
-`make deploy-production`
+This is used in the infra PR clusters to set the login referer and disable telemetry.
 
 ## Verification