ci(build): Reusable private cloud build (#4165)

.github/actions/api-deploy-ecs/action.yml

@@ -2,9 +2,6 @@ name: API Deploy to ECS
 description: Deploy the Flagsmith API to ECS
 
 inputs:
-  github_access_token:
-    description: The Github access token to use to access external repositories (e.g. flagsmith-saml)
-    required: true
   aws_access_key_id:
     description: The AWS access key ID to use for deploying to ECS.
     required: true
@@ -36,30 +33,11 @@ inputs:
   aws_identity_migration_task_role_arn:
     description: The ARN of the role to run the identity migration task.
     required: true
-  aws_ecr_repository_arn:
-    description: The ARN of the ECR repository to deploy docker image to.
-    required: true
   aws_task_definitions_directory_path:
     description: The local path in the repository to the json file containing the task definition template
     required: true
-  image_tag:
-    description: The tag to use when registering the docker image in ECR
-    required: false
-    default: ${GITHUB_REF#refs/*/}
-  flagsmith_saml_revision:
-    description: The flagsmith-saml repo git revision to use when building deployment package.
-    required: false
-    default: main
-  flagsmith_auth_controller_revision:
-    description: The flagsmith-auth-controller git revision to use when building deployment package.
-    required: false
-    default: main
-  flagsmith_rbac_revision:
-    description: The flagsmith-rbac git revision to use when building deployment package.
-    required: false
-    default: main
-  sse_pgp_private_key:
-    description: Private PGP key used for encrypting/decrypting access logs
+  api_ecr_image_url:
+    description: The ECR URL of the latest API image.
     required: true
 
 outputs:
@@ -71,96 +49,20 @@ runs:
   using: composite
 
   steps:
-    - name: Checkout SAML package
-      uses: actions/checkout@v4
-      with:
-        repository: flagsmith/flagsmith-saml
-        token: ${{ inputs.github_access_token }}
-        ref: ${{ inputs.flagsmith_saml_revision }}
-        path: ./flagsmith-saml
-
-    - name: Integrate SAML module
-      run: mv ./flagsmith-saml/saml ./api
-      shell: bash
-
-    - name: Checkout Auth Controller package
-      uses: actions/checkout@v4
-      with:
-        repository: flagsmith/flagsmith-auth-controller
-        token: ${{ inputs.github_access_token }}
-        ref: ${{ inputs.flagsmith_auth_controller_revision }}
-        path: ./flagsmith-auth-controller
-
-    - name: Integrate Auth Controller module
-      run: mv ./flagsmith-auth-controller/auth_controller ./api
-      shell: bash
-
-    - name: Checkout RBAC module
-      uses: actions/checkout@v4
-      with:
-        repository: flagsmith/flagsmith-rbac
-        token: ${{ inputs.github_access_token }}
-        ref: ${{ inputs.flagsmith_rbac_revision }}
-        path: ./flagsmith-rbac
-
-    - name: Integrate RBAC module
-      run: mv ./flagsmith-rbac/rbac ./api
-      shell: bash
-
-    - name: Set ECR tag
-      id: ecr-tag-variable
-      run: echo "tag=${{ inputs.image_tag }}" >> $GITHUB_OUTPUT
-      shell: bash
-
-    - name: Write git info to Docker image
-      run: |
-        echo ${{ github.sha }} > api/CI_COMMIT_SHA
-      shell: bash
-
-    - name: Write file to indicate SaaS
-      run: |
-        echo '' > api/SAAS_DEPLOYMENT
-      shell: bash
-
     - name: Configure AWS Credentials
       uses: aws-actions/configure-aws-credentials@v1-node16
       with:
         aws-access-key-id: ${{ inputs.aws_access_key_id }}
         aws-secret-access-key: ${{ inputs.aws_secret_access_key }}
         aws-region: eu-west-2
 
-    - name: Login to Amazon ECR
-      id: login-ecr
-      uses: aws-actions/amazon-ecr-login@v1
-
-    - name: Write PGP key to a file
-      run: |
-        cat << EOF > sse_pgp_pkey
-        ${{ inputs.sse_pgp_private_key }}
-        EOF
-      shell: bash
-
-    - name: Build, tag, and push image to Amazon ECR
-      id: build-image
-      env:
-        IMAGE_TAG: ${{ steps.ecr-tag-variable.outputs.tag }}
-        ECR_REPOSITORY: ${{ inputs.aws_ecr_repository_arn }}
-        DOCKER_BUILDKIT: '1'
-      run: |
-        echo "Building docker image with URL: "
-        echo $ECR_REPOSITORY:$IMAGE_TAG
-        docker build --secret id=sse_pgp_pkey,src=./sse_pgp_pkey -t $ECR_REPOSITORY:$IMAGE_TAG -f api/Dockerfile --build-arg SAML_INSTALLED=1 --build-arg POETRY_OPTS="--with saml,auth-controller,workflows" --build-arg GH_TOKEN=${{ inputs.github_access_token }} .
-        docker push $ECR_REPOSITORY:$IMAGE_TAG
-        echo "image=$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
-      shell: bash
-
     - name: Fill in the new image ID in the Amazon ECS API task definition
       id: task-def-api
       uses: aws-actions/amazon-ecs-render-task-definition@v1
       with:
         task-definition: ${{ inputs.aws_task_definitions_directory_path }}/ecs-task-definition-web.json
         container-name: flagsmith-api
-        image: ${{ steps.build-image.outputs.image }}
+        image: ${{ inputs.api_ecr_image_url }}
 
     # This is used in both the SQL migrations and the Dynamo Identity Migrations
     - name: Fill in the new image ID in the Amazon ECS migration task definition
@@ -169,7 +71,7 @@ runs:
       with:
         task-definition: ${{ inputs.aws_task_definitions_directory_path }}/ecs-task-definition-migration.json
         container-name: flagsmith-api-migration
-        image: ${{ steps.build-image.outputs.image }}
+        image: ${{ inputs.api_ecr_image_url }}
 
     - name: Register and perform SQL schema migration
       id: register-migrate-task

.github/actions/e2e-tests/action.yml

@@ -47,6 +47,5 @@ runs:
       env:
         E2E_TEST_TOKEN: ${{ inputs.e2e_test_token }}
         SLACK_TOKEN: ${{ inputs.slack_token }}
-        STATIC_ASSET_CDN_URL: /
         ENV: ${{ inputs.environment }}
         GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/.reusable-deploy-ecs.yml

@@ -0,0 +1,123 @@
+# reusable workflow
+name: Deploy to ECS and Test
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        type: string
+        description: The environment to deploy to
+        required: true
+      saas-image-name:
+        type: string
+        description: SaaS image name
+        required: false
+        default: flagsmith-saas-api
+
+jobs:
+  docker-build-saas-api:
+    environment: ${{ inputs.environment }}
+    runs-on: ubuntu-latest
+    outputs:
+      image-url: ${{ steps.login-ecr.outputs.registry }}/${{ inputs.saas-image-name }}:${{ steps.meta.outputs.version }}
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v4
+
+      - name: Set up Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          aws-access-key-id: ${{ vars.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ steps.login-ecr.outputs.registry }}/${{ inputs.saas-image-name }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
+
+      - name: Build saas-api image
+        uses: depot/build-push-action@v1
+        with:
+          target: saas-api
+          context: .
+          build-args: CI_COMMIT_SHA=${{ github.sha }}
+          secrets: |
+            github_private_cloud_token=${{ secrets.GH_PRIVATE_ACCESS_TOKEN }}
+            sse_pgp_pkey=${{ secrets.SSE_PGP_PRIVATE_KEY }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  deploy:
+    needs: docker-build-saas-api
+    environment: ${{ inputs.environment }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v4
+
+      - name: Deploy API to ${{ inputs.environment }}
+        id: deploy-api
+        uses: ./.github/actions/api-deploy-ecs
+        with:
+          aws_access_key_id: ${{ vars.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_ecs_cluster_name: ${{ vars.AWS_ECS_CLUSTER_NAME }}
+          aws_ecs_cluster_arn: ${{ vars.AWS_ECS_CLUSTER_ARN }}
+          aws_ecs_service_name: ${{ vars.AWS_ECS_SERVICE_NAME }}
+          aws_vpc_subnet_id: ${{ vars.AWS_VPC_SUBNET_ID }}
+          aws_ecs_security_group_id: ${{ vars.AWS_ECS_SECURITY_GROUP_ID }}
+          aws_ecr_repository_arn: ${{ vars.AWS_ECR_REPOSITORY_ARN }}
+          aws_identity_migration_event_bus_name: ${{ vars.AWS_IDENTITY_MIGRATION_EVENT_BUS_NAME }}
+          aws_identity_migration_event_bus_rule_id: ${{ vars.AWS_IDENTITY_MIGRATION_EVENT_BUS_RULE_ID }}
+          aws_identity_migration_task_role_arn: ${{ vars.AWS_IDENTITY_MIGRATION_TASK_ROLE_ARN }}
+          aws_task_definitions_directory_path: infrastructure/aws/${{ inputs.environment }}
+          api_ecr_image_url: ${{ needs.docker-build-saas-api.outputs.image-url }}
+
+      - name: Deploy Task processor to ${{ inputs.environment }}
+        uses: ./.github/actions/task-processor-deploy-ecs
+        with:
+          aws_access_key_id: ${{ vars.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_ecs_cluster_name: ${{ vars.AWS_ECS_CLUSTER_NAME }}
+          aws_ecs_service_name: ${{ vars.AWS_ECS_TASK_PROCESSOR_SERVICE_NAME }}
+          aws_task_definitions_directory_path: infrastructure/aws/${{ inputs.environment }}
+          api_ecr_image_url: ${{ needs.docker-build-saas-api.outputs.image-url }}
+
+  run-tests:
+    needs: deploy
+    runs-on: ubuntu-latest
+    name: Run E2E Tests
+    environment: ${{ inputs.environment }}
+    concurrency:
+      group: e2e-tests-${{ inputs.environment }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Cloning repo
+        uses: actions/checkout@v4
+
+      - name: Run E2E tests against ${{ inputs.environment }}
+        uses: ./.github/actions/e2e-tests
+        with:
+          e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }}
+          slack_token: ${{ secrets.SLACK_TOKEN }}
+          environment: ${{ inputs.environment }}

.github/workflows/.reusable-docker-build.yml

@@ -6,30 +6,39 @@ on:
     inputs:
       registry-url:
         type: string
-        description: Github container registry base URL
+        description: Github Container Registry base URL
         required: false
         default: ghcr.io
       file:
         type: string
         description: Path to the Dockerfile
-        required: true
+        required: false
+        default: Dockerfile
+      target:
+        type: string
+        description: Sets the target stage to build
+        required: false
       image-name:
         type: string
-        description: Image slug
+        description: Image name
         required: true
       build-args:
         type: string
         description: List of build-time variables
         required: false
       scan:
         type: boolean
-        description: Whether to scan image for vulnerabilities
+        description: Whether to scan built image for vulnerabilities
         required: false
         default: true
     outputs:
       image:
         description: Resulting image specifier
         value: ${{ inputs.registry-url }}/flagsmith/${{ inputs.image-name }}:${{ jobs.build.outputs.version }}
+    secrets:
+      secrets:
+        description: List of secrets to expose to the build (e.g., `key=string, GIT_AUTH_TOKEN=mytoken`)
+        required: false
 
 jobs:
   build:
@@ -50,7 +59,7 @@ jobs:
       - name: Set up Depot CLI
         uses: depot/setup-action@v1
 
-      - name: Login to GitHub Container Registry
+      - name: Login to Github Container Registry
         uses: docker/login-action@v3
         with:
           registry: ${{ inputs.registry-url }}
@@ -67,18 +76,39 @@ jobs:
             type=ref,event=branch
             type=ref,event=tag
             type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
 
       - name: Build and push image
         uses: depot/build-push-action@v1
         with:
+          context: .
           push: true
-          build-args: ${{ inputs.build-args }}
+          platforms: linux/amd64,linux/arm64
+          secrets: ${{ secrets.secrets }}
+          target: ${{ inputs.target }}
+          build-args: |
+            CI_COMMIT_SHA=${{ github.sha }}
+            ${{ inputs.build-args }}
           file: ${{ inputs.file }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          provenance: false
 
       - name: Run Trivy vulnerability scanner
-        if: ${{ inputs.scan }}
+        id: trivy
+        if: inputs.scan
         uses: aquasecurity/trivy-action@master
         with:
           image-ref: ${{ inputs.registry-url }}/flagsmith/${{ inputs.image-name }}:${{ steps.meta.outputs.version }}
+          format: sarif
+          output: trivy-results.sarif
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        if: inputs.scan && (success() || failure())
+        with:
+          sarif_file: trivy-results.sarif