diff --git a/.github/actions/api-deploy-ecs/action.yml b/.github/actions/api-deploy-ecs/action.yml index 2f9bb529d9a8..d81db5f8f195 100644 --- a/.github/actions/api-deploy-ecs/action.yml +++ b/.github/actions/api-deploy-ecs/action.yml @@ -2,9 +2,6 @@ name: API Deploy to ECS description: Deploy the Flagsmith API to ECS inputs: - github_access_token: - description: The Github access token to use to access external repositories (e.g. flagsmith-saml) - required: true aws_access_key_id: description: The AWS access key ID to use for deploying to ECS. required: true @@ -36,30 +33,11 @@ inputs: aws_identity_migration_task_role_arn: description: The ARN of the role to run the identity migration task. required: true - aws_ecr_repository_arn: - description: The ARN of the ECR repository to deploy docker image to. - required: true aws_task_definitions_directory_path: description: The local path in the repository to the json file containing the task definition template required: true - image_tag: - description: The tag to use when registering the docker image in ECR - required: false - default: ${GITHUB_REF#refs/*/} - flagsmith_saml_revision: - description: The flagsmith-saml repo git revision to use when building deployment package. - required: false - default: main - flagsmith_auth_controller_revision: - description: The flagsmith-auth-controller git revision to use when building deployment package. - required: false - default: main - flagsmith_rbac_revision: - description: The flagsmith-rbac git revision to use when building deployment package. - required: false - default: main - sse_pgp_private_key: - description: Private PGP key used for encrypting/decrypting access logs + api_ecr_image_url: + description: The ECR URL of the latest API image. required: true outputs: @@ -71,57 +49,6 @@ runs: using: composite steps: - - name: Checkout SAML package - uses: actions/checkout@v4 - with: - repository: flagsmith/flagsmith-saml - token: ${{ inputs.github_access_token }} - ref: ${{ inputs.flagsmith_saml_revision }} - path: ./flagsmith-saml - - - name: Integrate SAML module - run: mv ./flagsmith-saml/saml ./api - shell: bash - - - name: Checkout Auth Controller package - uses: actions/checkout@v4 - with: - repository: flagsmith/flagsmith-auth-controller - token: ${{ inputs.github_access_token }} - ref: ${{ inputs.flagsmith_auth_controller_revision }} - path: ./flagsmith-auth-controller - - - name: Integrate Auth Controller module - run: mv ./flagsmith-auth-controller/auth_controller ./api - shell: bash - - - name: Checkout RBAC module - uses: actions/checkout@v4 - with: - repository: flagsmith/flagsmith-rbac - token: ${{ inputs.github_access_token }} - ref: ${{ inputs.flagsmith_rbac_revision }} - path: ./flagsmith-rbac - - - name: Integrate RBAC module - run: mv ./flagsmith-rbac/rbac ./api - shell: bash - - - name: Set ECR tag - id: ecr-tag-variable - run: echo "tag=${{ inputs.image_tag }}" >> $GITHUB_OUTPUT - shell: bash - - - name: Write git info to Docker image - run: | - echo ${{ github.sha }} > api/CI_COMMIT_SHA - shell: bash - - - name: Write file to indicate SaaS - run: | - echo '' > api/SAAS_DEPLOYMENT - shell: bash - - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1-node16 with: @@ -129,38 +56,13 @@ runs: aws-secret-access-key: ${{ inputs.aws_secret_access_key }} aws-region: eu-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Write PGP key to a file - run: | - cat << EOF > sse_pgp_pkey - ${{ inputs.sse_pgp_private_key }} - EOF - shell: bash - - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - IMAGE_TAG: ${{ steps.ecr-tag-variable.outputs.tag }} - ECR_REPOSITORY: ${{ inputs.aws_ecr_repository_arn }} - DOCKER_BUILDKIT: '1' - run: | - echo "Building docker image with URL: " - echo $ECR_REPOSITORY:$IMAGE_TAG - docker build --secret id=sse_pgp_pkey,src=./sse_pgp_pkey -t $ECR_REPOSITORY:$IMAGE_TAG -f api/Dockerfile --build-arg SAML_INSTALLED=1 --build-arg POETRY_OPTS="--with saml,auth-controller,workflows" --build-arg GH_TOKEN=${{ inputs.github_access_token }} . - docker push $ECR_REPOSITORY:$IMAGE_TAG - echo "image=$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT - shell: bash - - name: Fill in the new image ID in the Amazon ECS API task definition id: task-def-api uses: aws-actions/amazon-ecs-render-task-definition@v1 with: task-definition: ${{ inputs.aws_task_definitions_directory_path }}/ecs-task-definition-web.json container-name: flagsmith-api - image: ${{ steps.build-image.outputs.image }} + image: ${{ inputs.api_ecr_image_url }} # This is used in both the SQL migrations and the Dynamo Identity Migrations - name: Fill in the new image ID in the Amazon ECS migration task definition @@ -169,7 +71,7 @@ runs: with: task-definition: ${{ inputs.aws_task_definitions_directory_path }}/ecs-task-definition-migration.json container-name: flagsmith-api-migration - image: ${{ steps.build-image.outputs.image }} + image: ${{ inputs.api_ecr_image_url }} - name: Register and perform SQL schema migration id: register-migrate-task diff --git a/.github/actions/e2e-tests/action.yml b/.github/actions/e2e-tests/action.yml index db9f9744bef1..3b68f8a25fcd 100644 --- a/.github/actions/e2e-tests/action.yml +++ b/.github/actions/e2e-tests/action.yml @@ -47,6 +47,5 @@ runs: env: E2E_TEST_TOKEN: ${{ inputs.e2e_test_token }} SLACK_TOKEN: ${{ inputs.slack_token }} - STATIC_ASSET_CDN_URL: / ENV: ${{ inputs.environment }} GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/.reusable-deploy-ecs.yml b/.github/workflows/.reusable-deploy-ecs.yml new file mode 100644 index 000000000000..c694eeb7f40b --- /dev/null +++ b/.github/workflows/.reusable-deploy-ecs.yml @@ -0,0 +1,123 @@ +# reusable workflow +name: Deploy to ECS and Test + +on: + workflow_call: + inputs: + environment: + type: string + description: The environment to deploy to + required: true + saas-image-name: + type: string + description: SaaS image name + required: false + default: flagsmith-saas-api + +jobs: + docker-build-saas-api: + environment: ${{ inputs.environment }} + runs-on: ubuntu-latest + outputs: + image-url: ${{ steps.login-ecr.outputs.registry }}/${{ inputs.saas-image-name }}:${{ steps.meta.outputs.version }} + + permissions: + id-token: write + contents: read + + steps: + - name: Cloning repo + uses: actions/checkout@v4 + + - name: Set up Depot CLI + uses: depot/setup-action@v1 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + aws-access-key-id: ${{ vars.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ steps.login-ecr.outputs.registry }}/${{ inputs.saas-image-name }} + tags: | + type=ref,event=branch + type=ref,event=tag + + - name: Build saas-api image + uses: depot/build-push-action@v1 + with: + target: saas-api + context: . + build-args: CI_COMMIT_SHA=${{ github.sha }} + secrets: | + github_private_cloud_token=${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + sse_pgp_pkey=${{ secrets.SSE_PGP_PRIVATE_KEY }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + deploy: + needs: docker-build-saas-api + environment: ${{ inputs.environment }} + runs-on: ubuntu-latest + steps: + - name: Cloning repo + uses: actions/checkout@v4 + + - name: Deploy API to ${{ inputs.environment }} + id: deploy-api + uses: ./.github/actions/api-deploy-ecs + with: + aws_access_key_id: ${{ vars.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_ecs_cluster_name: ${{ vars.AWS_ECS_CLUSTER_NAME }} + aws_ecs_cluster_arn: ${{ vars.AWS_ECS_CLUSTER_ARN }} + aws_ecs_service_name: ${{ vars.AWS_ECS_SERVICE_NAME }} + aws_vpc_subnet_id: ${{ vars.AWS_VPC_SUBNET_ID }} + aws_ecs_security_group_id: ${{ vars.AWS_ECS_SECURITY_GROUP_ID }} + aws_ecr_repository_arn: ${{ vars.AWS_ECR_REPOSITORY_ARN }} + aws_identity_migration_event_bus_name: ${{ vars.AWS_IDENTITY_MIGRATION_EVENT_BUS_NAME }} + aws_identity_migration_event_bus_rule_id: ${{ vars.AWS_IDENTITY_MIGRATION_EVENT_BUS_RULE_ID }} + aws_identity_migration_task_role_arn: ${{ vars.AWS_IDENTITY_MIGRATION_TASK_ROLE_ARN }} + aws_task_definitions_directory_path: infrastructure/aws/${{ inputs.environment }} + api_ecr_image_url: ${{ needs.docker-build-saas-api.outputs.image-url }} + + - name: Deploy Task processor to ${{ inputs.environment }} + uses: ./.github/actions/task-processor-deploy-ecs + with: + aws_access_key_id: ${{ vars.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_ecs_cluster_name: ${{ vars.AWS_ECS_CLUSTER_NAME }} + aws_ecs_service_name: ${{ vars.AWS_ECS_TASK_PROCESSOR_SERVICE_NAME }} + aws_task_definitions_directory_path: infrastructure/aws/${{ inputs.environment }} + api_ecr_image_url: ${{ needs.docker-build-saas-api.outputs.image-url }} + + run-tests: + needs: deploy + runs-on: ubuntu-latest + name: Run E2E Tests + environment: ${{ inputs.environment }} + concurrency: + group: e2e-tests-${{ inputs.environment }} + cancel-in-progress: true + + steps: + - name: Cloning repo + uses: actions/checkout@v4 + + - name: Run E2E tests against ${{ inputs.environment }} + uses: ./.github/actions/e2e-tests + with: + e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }} + slack_token: ${{ secrets.SLACK_TOKEN }} + environment: ${{ inputs.environment }} diff --git a/.github/workflows/.reusable-docker-build.yml b/.github/workflows/.reusable-docker-build.yml index 856180401aef..d22c6b095806 100644 --- a/.github/workflows/.reusable-docker-build.yml +++ b/.github/workflows/.reusable-docker-build.yml @@ -6,16 +6,21 @@ on: inputs: registry-url: type: string - description: Github container registry base URL + description: Github Container Registry base URL required: false default: ghcr.io file: type: string description: Path to the Dockerfile - required: true + required: false + default: Dockerfile + target: + type: string + description: Sets the target stage to build + required: false image-name: type: string - description: Image slug + description: Image name required: true build-args: type: string @@ -23,13 +28,17 @@ on: required: false scan: type: boolean - description: Whether to scan image for vulnerabilities + description: Whether to scan built image for vulnerabilities required: false default: true outputs: image: description: Resulting image specifier value: ${{ inputs.registry-url }}/flagsmith/${{ inputs.image-name }}:${{ jobs.build.outputs.version }} + secrets: + secrets: + description: List of secrets to expose to the build (e.g., `key=string, GIT_AUTH_TOKEN=mytoken`) + required: false jobs: build: @@ -50,7 +59,7 @@ jobs: - name: Set up Depot CLI uses: depot/setup-action@v1 - - name: Login to GitHub Container Registry + - name: Login to Github Container Registry uses: docker/login-action@v3 with: registry: ${{ inputs.registry-url }} @@ -67,18 +76,39 @@ jobs: type=ref,event=branch type=ref,event=tag type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} - name: Build and push image uses: depot/build-push-action@v1 with: + context: . push: true - build-args: ${{ inputs.build-args }} + platforms: linux/amd64,linux/arm64 + secrets: ${{ secrets.secrets }} + target: ${{ inputs.target }} + build-args: | + CI_COMMIT_SHA=${{ github.sha }} + ${{ inputs.build-args }} file: ${{ inputs.file }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + provenance: false - name: Run Trivy vulnerability scanner - if: ${{ inputs.scan }} + id: trivy + if: inputs.scan uses: aquasecurity/trivy-action@master with: image-ref: ${{ inputs.registry-url }}/flagsmith/${{ inputs.image-name }}:${{ steps.meta.outputs.version }} + format: sarif + output: trivy-results.sarif + env: + TRIVY_USERNAME: ${{ github.actor }} + TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: inputs.scan && (success() || failure()) + with: + sarif_file: trivy-results.sarif diff --git a/.github/workflows/.reusable-docker-e2e-tests.yml b/.github/workflows/.reusable-docker-e2e-tests.yml index d85dbc5cf1da..535e6384941a 100644 --- a/.github/workflows/.reusable-docker-e2e-tests.yml +++ b/.github/workflows/.reusable-docker-e2e-tests.yml @@ -4,6 +4,11 @@ name: Run Docker E2E tests on: workflow_call: inputs: + registry-url: + type: string + description: Github Container Registry base URL + required: false + default: ghcr.io api-image: type: string description: Core API Docker image to use, e.g., `ghcr.io/flagsmith/flagsmith-api:main` @@ -32,6 +37,13 @@ jobs: - name: Cloning repo uses: actions/checkout@v4 + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry-url }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Run tests on dockerised frontend uses: nick-fields/retry@v3 with: diff --git a/.github/workflows/.reusable-docker-publish.yml b/.github/workflows/.reusable-docker-publish.yml new file mode 100644 index 000000000000..32b12aab5701 --- /dev/null +++ b/.github/workflows/.reusable-docker-publish.yml @@ -0,0 +1,66 @@ +# reusable workflow +name: Publish Docker Image + +on: + workflow_call: + inputs: + registry-url: + type: string + description: Github Container Registry base URL + required: false + default: ghcr.io + source-images: + type: string + description: Source image to publish + required: true + target-images: + type: string + description: Target image names + required: true + +jobs: + publish: + name: Publish ${{ inputs.source-images }} to ${{ inputs.target-images }} + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + + steps: + - name: Cloning repo + uses: actions/checkout@v4 + with: + sparse-checkout: depot.json + sparse-checkout-cone-mode: false + + - name: Login to Github Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry-url }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.target-images }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + # Setup Docker buildx with Depot builder so imagetools have access to Depot cache + - uses: depot/use-action@v1 + + - name: Publish Image + uses: kphrx/docker-buildx-imagetools-action@v0.1.2 + with: + sources: ${{ inputs.source-images }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/api-deploy-production-ecs.yml b/.github/workflows/api-deploy-production-ecs.yml index 9624c6a12ecc..66118043bddf 100644 --- a/.github/workflows/api-deploy-production-ecs.yml +++ b/.github/workflows/api-deploy-production-ecs.yml @@ -10,62 +10,7 @@ on: - infrastructure/aws/production/** jobs: - deploy-production-ecs: - runs-on: ubuntu-latest - name: API Deploy to Production ECS - environment: production - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Deploy API to Production - id: deploy-api - uses: ./.github/actions/api-deploy-ecs - with: - github_access_token: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} - aws_access_key_id: AKIARHES7IUAU2LR2B5K - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_ecs_cluster_name: flagsmith-api-cluster-eu-west-2-f0b0108 - aws_ecs_cluster_arn: arn:aws:ecs:eu-west-2:084060095745:cluster/flagsmith-api-cluster-eu-west-2-f0b0108 - aws_ecs_service_name: flagsmith-svc-eu-west-2-c3cd356 - aws_vpc_subnet_id: subnet-40fd6629 - aws_ecs_security_group_id: sg-0ef0e8f66f890b80c - aws_ecr_repository_arn: 084060095745.dkr.ecr.eu-west-2.amazonaws.com/flagsmith-ecr-934e8a7 - aws_identity_migration_event_bus_name: identity_migration-d46ed1a - aws_identity_migration_event_bus_rule_id: identity_migration-b03c433 - aws_identity_migration_task_role_arn: arn:aws:iam::084060095745:role/task-exec-role-741a7e3 - aws_task_definitions_directory_path: infrastructure/aws/production - flagsmith_saml_revision: v1.6.2 - flagsmith_rbac_revision: v0.7.0 - sse_pgp_private_key: ${{ secrets.SSE_PGP_PRIVATE_KEY }} - - - name: Deploy task processor to Production - uses: ./.github/actions/task-processor-deploy-ecs - with: - aws_access_key_id: AKIARHES7IUAU2LR2B5K - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_ecs_cluster_name: flagsmith-api-cluster-eu-west-2-f0b0108 - aws_ecs_service_name: flagsmith-task-processor-svc-eu-west-2-bf77140 - aws_task_definitions_directory_path: infrastructure/aws/production - api_ecr_image_url: ${{ steps.deploy-api.outputs.api_ecr_image_url }} - - run-tests: - runs-on: ubuntu-latest - name: Run E2E Tests - environment: production - needs: deploy-production-ecs - concurrency: - group: e2e-tests-prod - cancel-in-progress: true - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Run E2E tests against production - uses: ./.github/actions/e2e-tests - with: - e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }} - slack_token: ${{ secrets.SLACK_TOKEN }} - environment: prod + deploy-ecs: + uses: ./.github/workflows/.reusable-deploy-ecs.yml + with: + environment: production diff --git a/.github/workflows/api-deploy-staging-ecs.yml b/.github/workflows/api-deploy-staging-ecs.yml index f843dce5e483..0cc139aeb5e1 100644 --- a/.github/workflows/api-deploy-staging-ecs.yml +++ b/.github/workflows/api-deploy-staging-ecs.yml @@ -11,62 +11,7 @@ on: workflow_dispatch: jobs: - deploy-staging-ecs: - runs-on: ubuntu-latest - name: API Deploy to Staging ECS - environment: staging - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Deploy API to Staging - id: deploy-api - uses: ./.github/actions/api-deploy-ecs - with: - github_access_token: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} - aws_access_key_id: AKIAUM26IRCPASKFW2X5 - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_ecs_cluster_name: flagsmith-api-cluster-eu-west-2-f241261 - aws_ecs_cluster_arn: arn:aws:ecs:eu-west-2:302456015006:cluster/flagsmith-api-cluster-eu-west-2-f241261 - aws_ecs_service_name: flagsmith-svc-eu-west-2-8bb18de - aws_vpc_subnet_id: subnet-1b0b8861 - aws_ecs_security_group_id: sg-08632d6fb4cb0fdf3 - aws_ecr_repository_arn: 302456015006.dkr.ecr.eu-west-2.amazonaws.com/flagsmith-ecr-d247ba2 - aws_identity_migration_event_bus_name: identity_migration-fb41b5d - aws_identity_migration_event_bus_rule_id: identity_migration-08330ed - aws_identity_migration_task_role_arn: arn:aws:iam::302456015006:role/task-exec-role-6fb76f6 - aws_task_definitions_directory_path: infrastructure/aws/staging - flagsmith_saml_revision: v1.6.2 - flagsmith_rbac_revision: v0.7.0 - sse_pgp_private_key: ${{ secrets.SSE_PGP_PRIVATE_KEY }} - - - name: Deploy task processor to Staging - uses: ./.github/actions/task-processor-deploy-ecs - with: - aws_access_key_id: AKIAUM26IRCPASKFW2X5 - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_ecs_cluster_name: flagsmith-api-cluster-eu-west-2-f241261 - aws_ecs_service_name: flagsmith-task-processor-svc-eu-west-2-792c644 - aws_task_definitions_directory_path: infrastructure/aws/staging - api_ecr_image_url: ${{ steps.deploy-api.outputs.api_ecr_image_url }} - - run-tests: - runs-on: ubuntu-latest - name: Run E2E Tests - environment: staging - needs: deploy-staging-ecs - concurrency: - group: e2e-tests-staging - cancel-in-progress: true - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Run E2E tests against staging - uses: ./.github/actions/e2e-tests - with: - e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }} - slack_token: ${{ secrets.SLACK_TOKEN }} - environment: staging + deploy-ecs: + uses: ./.github/workflows/.reusable-deploy-ecs.yml + with: + environment: staging diff --git a/.github/workflows/api-docker-publish-image.yml b/.github/workflows/api-docker-publish-image.yml deleted file mode 100644 index c74e4cd87ef3..000000000000 --- a/.github/workflows/api-docker-publish-image.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: API Publish Docker Images - -on: - push: - tags: - - '*' - paths: - - api/** - - .github/** - -jobs: - build-api-dockerhub: - runs-on: ubuntu-latest - name: API Publish Docker Image - - permissions: - id-token: write - contents: read - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - flagsmith/flagsmith-api - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Write git info to Docker image - run: | - echo ${{ github.sha }} > api/CI_COMMIT_SHA - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push images - id: docker_build - uses: depot/build-push-action@v1 - with: - platforms: linux/amd64,linux/arm64 - file: api/Dockerfile - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/frontend-docker-publish-image.yml b/.github/workflows/frontend-docker-publish-image.yml deleted file mode 100644 index e0066f3b3ea1..000000000000 --- a/.github/workflows/frontend-docker-publish-image.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Frontend Publish Docker Images - -on: - push: - tags: - - '*' - paths: - - frontend/** - - .github/** - -jobs: - build-frontend-dockerhub: - runs-on: ubuntu-latest - name: Frontend Publish Docker Image - - permissions: - id-token: write - contents: read - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - flagsmith/flagsmith-frontend - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Write git info to Docker image - run: | - cd frontend - echo ${{ github.sha }} > CI_COMMIT_SHA - - - name: Download features - run: > - curl --location --request GET 'https://api.flagsmith.com/api/v1/flags/' --header 'X-Environment-Key: ${{ - secrets.FLAGSMITH_ENVIRONMENT_KEY }}' > frontend/flags.json - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push images - id: docker_build - uses: depot/build-push-action@v1 - with: - platforms: linux/amd64,linux/arm64 - file: frontend/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/frontend-e2e-docker-publish-image.yml b/.github/workflows/frontend-e2e-docker-publish-image.yml deleted file mode 100644 index 25911796a35a..000000000000 --- a/.github/workflows/frontend-e2e-docker-publish-image.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Frontend E2E Publish Docker Images - -on: - push: - branches: - - main - tags: - - '*' - paths: - - frontend/** - - .github/** - -jobs: - build-frontend-e2e-dockerhub: - runs-on: ubuntu-latest - name: Frontend E2E Publish Docker Image - - permissions: - id-token: write - contents: read - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - flagsmith/flagsmith-e2e-tests - flavor: | - latest=${{ startsWith(github.ref, 'refs/heads/main') }} - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Build and push E2E testing docker image - id: docker_build - uses: depot/build-push-action@v1 - with: - file: frontend/Dockerfile.e2e - push: true - tags: ${{ steps.meta.outputs.tags }} - build-args: | - CHROME_URL=${{ secrets.CHROME_URL }} diff --git a/.github/workflows/frontend-test-staging.yml b/.github/workflows/frontend-test-staging.yml index 814c43f0e1d0..4828f38cd794 100644 --- a/.github/workflows/frontend-test-staging.yml +++ b/.github/workflows/frontend-test-staging.yml @@ -4,25 +4,20 @@ on: workflow_dispatch jobs: run-staging-e2e-tests: - if: github.event.pull_request.draft == false runs-on: ubuntu-latest - name: Full Staging E2E tests + name: Run E2E Tests environment: staging + concurrency: + group: e2e-tests-staging + cancel-in-progress: true steps: - name: Cloning repo uses: actions/checkout@v4 - - name: Run Staging E2E Tests - working-directory: frontend - env: - ENV: staging - SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - STATIC_ASSET_CDN_URL: / - run: |- - wget -q ${{ secrets.CHROME_URL }} - sudo apt install --allow-downgrades -y ./google-chrome*.deb -f - google-chrome --version - node -v - npm ci - npm run test:staging + - name: Run E2E tests against staging + uses: ./.github/actions/e2e-tests + with: + e2e_test_token: ${{ secrets.E2E_TEST_TOKEN }} + slack_token: ${{ secrets.SLACK_TOKEN }} + environment: staging diff --git a/.github/workflows/platform-docker-build-e2e-image.yml b/.github/workflows/platform-docker-build-e2e-image.yml index d0926e388900..54e305d18b03 100644 --- a/.github/workflows/platform-docker-build-e2e-image.yml +++ b/.github/workflows/platform-docker-build-e2e-image.yml @@ -14,7 +14,7 @@ jobs: - name: Cloning repo uses: actions/checkout@v4 - - name: Login to GitHub Container Registry + - name: Login to Github Container Registry uses: docker/login-action@v3 with: registry: ghcr.io diff --git a/.github/workflows/platform-docker-build-test-publish.yml b/.github/workflows/platform-docker-build-test-publish.yml new file mode 100644 index 000000000000..c35e3919dce8 --- /dev/null +++ b/.github/workflows/platform-docker-build-test-publish.yml @@ -0,0 +1,161 @@ +name: Platform Build Docker Images, Test, and Publish + +on: + push: + branches: + - main + release: + types: + - released + +jobs: + docker-build-api: + name: Build API Image + uses: ./.github/workflows/.reusable-docker-build.yml + with: + target: oss-api + image-name: flagsmith-api + + docker-build-frontend: + name: Build Frontend Image + uses: ./.github/workflows/.reusable-docker-build.yml + with: + target: oss-frontend + image-name: flagsmith-frontend + + docker-build-unified: + name: Build Unified Image + uses: ./.github/workflows/.reusable-docker-build.yml + with: + target: oss-unified + image-name: flagsmith + + docker-build-private-cloud-api: + name: Build Private Cloud API Image + uses: ./.github/workflows/.reusable-docker-build.yml + with: + target: private-cloud-api + image-name: flagsmith-private-cloud-api + secrets: + secrets: | + github_private_cloud_token=${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + + docker-build-private-cloud: + name: Build Private Cloud Image + uses: ./.github/workflows/.reusable-docker-build.yml + with: + target: private-cloud-unified + image-name: flagsmith-private-cloud + secrets: + secrets: | + github_private_cloud_token=${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + + docker-build-e2e: + name: Build E2E Image + uses: ./.github/workflows/.reusable-docker-build.yml + with: + file: frontend/Dockerfile.e2e + image-name: flagsmith-e2e + scan: false + + run-e2e-tests: + needs: [docker-build-api, docker-build-private-cloud-api, docker-build-e2e] + uses: ./.github/workflows/.reusable-docker-e2e-tests.yml + with: + api-image: ${{ matrix.api-image }} + e2e-image: ${{ needs.docker-build-e2e.outputs.image }} + + strategy: + matrix: + api-image: + - ${{ needs.docker-build-api.outputs.image }} + - ${{ needs.docker-build-private-cloud-api.outputs.image }} + + docker-publish-api: + needs: [docker-build-api, run-e2e-tests] + uses: ./.github/workflows/.reusable-docker-publish.yml + if: github.event_name == 'release' + with: + source-images: ${{ needs.docker-build-api.outputs.image }} + target-images: flagsmith/flagsmith-api + + docker-publish-frontend: + needs: [docker-build-frontend, run-e2e-tests] + uses: ./.github/workflows/.reusable-docker-publish.yml + if: github.event_name == 'release' + with: + source-images: ${{ needs.docker-build-frontend.outputs.image }} + target-images: flagsmith/flagsmith-frontend + + docker-publish-unified: + needs: [docker-build-unified, run-e2e-tests] + uses: ./.github/workflows/.reusable-docker-publish.yml + if: github.event_name == 'release' + with: + source-images: ${{ needs.docker-build-unified.outputs.image }} + target-images: flagsmith/flagsmith + + docker-publish-private-cloud-api: + needs: [docker-build-private-cloud-api, run-e2e-tests] + uses: ./.github/workflows/.reusable-docker-publish.yml + if: github.event_name == 'release' + with: + source-images: ${{ needs.docker-build-private-cloud-api.outputs.image }} + target-images: flagsmith/flagsmith-private-cloud-api + + docker-publish-private-cloud: + needs: [docker-build-private-cloud, run-e2e-tests] + uses: ./.github/workflows/.reusable-docker-publish.yml + if: github.event_name == 'release' + with: + source-images: ${{ needs.docker-build-private-cloud.outputs.image }} + target-images: flagsmith/flagsmith-private-cloud + + update-charts: + needs: [docker-publish-api, docker-publish-frontend, docker-publish-unified] + runs-on: ubuntu-latest + steps: + - name: Checkout Target Charts Repository to update yaml + uses: actions/checkout@v4 + with: + repository: flagsmith/flagsmith-charts + path: chart + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up release tag variables + id: version-trim + run: | + TAG=${{github.ref_name}} + echo "version=${TAG#v}" >> $GITHUB_OUTPUT + + - name: Run YAML to Github Output Action + id: yaml-output + uses: christian-ci/action-yaml-github-output@v2 + with: + file_path: './chart/charts/flagsmith/Chart.yaml' + + - uses: us-ignite/action-bump-semver@main + id: bump-semver + with: + current_version: ${{ steps.yaml-output.outputs.version }} + level: minor + + - name: Update flagsmith-charts values.yaml with latest docker version + uses: fjogeleit/yaml-update-action@main + with: + token: ${{ secrets.FLAGSMITH_CHARTS_GITHUB_TOKEN }} + repository: flagsmith/flagsmith-charts + workDir: chart + masterBranchName: 'main' + targetBranch: 'main' + branch: docker-image-version-bump-${{ steps.version-trim.outputs.version }} + commitChange: true + createPR: true + message: 'Flagsmith docker image version bump' + description: 'Automated PR generated by a release event in https://github.com/Flagsmith/flagsmith' + valueFile: 'charts/flagsmith/Chart.yaml' + changes: | + { + "appVersion": "${{ steps.version-trim.outputs.version }}", + "version": "${{ steps.bump-semver.outputs.new_version }}" + } diff --git a/.github/workflows/platform-docker-publish-all-features-image.yml b/.github/workflows/platform-docker-publish-all-features-image.yml deleted file mode 100644 index f93f5d9883ff..000000000000 --- a/.github/workflows/platform-docker-publish-all-features-image.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Publish Flagsmith Private Cloud Image - -on: - push: - tags: - - '*' - -env: - FLAGSMITH_SAML_REVISION: v1.6.2 - FLAGSMITH_RBAC_REVISION: v0.7.0 - -jobs: - build-dockerhub: - name: Platform Publish Docker Image - runs-on: ubuntu-latest - - permissions: - id-token: write - contents: read - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Checkout SAML package - uses: actions/checkout@v4 - with: - repository: flagsmith/flagsmith-saml - token: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} - ref: ${{ env.flagsmith_saml_revision }} - path: ./flagsmith-saml - - - name: Integrate SAML module - run: | - mv ./flagsmith-saml/saml ./api - - - name: Checkout Auth Controller package - uses: actions/checkout@v4 - with: - repository: flagsmith/flagsmith-auth-controller - token: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} - ref: ${{ env.FLAGSMITH_AUTH_CONTROLLER_REVISION }} - path: ./flagsmith-auth-controller - - - name: Integrate Auth Controller module - run: mv ./flagsmith-auth-controller/auth_controller ./api - shell: bash - - - name: Checkout RBAC module - uses: actions/checkout@v4 - with: - repository: flagsmith/flagsmith-rbac - token: ${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} - ref: ${{ env.flagsmith_rbac_revision }} - path: ./flagsmith-rbac - - - name: Integrate RBAC module - run: mv ./flagsmith-rbac/rbac ./api - shell: bash - - - name: Write git info to Docker image - run: | - cd api - echo ${{ github.sha }} > CI_COMMIT_SHA - echo '' > ENTERPRISE_VERSION - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - flagsmith/flagsmith-private-cloud - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push images - id: docker_build - uses: depot/build-push-action@v1 - with: - platforms: linux/amd64,linux/arm64 - file: Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - context: . - build-args: | - SAML_INSTALLED=1 - POETRY_OPTS=--with saml,auth-controller,ldap,workflows - GH_TOKEN=${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} diff --git a/.github/workflows/platform-docker-publish-image.yml b/.github/workflows/platform-docker-publish-image.yml deleted file mode 100644 index fedfa24ca4ef..000000000000 --- a/.github/workflows/platform-docker-publish-image.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Platform Publish Docker Images - -on: - push: - tags: - - '*' - -jobs: - build-dockerhub: - runs-on: ubuntu-latest - name: Platform Publish Docker Image - - permissions: - id-token: write - contents: read - - steps: - - name: Cloning repo - uses: actions/checkout@v4 - - - name: Checkout Target Charts Repository to update yaml - uses: actions/checkout@v4 - with: - repository: flagsmith/flagsmith-charts - path: chart - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - flagsmith/flagsmith - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - - name: Write git info to Docker image - run: | - echo ${{ github.sha }} > api/CI_COMMIT_SHA - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push images - id: docker_build - uses: depot/build-push-action@v1 - with: - platforms: linux/amd64,linux/arm64 - file: Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - context: . - - - name: Set up release tag variables - run: | - TAG=${{github.ref_name}} - echo "version_trim=${TAG#v}" >> $GITHUB_ENV - - - name: Run YAML to Github Output Action - id: yaml-output - uses: christian-ci/action-yaml-github-output@v2 - with: - file_path: './chart/charts/flagsmith/Chart.yaml' - - - uses: us-ignite/action-bump-semver@main - id: bump-semver - with: - current_version: ${{ steps.yaml-output.outputs.version }} - level: minor - - - name: Update flagsmith-charts values.yaml with latest docker version - uses: fjogeleit/yaml-update-action@main - with: - token: ${{ secrets.FLAGSMITH_CHARTS_GITHUB_TOKEN }} - repository: flagsmith/flagsmith-charts - workDir: chart - masterBranchName: 'main' - targetBranch: 'main' - branch: docker-image-version-bump-${{ env.version_trim }} - commitChange: true - createPR: true - message: 'Flagsmith docker image version bump' - description: 'Automated PR generated by a release event in https://github.com/Flagsmith/flagsmith' - valueFile: 'charts/flagsmith/Chart.yaml' - changes: | - { - "appVersion": "${{ env.version_trim }}", - "version": "${{ steps.bump-semver.outputs.new_version }}" - } diff --git a/.github/workflows/platform-pull-request.yml b/.github/workflows/platform-pull-request.yml index ec57890dfa69..3011ca6d6096 100644 --- a/.github/workflows/platform-pull-request.yml +++ b/.github/workflows/platform-pull-request.yml @@ -60,7 +60,7 @@ jobs: name: Build Unified Image uses: ./.github/workflows/.reusable-docker-build.yml with: - file: Dockerfile + target: oss-unified image-name: flagsmith docker-build-api: @@ -68,7 +68,7 @@ jobs: name: Build API Image uses: ./.github/workflows/.reusable-docker-build.yml with: - file: api/Dockerfile + target: oss-api image-name: flagsmith-api docker-build-frontend: @@ -76,7 +76,7 @@ jobs: name: Build Frontend Image uses: ./.github/workflows/.reusable-docker-build.yml with: - file: frontend/Dockerfile + target: oss-frontend image-name: flagsmith-frontend docker-build-e2e: @@ -88,17 +88,31 @@ jobs: image-name: flagsmith-e2e scan: false + docker-build-private-cloud: + if: github.event.pull_request.draft == false + name: Build Private Cloud Image + uses: ./.github/workflows/.reusable-docker-build.yml + with: + target: private-cloud-unified + image-name: flagsmith-private-cloud + secrets: + secrets: | + github_private_cloud_token=${{ secrets.GH_PRIVATE_ACCESS_TOKEN }} + run-e2e-tests: - needs: [docker-build-api, docker-build-e2e] + needs: [docker-build-api, docker-build-private-cloud, docker-build-e2e] uses: ./.github/workflows/.reusable-docker-e2e-tests.yml with: - api-image: ${{ needs.docker-build-api.outputs.image }} e2e-image: ${{ needs.docker-build-e2e.outputs.image }} + api-image: ${{ matrix.api-image }} concurrency: ${{ matrix.args.concurrency }} tests: ${{ matrix.args.tests }} strategy: matrix: + api-image: + - ${{ needs.docker-build-api.outputs.image }} + - ${{ needs.docker-build-private-cloud.outputs.image }} args: - tests: segment-part-1 environment concurrency: 1 diff --git a/.github/workflows/platform-test-merge-to-main.yml b/.github/workflows/platform-test-merge-to-main.yml deleted file mode 100644 index aefc752e4087..000000000000 --- a/.github/workflows/platform-test-merge-to-main.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Platform Test Merge to Main - -on: - push: - branches: - - main - -jobs: - docker-build-api: - name: Build API Image - uses: ./.github/workflows/.reusable-docker-build.yml - with: - file: api/Dockerfile - image-name: flagsmith-api - - docker-build-e2e: - name: Build E2E Image - uses: ./.github/workflows/.reusable-docker-build.yml - with: - file: frontend/Dockerfile.e2e - image-name: flagsmith-e2e - scan: false - - run-e2e-tests: - needs: [docker-build-api, docker-build-e2e] - uses: ./.github/workflows/.reusable-docker-e2e-tests.yml - with: - api-image: ${{ needs.docker-build-api.outputs.image }} - e2e-image: ${{ needs.docker-build-e2e.outputs.image }} diff --git a/Dockerfile b/Dockerfile index d983ae6e66ac..8afc45d4f8d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,65 @@ -# Step 1 - Build Front End Application -FROM node:16 AS build-frontend +# Your usual Flagsmith release produces of a number of shippable Docker images: + +# - Private cloud: API, Unified +# - SaaS: API +# - Open Source: API, Frontend, Unified + +# This Dockerfile is meant to build all of the above via composable, interdependent stages. +# The goal is to have as DRY as possible build for all the targets. + +# Usage Examples + +# Build an Open Source API image: +# $ docker build -t flagsmith-api:dev --target oss-api . + +# Build an Open Source Unified image: +# (`oss-unified` stage is the default one, so there's no need to specify a target stage) +# $ docker build -t flagsmith:dev . + +# Build a SaaS API image: +# $ GH_TOKEN=$(gh auth token) docker build -t flagsmith-saas-api:dev --target saas-api \ +# --secret="id=sse_pgp_pkey,src=./sse_pgp_pkey.key"\ +# --secret="id=github_private_cloud_token,env=GH_TOKEN" . + +# Build a Private Cloud Unified image: +# $ GH_TOKEN=$(gh auth token) docker build -t flagsmith-private-cloud:dev --target private-cloud-unified \ +# --secret="id=github_private_cloud_token,env=GH_TOKEN" . + +# Table of Contents +# Stages are described as stage-name [dependencies] + +# - Intermediary stages +# * build-node [node] +# * build-node-django [build-node] +# * build-node-selfhosted [build-node] +# * build-python [python] +# * build-python-private [build-python] +# * api-runtime [python:slim] +# * api-runtime-private [api-runtime] + +# - Target (shippable) stages +# * private-cloud-api [api-runtime-private, build-python-private] +# * private-cloud-unified [api-runtime-private, build-python-private, build-node-django] +# * saas-api [api-runtime-private, build-python-private] +# * oss-api [api-runtime, build-python] +# * oss-frontend [node:slim, build-node-selfhosted] +# * oss-unified [api-runtime, build-python, build-node-django] + +ARG CI_COMMIT_SHA=dev + +# Pin runtimes versions +ARG NODE_VERSION=16 +ARG PYTHON_VERSION=3.11 +ARG PYTHON_SITE_DIR=/usr/local/lib/python${PYTHON_VERSION}/site-packages + +FROM node:${NODE_VERSION} as node +FROM node:${NODE_VERSION}-slim as node-slim +FROM python:${PYTHON_VERSION} as python +FROM python:${PYTHON_VERSION}-slim as python-slim + +# - Intermediary stages +# * build-node +FROM node AS build-node # Copy the files required to install npm packages WORKDIR /app @@ -7,71 +67,155 @@ COPY frontend/package.json frontend/package-lock.json frontend/.npmrc ./frontend COPY frontend/bin/ ./frontend/bin/ COPY frontend/env/ ./frontend/env/ -# since ENV is only used for the purposes of copying the correct -# project_${env}.js file to common/project.js, this is a build arg -# which subsequently gets set as an environment variable. This is -# done to avoid confusion since it is not a required run time var. ARG ENV=selfhosted RUN cd frontend && ENV=${ENV} npm ci --quiet --production -# Copy the entire project - Webpack puts compiled assets into the Django folder -COPY . . -ENV STATIC_ASSET_CDN_URL=/static/ -RUN cd frontend && npm run bundledjango +COPY frontend /app/frontend -# Step 2 - Build Python virtualenv -FROM python:3.11 as build-python -WORKDIR /app +# * build-node-django [build-node] +FROM build-node as build-node-django -COPY api/pyproject.toml api/poetry.lock api/Makefile ./ -ARG POETRY_VIRTUALENVS_CREATE=false -RUN make install-poetry -ENV PATH="$PATH:/root/.local/bin" +RUN mkdir /app/api && cd frontend && npm run bundledjango -ARG GH_TOKEN -RUN if [ -n "${GH_TOKEN}" ]; \ - then echo "https://${GH_TOKEN}:@github.com" > ${HOME}/.git-credentials \ - && git config --global credential.helper store; fi; +# * build-node-selfhosted [build-node] +FROM build-node as build-node-selfhosted -ARG POETRY_OPTS -RUN make install-packages opts="${POETRY_OPTS}" +RUN cd frontend && npm run bundle -# Step 3 - Build Django Application -FROM python:3.11-slim as application +# * build-python +FROM python as build-python WORKDIR /app -# Install SAML dependency if required -ARG SAML_INSTALLED="0" -RUN if [ "${SAML_INSTALLED}" = "1" ]; then apt-get update && apt-get install -y xmlsec1; fi; +COPY api/pyproject.toml api/poetry.lock api/Makefile ./ +ENV POETRY_VIRTUALENVS_CREATE=false POETRY_HOME=/usr/local +RUN make install opts='--without dev' + +# * build-python-private [build-python] +FROM build-python AS build-python-private + +# Authenticate git with token, install SAML binary dependency, +# private Python dependencies, and integrate private modules +ARG SAML_REVISION +ARG RBAC_REVISION +RUN --mount=type=secret,id=github_private_cloud_token \ + echo "https://$(cat /run/secrets/github_private_cloud_token):@github.com" > ${HOME}/.git-credentials && \ + git config --global credential.helper store && \ + make install-packages opts='--without dev --with saml,auth-controller,ldap,workflows' && \ + make install-private-modules + +# * api-runtime +FROM python-slim as api-runtime -# arm architecture platform builds need postgres drivers installing via apt ARG TARGETARCH -RUN if [ "${TARGETARCH}" != "amd64" ]; then apt-get update && apt-get install -y libpq-dev && rm -rf /var/lib/apt/lists/*; fi; +RUN if [ "${TARGETARCH}" != "amd64" ]; then \ + apt-get update && apt-get install -y libpq-dev && rm -rf /var/lib/apt/lists/*; fi; -# Copy the python venv from step 2 -COPY --from=build-python /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages -# Copy the bin folder as well to copy the executables created in package installation -COPY --from=build-python /usr/local/bin /usr/local/bin +WORKDIR /app COPY api /app/ COPY .release-please-manifest.json /app/.versions.json -# Compile static Django assets -RUN python /app/manage.py collectstatic --no-input - -# Copy the compiled front end assets from the previous build step -COPY --from=build-frontend /app/api/static /app/static/ -COPY --from=build-frontend /app/api/app/templates/webpack /app/app/templates/webpack - ARG ACCESS_LOG_LOCATION="/dev/null" -ENV ACCESS_LOG_LOCATION=${ACCESS_LOG_LOCATION} -ENV DJANGO_SETTINGS_MODULE=app.settings.production +ENV ACCESS_LOG_LOCATION=${ACCESS_LOG_LOCATION} DJANGO_SETTINGS_MODULE=app.settings.production + +ARG CI_COMMIT_SHA +RUN echo ${CI_COMMIT_SHA} > /app/CI_COMMIT_SHA EXPOSE 8000 +ENTRYPOINT ["/app/scripts/run-docker.sh"] + +CMD ["migrate-and-serve"] + +# * api-runtime-private [api-runtime] +FROM api-runtime as api-runtime-private + +# Install SAML binary dependency +RUN apt-get update && apt-get install -y xmlsec1 && rm -rf /var/lib/apt/lists/* + +# - Target (shippable) stages +# * private-cloud-api [api-runtime-private, build-python-private] +FROM api-runtime-private as private-cloud-api + +ARG PYTHON_SITE_DIR +COPY --from=build-python-private ${PYTHON_SITE_DIR} ${PYTHON_SITE_DIR} +COPY --from=build-python-private /usr/local/bin /usr/local/bin + +RUN python manage.py collectstatic --no-input +RUN touch ./ENTERPRISE_VERSION + USER nobody -ENTRYPOINT ["./scripts/run-docker.sh"] +# * private-cloud-unified [api-runtime-private, build-python-private, build-node-django] +FROM api-runtime-private as private-cloud-unified -# other options below are `migrate` or `serve` -CMD ["migrate-and-serve"] +ARG PYTHON_SITE_DIR +COPY --from=build-python-private ${PYTHON_SITE_DIR} ${PYTHON_SITE_DIR} +COPY --from=build-python-private /usr/local/bin /usr/local/bin +COPY --from=build-node-django /app/api/static /app/static +COPY --from=build-node-django /app/api/app/templates/webpack /app/app/templates/webpack + +RUN python manage.py collectstatic --no-input +RUN touch ./ENTERPRISE_VERSION + +USER nobody + +# * saas-api [api-runtime-private, build-python-private] +FROM api-runtime-private as saas-api + +# Install GnuPG and import private key +RUN --mount=type=secret,id=sse_pgp_pkey \ + apt-get update && apt-get install -y gnupg && \ + gpg --import /run/secrets/sse_pgp_pkey && \ + mv /root/.gnupg /app/; \ + chown -R nobody /app/.gnupg + +ARG PYTHON_SITE_DIR +COPY --from=build-python-private ${PYTHON_SITE_DIR} ${PYTHON_SITE_DIR} +COPY --from=build-python-private /usr/local/bin /usr/local/bin + +RUN python manage.py collectstatic --no-input +RUN touch ./SAAS_DEPLOYMENT + +USER nobody + +# * oss-api [api-runtime, build-python] +FROM api-runtime as oss-api + +ARG PYTHON_SITE_DIR +COPY --from=build-python ${PYTHON_SITE_DIR} ${PYTHON_SITE_DIR} +COPY --from=build-python /usr/local/bin /usr/local/bin + +RUN python manage.py collectstatic --no-input + +USER nobody + +# * oss-frontend [build-node-selfhosted] +FROM node-slim AS oss-frontend + +USER node +WORKDIR /srv/bt + +COPY --from=build-node-selfhosted --chown=node:node /app/frontend . + +ENV NODE_ENV=production + +ARG CI_COMMIT_SHA +RUN echo ${CI_COMMIT_SHA} > /srv/bt/CI_COMMIT_SHA +COPY .release-please-manifest.json /srv/bt/.versions.json + +EXPOSE 8080 +CMD ["node", "./api/index.js"] + +# * oss-unified [api-runtime, build-python, build-node-django] +FROM api-runtime as oss-unified + +ARG PYTHON_SITE_DIR +COPY --from=build-python ${PYTHON_SITE_DIR} ${PYTHON_SITE_DIR} +COPY --from=build-python /usr/local/bin /usr/local/bin +COPY --from=build-node-django /app/api/static /app/static/ +COPY --from=build-node-django /app/api/app/templates/webpack /app/app/templates/webpack + +RUN python manage.py collectstatic --no-input + +USER nobody diff --git a/api/Dockerfile b/api/Dockerfile deleted file mode 100644 index 0e7382b4bc40..000000000000 --- a/api/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ - -# Step 1 - Build Python virtualenv -FROM python:3.11 as build-python -WORKDIR /app - -COPY api/pyproject.toml api/poetry.lock api/Makefile ./ - -ARG POETRY_VIRTUALENVS_CREATE=false -RUN make install-poetry -ENV PATH="$PATH:/root/.local/bin" - -ARG GH_TOKEN -RUN if [ -n "${GH_TOKEN}" ]; \ - then echo "https://${GH_TOKEN}:@github.com" > ${HOME}/.git-credentials \ - && git config --global credential.helper store; fi; - -ARG POETRY_OPTS -RUN make install-packages opts="${POETRY_OPTS}" - -# Step 2 - Build Django Application -FROM python:3.11-slim as application -WORKDIR /app - -# Install SAML dependency if required -ARG SAML_INSTALLED="0" -RUN if [ "${SAML_INSTALLED}" = "1" ]; then apt-get update && apt-get install -y xmlsec1; fi; - -# arm architecture platform builds need postgres drivers installing via apt -ARG TARGETARCH -RUN if [ "${TARGETARCH}" != "amd64" ]; then apt-get update && apt-get install -y libpq-dev && rm -rf /var/lib/apt/lists/*; fi; - -# Install GnuPG(and import private key) if secret file exists -RUN --mount=type=secret,id=sse_pgp_pkey if [ -f /run/secrets/sse_pgp_pkey ]; then \ - apt-get update && apt-get install -y gnupg && gpg --import /run/secrets/sse_pgp_pkey; \ - mv /root/.gnupg /app/; \ - chown -R nobody /app/.gnupg ; fi; - - -# Copy the python venv from step 2 -COPY --from=build-python /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages -# Copy the bin folder as well to copy the executables created in package installation -COPY --from=build-python /usr/local/bin /usr/local/bin - -COPY api/ . -COPY .release-please-manifest.json ./.versions.json - -# Compile static Django assets -RUN python manage.py collectstatic --no-input - -ARG ACCESS_LOG_LOCATION="/dev/null" -ENV ACCESS_LOG_LOCATION=${ACCESS_LOG_LOCATION} -ENV DJANGO_SETTINGS_MODULE=app.settings.production - -EXPOSE 8000 - -USER nobody - -ENTRYPOINT ["./scripts/run-docker.sh"] - -# other options below are `migrate` or `serve` -CMD ["migrate-and-serve"] diff --git a/api/Dockerfile.redhat-ubi b/api/Dockerfile.redhat-ubi deleted file mode 100644 index 598c17227fd5..000000000000 --- a/api/Dockerfile.redhat-ubi +++ /dev/null @@ -1,34 +0,0 @@ -FROM registry.redhat.io/ubi8/python-38 as application - -MAINTAINER Ben Rometsch - -LABEL name="flagsmith-api" \ - vendor="Flagsmith" \ - maintainer="support@flagsmith.com" \ - version="0.0.1" \ - summary="Feature Flags and Remote Config API" \ - description="Feature Flags and Remote Config API" -COPY License /licenses/License - -USER root -RUN yum -y update-minimal --security --sec-severity=Important --sec-severity=Critical -USER 1001 - -ARG POETRY_VIRTUALENVS_CREATE=false -ADD --chown=1001:0 pyproject.toml poetry.lock Makefile . -RUN make install-packages -ENV PATH="$PATH:/root/.local/bin" -RUN make install-packages opts="--no-root --only main" - -ADD --chown=1001:0 docker/ bin/ -ADD --chown=1001:0 . src/ - -RUN python src/manage.py collectstatic --no-input - -ARG ACCESS_LOG_LOCATION="/dev/null" -ENV ACCESS_LOG_LOCATION=${ACCESS_LOG_LOCATION} -ENV DJANGO_SETTINGS_MODULE=app.settings.production - -EXPOSE 8000 - -CMD ["./bin/docker"] diff --git a/api/Makefile b/api/Makefile index b7ef54c76b2c..af30eb4fd1d0 100644 --- a/api/Makefile +++ b/api/Makefile @@ -11,6 +11,9 @@ POETRY_VERSION ?= 1.8.3 GUNICORN_LOGGER_CLASS ?= util.logging.GunicornJsonCapableLogger +SAML_REVISION ?= v1.6.2 +RBAC_REVISION ?= v0.7.0 + -include .env-local -include $(DOTENV_OVERRIDE_FILE) @@ -26,6 +29,13 @@ install-poetry: install-packages: poetry install --no-root $(opts) +.PHONY: install-private-modules +install-private-modules: + $(eval SITE_PACKAGES_DIR := $(shell poetry run python -c 'import site; print(site.getsitepackages()[0])')) + git clone https://github.com/flagsmith/flagsmith-saml --depth 1 --branch ${SAML_REVISION} && mv ./flagsmith-saml/saml $(SITE_PACKAGES_DIR) + git clone https://github.com/flagsmith/flagsmith-rbac --depth 1 --branch ${RBAC_REVISION} && mv ./flagsmith-rbac/rbac $(SITE_PACKAGES_DIR) + rm -rf ./flagsmith-saml ./flagsmith-rbac + .PHONY: install install: install-pip install-poetry install-packages @@ -46,23 +56,24 @@ lint: lint-black lint-isort lint-flake8 .PHONY: docker-up docker-up: - docker-compose up --force-recreate --remove-orphans -d - docker-compose ps + docker compose up --force-recreate --remove-orphans -d + docker compose ps .PHONY: docker-down docker-down: - docker-compose stop + docker compose stop .PHONY: docker-logs docker-logs: - docker-compose logs --follow + docker compose logs --follow .PHONY: docker-build docker-build: @docker build \ --tag=$(DOCKER_TAG) \ - --file=Dockerfile \ - . + --file=../Dockerfile \ + --target=oss-api \ + ../ .PHONY: test test: diff --git a/api/app/utils.py b/api/app/utils.py index a843cc6ce946..0b3fa4fa72f2 100644 --- a/api/app/utils.py +++ b/api/app/utils.py @@ -1,5 +1,7 @@ import json import pathlib +from functools import lru_cache +from typing import TypedDict import shortuuid @@ -7,20 +9,28 @@ VERSIONS_INFO_FILE_LOCATION = ".versions.json" -def create_hash(): +class VersionInfo(TypedDict): + ci_commit_sha: str + image_tag: str + is_enterprise: bool + is_saas: bool + + +def create_hash() -> str: """Helper function to create a short hash""" return shortuuid.uuid() -def is_enterprise(): +def is_enterprise() -> bool: return pathlib.Path("./ENTERPRISE_VERSION").exists() -def is_saas(): +def is_saas() -> bool: return pathlib.Path("./SAAS_DEPLOYMENT").exists() -def get_version_info() -> dict: +@lru_cache +def get_version_info() -> VersionInfo: """Reads the version info baked into src folder of the docker container""" version_json = {} image_tag = UNKNOWN diff --git a/api/tests/unit/app/test_unit_app_utils.py b/api/tests/unit/app/test_unit_app_utils.py index 6fd992e91680..2e5ac6556be7 100644 --- a/api/tests/unit/app/test_unit_app_utils.py +++ b/api/tests/unit/app/test_unit_app_utils.py @@ -1,12 +1,20 @@ import json import pathlib +from typing import Generator from unittest import mock +import pytest from pytest_mock import MockerFixture from app.utils import get_version_info +@pytest.fixture(autouse=True) +def clear_get_version_info_cache() -> Generator[None, None, None]: + yield + get_version_info.cache_clear() + + def test_get_version_info(mocker: MockerFixture) -> None: # Given mocked_pathlib = mocker.patch("app.utils.pathlib") diff --git a/docker-compose.uffizzi.yml b/docker-compose.uffizzi.yml index b6c10d845b65..94666293c1bd 100644 --- a/docker-compose.uffizzi.yml +++ b/docker-compose.uffizzi.yml @@ -79,7 +79,8 @@ services: # https://docs.flagsmith.com/advanced-use/task-processor # flagsmith_processor: # build: -# dockerfile: api/Dockerfile +# dockerfile: Dockerfile +# target: oss-api # context: . # environment: # DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith diff --git a/docs/docs/deployment/configuration/task-processor.md b/docs/docs/deployment/configuration/task-processor.md index 43b75392daae..9da90587812c 100644 --- a/docs/docs/deployment/configuration/task-processor.md +++ b/docs/docs/deployment/configuration/task-processor.md @@ -12,38 +12,36 @@ set the `TASK_RUN_METHOD` to `TASK_PROCESSOR` in the flagsmith container running A basic docker-compose setup might look like: ```yaml - postgres: - image: postgres:15.5-alpine - environment: - POSTGRES_PASSWORD: password - POSTGRES_DB: flagsmith - container_name: flagsmith_postgres - - flagsmith: - image: flagsmith/flagsmith-api - environment: - DJANGO_ALLOWED_HOSTS: '*' - DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith - ENV: prod - TASK_RUN_METHOD: TASK_PROCESSOR - ports: - - '8000:8000' - depends_on: - - postgres - links: - - postgres - - flagsmith_processor: - build: - dockerfile: api/Dockerfile - context: . - environment: - DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith - command: - - run-task-processor - depends_on: - - flagsmith - - postgres +postgres: + image: postgres:15.5-alpine + environment: + POSTGRES_PASSWORD: password + POSTGRES_DB: flagsmith + container_name: flagsmith_postgres + +flagsmith: + image: flagsmith/flagsmith-api:latest + environment: + DJANGO_ALLOWED_HOSTS: '*' + DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith + ENV: prod + TASK_RUN_METHOD: TASK_PROCESSOR + ports: + - '8000:8000' + depends_on: + - postgres + links: + - postgres + +flagsmith_processor: + image: flagsmith/flagsmith-api:latest + environment: + DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith + command: + - run-task-processor + depends_on: + - flagsmith + - postgres ``` ## Configuring the Processor diff --git a/docs/docs/deployment/hosting/locally-frontend.md b/docs/docs/deployment/hosting/locally-frontend.md index 1b0f2099953e..9b023a0e0476 100644 --- a/docs/docs/deployment/hosting/locally-frontend.md +++ b/docs/docs/deployment/hosting/locally-frontend.md @@ -92,8 +92,6 @@ Current variables used between 'frontend/environment.js' and 'frontend/common/pr - `MIXPANEL_API_KEY`: Mixpanel analytics key to use for behaviour tracking. - `SENTRY_API_KEY`: Sentry key for error reporting. - `ALBACROSS_CLIENT_ID`: Albacross client ID key for behaviour tracking. -- `STATIC_ASSET_CDN_URL`: Used for replacing local static paths with a cdn, .e.g https://cdn.flagsmith.com. Defaults to - `/`, i.e. no CDN. - `BASE_URL`: Used for specifying a base url path that's ignored during routing if serving from a subdirectory. - `USE_SECURE_COOKIES`: Enable / disable the use of secure cookies. If deploying the FE in a private network without a domain / SSL cert, disable secure cookies to ensure that session token is persisted. Default: true. diff --git a/frontend/Dockerfile b/frontend/Dockerfile deleted file mode 100644 index 5cce021d51e4..000000000000 --- a/frontend/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Build Assets -FROM node:16 AS build - -RUN mkdir /srv/bt && chown node:node /srv/bt - -USER node - -WORKDIR /srv/bt - -COPY --chown=node:node frontend/package.json frontend/package-lock.json frontend/.npmrc frontend/.nvmrc ./ -COPY --chown=node:node frontend/bin/ ./bin/ -COPY --chown=node:node frontend/env/ ./env/ - -# since ENV is only used for the purposes of copying the correct -# project_${env}.js file to common/project.js, this is a build arg -# which subsequently gets set as an environment variable. This is -# done to avoid confusion since it is not a required run time var. -ARG ENV=selfhosted -RUN ENV=${ENV} npm ci --quiet --production - -COPY --chown=node:node frontend . -COPY .release-please-manifest.json . -RUN npm run bundle - - -# Set up runtime container -FROM node:16-slim AS production -USER node - -WORKDIR /srv/bt -COPY --from=build --chown=node:node /srv/bt/ . - -ENV NODE_ENV=production - -EXPOSE 8080 -CMD ["node", "./api/index.js"] diff --git a/frontend/Dockerfile.e2e b/frontend/Dockerfile.e2e index b67655ed8232..b0e536225318 100644 --- a/frontend/Dockerfile.e2e +++ b/frontend/Dockerfile.e2e @@ -14,4 +14,7 @@ COPY frontend . COPY .release-please-manifest.json ./.versions.json RUN npm run env +ARG CI_COMMIT_SHA=dev +RUN echo ${CI_COMMIT_SHA} > ./CI_COMMIT_SHA + CMD ["bash", "-l"] \ No newline at end of file diff --git a/frontend/Dockerfile.ubi b/frontend/Dockerfile.ubi deleted file mode 100644 index 9015d857910f..000000000000 --- a/frontend/Dockerfile.ubi +++ /dev/null @@ -1,34 +0,0 @@ -# Build Assets -FROM registry.redhat.io/ubi8/nodejs-12 AS build - -COPY . . -RUN pwd -RUN npm ci --production -ENV ENV=prod -RUN npm run bundle - - -# Set up runtime container -FROM registry.redhat.io/ubi8/nodejs-12 as application - -MAINTAINER Ben Rometsch - -LABEL name="flagsmith-api" \ - vendor="Flagsmith" \ - maintainer="support@flagsmith.com" \ - version="0.0.1" \ - summary="Feature Flags and Remote Config API" \ - description="Feature Flags and Remote Config API" -COPY LICENSE /licenses/LICENSE - -USER root -RUN yum -y update-minimal --security --sec-severity=Important --sec-severity=Critical -USER 1001 - -COPY --from=build /opt/app-root/src . - -ENV ENV=prod -ENV NODE_ENV=production - -EXPOSE 8080 -CMD ["node", "./api/index.js"] diff --git a/frontend/docker-compose-e2e-tests.yml b/frontend/docker-compose-e2e-tests.yml index 4755ea1c8857..0a620713ec6f 100644 --- a/frontend/docker-compose-e2e-tests.yml +++ b/frontend/docker-compose-e2e-tests.yml @@ -16,7 +16,7 @@ services: image: ${API_IMAGE:-ghcr.io/flagsmith/flagsmith-api:dev} build: context: ../ - dockerfile: Dockerfile + target: oss-api environment: E2E_TEST_AUTH_TOKEN: some-token ENABLE_FE_E2E: 'True'