diff --git a/.gitattributes b/.gitattributes index 1edbed7..5a4b20c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ .conda_lock_files/** linguist-generated=true * text=auto *.{py,yaml,yml,sh} text eol=lf +**/*.tfstate linguist-generated=true diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 223b911..20f509a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,19 +1,66 @@ name: Deploy permissions: write-all on: - pull_request: + push: branches: [main] release: types: [published] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + defaults: run: shell: bash -el {0} +env: + TF_VAR_GITHUB_TOKEN: ${{ secrets.MINING_GITHUB_TOKEN }} + TF_VAR_FIREBASE_ADMIN_SDK: ${{ github.event_name == 'release' && secrets.PROD_FIREBASE_ADMIN_SDK || secrets.STAGING_FIREBASE_ADMIN_SDK}} + jobs: - dummy_pr_test: - if: github.event_name == 'pull_request' + deploy: + name: Deploy - ${{ matrix.DEPLOYMENT_ENV }} runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + matrix: + DEPLOYMENT_ENV: + ["${{ github.event_name == 'release' && 'prod' || 'staging' }}"] + defaults: + run: + working-directory: ./terraform/${{ matrix.DEPLOYMENT_ENV }} steps: - - name: Dummy PR Test - run: echo "Test" + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Configure gcloud CLI + uses: google-github-actions/setup-gcloud@63496e652100112a8db8a71668b77c67aa1ab071 + with: + version: "416.0.0" + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - name: Test gcloud cli + run: gcloud auth list --filter=status:ACTIVE --format="value(account)" + - name: Terraform setup + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.5.3 + - name: Store credentials file + run: | + mkdir -p ./../.secrets/ + echo -n "$KEYSTORE" > ./../.secrets/la-famiglia-parma-ai.json + env: + KEYSTORE: ${{ secrets.GCP_SA_KEY }} + - name: Activate service account + run: gcloud auth activate-service-account --key-file=./../.secrets/la-famiglia-parma-ai.json + - name: Authenticate with GCR + run: gcloud auth configure-docker europe-west1-docker.pkg.dev + - name: Terraform init + run: terraform init + - name: Terraform plan + run: terraform plan + - name: Terraform apply + run: terraform apply -auto-approve diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index c8b3cce..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Deploy -on: - release: - types: [published] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash -el {0} - -jobs: - prod-deployment: - name: Publish release - runs-on: ubuntu-latest - steps: - - name: dummy - run: echo "TODO" diff --git a/.gitignore b/.gitignore index 7feb95b..4910fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -573,4 +573,7 @@ $RECYCLE.BIN/ # End of https://www.toptal.com/developers/gitignore/api/vim,node,linux,macos,windows,pycharm+all,webstorm+all,visualstudiocode,direnv,python # [CUSTOMIZATIONS] -.data +.secrets/ +.image.name +**/.terraform/ +**/*.tfstate.backup diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e171243..69d4e3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,3 +43,8 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer + - repo: https://github.com/bridgecrewio/checkov.git + rev: 3.1.15 + hooks: + - id: checkov + files: ^terraform/.*\.yml$ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..60a8907 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM --platform=linux/amd64 mambaorg/micromamba:1.5.3 + +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml + +RUN micromamba install -y -n base -f /tmp/environment.yml && \ + micromamba clean --all --yes + +WORKDIR /app + +COPY --chown=$MAMBA_USER:$MAMBA_USER parma_mining /app/parma_mining + +ENV GITHUB_TOKEN=$GITHUB_TOKEN +ENV FIREBASE_ADMIN_SDK=$FIREBASE_ADMIN_SDK + +EXPOSE 8080 + +ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] +CMD ["uvicorn", "parma_mining.github.api:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/terraform/module/main.tf b/terraform/module/main.tf new file mode 100644 index 0000000..8105b73 --- /dev/null +++ b/terraform/module/main.tf @@ -0,0 +1,19 @@ +terraform { + required_version = "1.5.3" + required_providers { + google = { + source = "hashicorp/google" + version = "5.6" + } + docker = { + source = "kreuzwerker/docker" + version = "~> 3.0.1" + } + } +} + +provider "google" { + credentials = file("../.secrets/la-famiglia-parma-ai.json") + project = var.project + region = var.region +} diff --git a/terraform/module/service.tf b/terraform/module/service.tf new file mode 100644 index 0000000..a3b4b23 --- /dev/null +++ b/terraform/module/service.tf @@ -0,0 +1,73 @@ + +/* ---------------------------------- Service Image --------------------------------- */ + +# Note: Generally it is NOT best practise to build images in Terraform. We are still +# doing it here for simplicity. In industry, you should think twice before doing this. +resource "null_resource" "docker_build" { + + provisioner "local-exec" { + working_dir = path.module + command = "IMG=${var.region}-docker.pkg.dev/${var.project}/parma-registry/parma-mining-github:${var.env}-$(git rev-parse --short HEAD) && docker build -t $IMG ./../../ && docker push $IMG && echo $IMG > .image.name" + } + + triggers = { + always_run = timestamp() + } +} + +# get output from docker_build +data "local_file" "image_name" { + filename = "${path.module}/.image.name" + depends_on = [null_resource.docker_build] +} + + +/* ------------------------------------ Cloud Run ----------------------------------- */ + +resource "google_cloud_run_service" "parma_mining_github_cloud_run" { + name = "parma-mining-github-${var.env}" + location = var.region + + template { + spec { + containers { + image = data.local_file.image_name.content + ports { + container_port = 8080 + } + env { + name = "GITHUB_TOKEN" + value = var.GITHUB_TOKEN + } + env { + name = "FIREBASE_ADMIN_SDK" + value = var.FIREBASE_ADMIN_SDK + } + } + } + } + + traffic { + percent = 100 + latest_revision = true + } +} + +/* --------------------------------------- IAM -------------------------------------- */ + +// Define a policy that allows any user to invoke the Cloud Run service. +data "google_iam_policy" "noauth" { + binding { + role = "roles/run.invoker" + members = ["allUsers"] + } +} + +// Apply the policy to the Cloud Run service. +resource "google_cloud_run_service_iam_policy" "noauth" { + location = google_cloud_run_service.parma_mining_github_cloud_run.location + project = google_cloud_run_service.parma_mining_github_cloud_run.project + service = google_cloud_run_service.parma_mining_github_cloud_run.name + + policy_data = data.google_iam_policy.noauth.policy_data +} diff --git a/terraform/module/variables.tf b/terraform/module/variables.tf new file mode 100644 index 0000000..c9a9c33 --- /dev/null +++ b/terraform/module/variables.tf @@ -0,0 +1,26 @@ +variable "env" { + description = "staging or prod environment" + type = string +} + +variable "project" { + description = "Google Cloud Project ID" + type = string +} + +variable "region" { + description = "Google Cloud Region" + type = string +} + +variable "GITHUB_TOKEN" { + description = "value" + type = string + sensitive = true +} + +variable "FIREBASE_ADMIN_SDK" { + description = "value" + type = string + sensitive = true +} diff --git a/terraform/prod/main.tf b/terraform/prod/main.tf new file mode 100644 index 0000000..ce0a716 --- /dev/null +++ b/terraform/prod/main.tf @@ -0,0 +1,38 @@ +terraform { + required_version = "1.5.3" + required_providers { + google = { + source = "hashicorp/google" + version = "5.6" + } + docker = { + source = "kreuzwerker/docker" + version = "~> 3.0.1" + } + } + backend "gcs" { + bucket = "la-famiglia-jst2324-tf-state" + prefix = "terraform/state/prod/mining/github" + credentials = "../.secrets/la-famiglia-parma-ai.json" + } +} + +locals { + project = "la-famiglia-parma-ai" + region = "europe-west1" +} + +provider "google" { + credentials = file("../.secrets/la-famiglia-parma-ai.json") + project = local.project + region = local.region +} + +module "main" { + source = "../module" + env = "prod" + project = local.project + region = local.region + GITHUB_TOKEN = var.GITHUB_TOKEN + FIREBASE_ADMIN_SDK = var.FIREBASE_ADMIN_SDK +} diff --git a/terraform/prod/variables.tf b/terraform/prod/variables.tf new file mode 100644 index 0000000..3bcee04 --- /dev/null +++ b/terraform/prod/variables.tf @@ -0,0 +1,11 @@ +variable "GITHUB_TOKEN" { + description = "value" + type = string + sensitive = true +} + +variable "FIREBASE_ADMIN_SDK" { + description = "value" + type = string + sensitive = true +} diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf new file mode 100644 index 0000000..c234a49 --- /dev/null +++ b/terraform/staging/main.tf @@ -0,0 +1,38 @@ +terraform { + required_version = "1.5.3" + required_providers { + google = { + source = "hashicorp/google" + version = "5.6" + } + docker = { + source = "kreuzwerker/docker" + version = "~> 3.0.1" + } + } + backend "gcs" { + bucket = "la-famiglia-jst2324-tf-state" + prefix = "terraform/state/staging/mining/github" + credentials = "../.secrets/la-famiglia-parma-ai.json" + } +} + +locals { + project = "la-famiglia-parma-ai" + region = "europe-west1" +} + +provider "google" { + credentials = file("../.secrets/la-famiglia-parma-ai.json") + project = local.project + region = local.region +} + +module "main" { + source = "../module" + env = "staging" + project = local.project + region = local.region + GITHUB_TOKEN = var.GITHUB_TOKEN + FIREBASE_ADMIN_SDK = var.FIREBASE_ADMIN_SDK +} diff --git a/terraform/staging/variables.tf b/terraform/staging/variables.tf new file mode 100644 index 0000000..3bcee04 --- /dev/null +++ b/terraform/staging/variables.tf @@ -0,0 +1,11 @@ +variable "GITHUB_TOKEN" { + description = "value" + type = string + sensitive = true +} + +variable "FIREBASE_ADMIN_SDK" { + description = "value" + type = string + sensitive = true +} diff --git a/terraform/state/main.tf b/terraform/state/main.tf new file mode 100644 index 0000000..e69de29