From c36206243c78d2aec80af665eabdb00d9132bf47 Mon Sep 17 00:00:00 2001 From: Ara Ghukasyan <38226926+araghukas@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:27:23 -0500 Subject: [PATCH] Require no variables to covalent deploy gcpbatch (#32) * use `templatefile` for darwin compatibility * fall back to vars if google_client_config is null * stop requiring vars key_path and prefix * stop requiring var access_token * remove var context * clean up, use locals, docker push working * fix paths for normal vs editable install * move requirements and exec file to build args * create docker folder in non-editable installs * update changelog * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * copy files directly, fix build context * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * revert setup.py * move docker image files * include docker image files * modify dockerfile for new paths * remove normal vs. editable distinction * update exec test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update changelog * move iam stuff into own file * update config output --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 + MANIFEST.in | 1 + .../assets/docker/Dockerfile | 9 +- .../{ => assets/docker}/exec.py | 0 .../assets/docker/requirements-image.txt | 23 +++ .../assets/infra/gcpbatch.conf.tftpl | 5 +- covalent_gcpbatch_plugin/assets/infra/iam.tf | 58 ++++++++ covalent_gcpbatch_plugin/assets/infra/main.tf | 135 +++++++----------- .../assets/infra/outputs.tf | 8 +- .../assets/infra/variables.tf | 28 ++-- tests/gcpbatch_exec_test.py | 35 ++--- 11 files changed, 182 insertions(+), 124 deletions(-) rename Dockerfile => covalent_gcpbatch_plugin/assets/docker/Dockerfile (88%) rename covalent_gcpbatch_plugin/{ => assets/docker}/exec.py (100%) create mode 100644 covalent_gcpbatch_plugin/assets/docker/requirements-image.txt create mode 100644 covalent_gcpbatch_plugin/assets/infra/iam.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index d560e39..1ed28f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Changed + +- terraform scripts to require no variables + ## [0.13.0] - 2023-11-27 ### Authors diff --git a/MANIFEST.in b/MANIFEST.in index 0ce1ecd..e9e5f0d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include VERSION include requirements.txt include covalent_gcpbatch_plugin/assets/infra/* +include covalent_gcpbatch_plugin/assets/docker/* diff --git a/Dockerfile b/covalent_gcpbatch_plugin/assets/docker/Dockerfile similarity index 88% rename from Dockerfile rename to covalent_gcpbatch_plugin/assets/docker/Dockerfile index 7717342..07efd7e 100644 --- a/Dockerfile +++ b/covalent_gcpbatch_plugin/assets/docker/Dockerfile @@ -19,13 +19,12 @@ FROM ${COVALENT_BASE_IMAGE} # Install dependencies ARG COVALENT_TASK_ROOT=/usr/src + ARG COVALENT_PACKAGE_VERSION ARG PRE_RELEASE -COPY requirements.txt requirements.txt -RUN apt-get update && \ - pip install -r requirements.txt - +COPY requirements-image.txt requirements.txt +RUN apt-get update && pip install -r requirements.txt RUN if [ -z "$PRE_RELEASE" ]; then \ pip install "$COVALENT_PACKAGE_VERSION"; else \ @@ -33,7 +32,7 @@ RUN if [ -z "$PRE_RELEASE" ]; then \ fi -COPY covalent_gcpbatch_plugin/exec.py ${COVALENT_TASK_ROOT} +COPY exec.py ${COVALENT_TASK_ROOT} WORKDIR ${COVALENT_TASK_ROOT} ENV PYTHONPATH ${COVALENT_TASK_ROOT}:${PYTHONPATH} diff --git a/covalent_gcpbatch_plugin/exec.py b/covalent_gcpbatch_plugin/assets/docker/exec.py similarity index 100% rename from covalent_gcpbatch_plugin/exec.py rename to covalent_gcpbatch_plugin/assets/docker/exec.py diff --git a/covalent_gcpbatch_plugin/assets/docker/requirements-image.txt b/covalent_gcpbatch_plugin/assets/docker/requirements-image.txt new file mode 100644 index 0000000..a2edcf1 --- /dev/null +++ b/covalent_gcpbatch_plugin/assets/docker/requirements-image.txt @@ -0,0 +1,23 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Mirror plugin dependencies. + +covalent>=0.218.0,<1 +google-cloud-batch==0.9.0 +google-cloud-storage==2.7.0 + +# This file is copied into the docker image during `covalent deploy up` diff --git a/covalent_gcpbatch_plugin/assets/infra/gcpbatch.conf.tftpl b/covalent_gcpbatch_plugin/assets/infra/gcpbatch.conf.tftpl index 11662d8..baa5f95 100644 --- a/covalent_gcpbatch_plugin/assets/infra/gcpbatch.conf.tftpl +++ b/covalent_gcpbatch_plugin/assets/infra/gcpbatch.conf.tftpl @@ -1,4 +1,7 @@ [gcpbatch] +region = ${region} +key_path = ${key_path} project_id = ${project_id} +bucket_name = ${bucket_name} +container_image_uri = ${container_image_uri} covalent_package_version = ${covalent_package_version} -key_path = ${key_path} diff --git a/covalent_gcpbatch_plugin/assets/infra/iam.tf b/covalent_gcpbatch_plugin/assets/infra/iam.tf new file mode 100644 index 0000000..15abc08 --- /dev/null +++ b/covalent_gcpbatch_plugin/assets/infra/iam.tf @@ -0,0 +1,58 @@ +# Copyright 2023 Agnostiq Inc. +# +# This file is part of Covalent. +# +# Licensed under the Apache License 2.0 (the "License"). A copy of the +# License may be obtained with this software package or at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Use of this file is prohibited except in compliance with the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_service_account" "covalent" { + account_id = join("-", ["covalent", "sa", local.prefix]) + display_name = "CovalentBatchExecutorServiceAccount" + description = "Service account created by Covalent deployment" + project = local.project_id +} + +resource "google_project_iam_member" "agent_reporter" { + project = local.project_id + role = "roles/batch.agentReporter" + member = google_service_account.covalent.member +} + +resource "google_project_iam_member" "log_writer" { + project = local.project_id + role = "roles/logging.logWriter" + member = google_service_account.covalent.member +} + +resource "google_project_iam_member" "log_viewer" { + project = local.project_id + role = "roles/logging.viewer" + member = google_service_account.covalent.member +} + +resource "google_project_iam_member" "registry_writer" { + project = local.project_id + role = "roles/artifactregistry.writer" + member = google_service_account.covalent.member +} + +resource "google_project_iam_member" "storage_object_creator" { + project = local.project_id + role = "roles/storage.objectCreator" + member = google_service_account.covalent.member +} + +resource "google_project_iam_member" "storage_object_reader" { + project = local.project_id + role = "roles/storage.objectViewer" + member = google_service_account.covalent.member +} diff --git a/covalent_gcpbatch_plugin/assets/infra/main.tf b/covalent_gcpbatch_plugin/assets/infra/main.tf index e7c673a..4484753 100644 --- a/covalent_gcpbatch_plugin/assets/infra/main.tf +++ b/covalent_gcpbatch_plugin/assets/infra/main.tf @@ -23,122 +23,97 @@ terraform { } } +resource "random_string" "default_prefix" { + length = 9 + upper = false + special = false +} + +data "google_client_config" "current" {} + + +locals { + # Try to get region from current config, otherwise use vars. + region = coalesce(data.google_client_config.current.region, var.region) + project_id = coalesce(data.google_client_config.current.project, var.project_id) + + # Use random prefix if var not set. + prefix = var.prefix != "" ? var.prefix : random_string.default_prefix.result + + # Repository and iamge configuration. + repository_base_url = join("-", [local.region, "docker.pkg.dev"]) + repository_id = "covalent-executor-${local.prefix}" + + executor_image_name = join("/", [ + local.repository_base_url, + local.project_id, + local.repository_id, + "covalent-gcpbatch-executor" + ]) + + # Use default key path if var not set. + key_path_default = "${pathexpand("~")}/.config/gcloud/application_default_credentials.json" + key_path = var.key_path != "" ? var.key_path : local.key_path_default +} + provider "google" { project = var.project_id region = "us-east1" - credentials = file(var.key_path) + credentials = local.key_path } provider "docker" { host = "unix:///var/run/docker.sock" registry_auth { - address = "https://${data.google_client_config.current.region}-docker.pkg.dev" - username = "oauth2accesstoken" - password = var.access_token + address = "https://${local.region}-docker.pkg.dev" + config_file = pathexpand("~/.docker/config.json") } } -data "google_client_config" "current" {} - -locals { - executor_image_tag = join("/", [join("-", [data.google_client_config.current.region, "docker.pkg.dev"]), var.project_id, "covalent", "covalent-gcpbatch-executor"]) -} - -resource "random_string" "sasuffix" { - length = 16 - lower = false - special = false -} - -# Create the docker artifact registry resource "google_artifact_registry_repository" "covalent" { - location = data.google_client_config.current.region - repository_id = "covalent" + location = local.region + repository_id = local.repository_id description = "Covalent Batch executor base images" format = "DOCKER" } resource "docker_image" "base_executor" { - name = local.executor_image_tag + name = local.executor_image_name + build { - context = var.context + context = "../docker" + platform = "linux/amd64" + build_args = { - "PRE_RELEASE" : var.prerelease "COVALENT_PACKAGE_VERSION" : var.covalent_package_version + "PRE_RELEASE" : var.prerelease } label = { author = "Agnostiq Inc" } - platform = "linux/amd64" } } resource "docker_registry_image" "base_executor" { name = docker_image.base_executor.name - keep_remotely = true + keep_remotely = false } -# Create a storage bucket resource "google_storage_bucket" "covalent" { - name = join("-", [var.prefix, "covalent", "storage", "bucket"]) - location = data.google_client_config.current.region + name = join("-", ["covalent", "storage", local.prefix]) + location = local.region force_destroy = true } -# Create custom service account for running the batch job -resource "google_service_account" "covalent" { - account_id = join("", [var.prefix, "covalent", "saaccount"]) - display_name = "CovalentBatchExecutorServiceAccount" -} - -resource "google_project_iam_member" "agent_reporter" { - project = var.project_id - role = "roles/batch.agentReporter" - member = google_service_account.covalent.member -} - -resource "google_project_iam_member" "log_writer" { - project = var.project_id - role = "roles/logging.logWriter" - member = google_service_account.covalent.member -} - -resource "google_project_iam_member" "log_viewer" { - project = var.project_id - role = "roles/logging.viewer" - member = google_service_account.covalent.member -} - -resource "google_project_iam_member" "registry_writer" { - project = var.project_id - role = "roles/artifactregistry.writer" - member = google_service_account.covalent.member -} - -resource "google_project_iam_member" "storage_object_creator" { - project = var.project_id - role = "roles/storage.objectCreator" - member = google_service_account.covalent.member -} - -resource "google_project_iam_member" "storage_object_reader" { - project = var.project_id - role = "roles/storage.objectViewer" - member = google_service_account.covalent.member -} - -data "template_file" "executor_config" { - template = file("${path.module}/gcpbatch.conf.tftpl") - - vars = { - project_id = var.project_id - covalent_package_version = var.covalent_package_version - key_path = var.key_path - } -} - resource "local_file" "executor_config" { - content = data.template_file.executor_config.rendered filename = "${path.module}/gcpbatch.conf" + content = templatefile("${path.module}/gcpbatch.conf.tftpl", { + region = local.region + key_path = local.key_path + project_id = local.project_id + bucket_name = google_storage_bucket.covalent.name + container_image_uri = docker_registry_image.base_executor.name + covalent_package_version = var.covalent_package_version + }) } diff --git a/covalent_gcpbatch_plugin/assets/infra/outputs.tf b/covalent_gcpbatch_plugin/assets/infra/outputs.tf index 0c65b50..ee3bb17 100644 --- a/covalent_gcpbatch_plugin/assets/infra/outputs.tf +++ b/covalent_gcpbatch_plugin/assets/infra/outputs.tf @@ -19,7 +19,7 @@ output service_account_email { } output container_image_uri { - value = local.executor_image_tag + value = local.executor_image_name } output storage_bucket_name { @@ -29,10 +29,10 @@ output storage_bucket_name { output GCPBatchExecutor { value = <