Require no variables to covalent deploy gcpbatch (#32)

* use `templatefile` for darwin compatibility

* fall back to vars if google_client_config is null

* stop requiring vars key_path and prefix

* stop requiring var access_token

* remove var context

* clean up, use locals, docker push working

* fix paths for normal vs editable install

* move requirements and exec file to build args

* create docker folder in non-editable installs

* update changelog

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* copy files directly, fix build context

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* revert setup.py

* move docker image files

* include docker image files

* modify dockerfile for new paths

* remove normal vs. editable distinction

* update exec test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* update changelog

* move iam stuff into own file

* update config output

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Changed
+
+- terraform scripts to require no variables
+
 ## [0.13.0] - 2023-11-27
 
 ### Authors

MANIFEST.in

@@ -1,3 +1,4 @@
 include VERSION
 include requirements.txt
 include covalent_gcpbatch_plugin/assets/infra/*
+include covalent_gcpbatch_plugin/assets/docker/*

Dockerfile → covalent_gcpbatch_plugin/assets/docker/Dockerfile

@@ -19,21 +19,20 @@ FROM ${COVALENT_BASE_IMAGE}
 
 # Install dependencies
 ARG COVALENT_TASK_ROOT=/usr/src
+
 ARG COVALENT_PACKAGE_VERSION
 ARG PRE_RELEASE
 
-COPY requirements.txt requirements.txt
-RUN apt-get update && \
-		pip install -r requirements.txt
-
+COPY requirements-image.txt requirements.txt
+RUN apt-get update && pip install -r requirements.txt
 
 RUN if [ -z "$PRE_RELEASE" ]; then \
 		pip install "$COVALENT_PACKAGE_VERSION"; else \
 		pip install --pre "$COVALENT_PACKAGE_VERSION"; \
 	fi
 
 
-COPY covalent_gcpbatch_plugin/exec.py ${COVALENT_TASK_ROOT}
+COPY exec.py ${COVALENT_TASK_ROOT}
 
 WORKDIR ${COVALENT_TASK_ROOT}
 ENV PYTHONPATH ${COVALENT_TASK_ROOT}:${PYTHONPATH}

covalent_gcpbatch_plugin/assets/docker/requirements-image.txt

@@ -0,0 +1,23 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Mirror plugin dependencies.
+
+covalent>=0.218.0,<1
+google-cloud-batch==0.9.0
+google-cloud-storage==2.7.0
+
+# This file is copied into the docker image during `covalent deploy up`

covalent_gcpbatch_plugin/assets/infra/gcpbatch.conf.tftpl

@@ -1,4 +1,7 @@
 [gcpbatch]
+region = ${region}
+key_path = ${key_path}
 project_id = ${project_id}
+bucket_name = ${bucket_name}
+container_image_uri = ${container_image_uri}
 covalent_package_version = ${covalent_package_version}
-key_path = ${key_path}

covalent_gcpbatch_plugin/assets/infra/iam.tf

@@ -0,0 +1,58 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "google_service_account" "covalent" {
+  account_id   = join("-", ["covalent", "sa", local.prefix])
+  display_name = "CovalentBatchExecutorServiceAccount"
+  description  = "Service account created by Covalent deployment"
+  project      = local.project_id
+}
+
+resource "google_project_iam_member" "agent_reporter" {
+  project = local.project_id
+  role    = "roles/batch.agentReporter"
+  member  = google_service_account.covalent.member
+}
+
+resource "google_project_iam_member" "log_writer" {
+  project = local.project_id
+  role    = "roles/logging.logWriter"
+  member  = google_service_account.covalent.member
+}
+
+resource "google_project_iam_member" "log_viewer" {
+  project = local.project_id
+  role    = "roles/logging.viewer"
+  member  = google_service_account.covalent.member
+}
+
+resource "google_project_iam_member" "registry_writer" {
+  project = local.project_id
+  role    = "roles/artifactregistry.writer"
+  member  = google_service_account.covalent.member
+}
+
+resource "google_project_iam_member" "storage_object_creator" {
+  project = local.project_id
+  role    = "roles/storage.objectCreator"
+  member  = google_service_account.covalent.member
+}
+
+resource "google_project_iam_member" "storage_object_reader" {
+  project = local.project_id
+  role    = "roles/storage.objectViewer"
+  member  = google_service_account.covalent.member
+}

covalent_gcpbatch_plugin/assets/infra/main.tf

@@ -23,122 +23,97 @@ terraform {
   }
 }
 
+resource "random_string" "default_prefix" {
+  length  = 9
+  upper   = false
+  special = false
+}
+
+data "google_client_config" "current" {}
+
+
+locals {
+  # Try to get region from current config, otherwise use vars.
+  region     = coalesce(data.google_client_config.current.region, var.region)
+  project_id = coalesce(data.google_client_config.current.project, var.project_id)
+
+  # Use random prefix if var not set.
+  prefix = var.prefix != "" ? var.prefix : random_string.default_prefix.result
+
+  # Repository and iamge configuration.
+  repository_base_url = join("-", [local.region, "docker.pkg.dev"])
+  repository_id       = "covalent-executor-${local.prefix}"
+
+  executor_image_name = join("/", [
+    local.repository_base_url,
+    local.project_id,
+    local.repository_id,
+    "covalent-gcpbatch-executor"
+  ])
+
+  # Use default key path if var not set.
+  key_path_default = "${pathexpand("~")}/.config/gcloud/application_default_credentials.json"
+  key_path         = var.key_path != "" ? var.key_path : local.key_path_default
+}
+
 provider "google" {
   project     = var.project_id
   region      = "us-east1"
-  credentials = file(var.key_path)
+  credentials = local.key_path
 }
 
 provider "docker" {
   host = "unix:///var/run/docker.sock"
   registry_auth {
-    address  = "https://${data.google_client_config.current.region}-docker.pkg.dev"
-    username = "oauth2accesstoken"
-    password = var.access_token
+    address     = "https://${local.region}-docker.pkg.dev"
+    config_file = pathexpand("~/.docker/config.json")
   }
 }
 
-data "google_client_config" "current" {}
-
-locals {
-  executor_image_tag = join("/", [join("-", [data.google_client_config.current.region, "docker.pkg.dev"]), var.project_id, "covalent", "covalent-gcpbatch-executor"])
-}
-
-resource "random_string" "sasuffix" {
-  length  = 16
-  lower   = false
-  special = false
-}
-
-# Create the docker artifact registry
 resource "google_artifact_registry_repository" "covalent" {
-  location      = data.google_client_config.current.region
-  repository_id = "covalent"
+  location      = local.region
+  repository_id = local.repository_id
   description   = "Covalent Batch executor base images"
   format        = "DOCKER"
 }
 
 
 resource "docker_image" "base_executor" {
-  name = local.executor_image_tag
+  name = local.executor_image_name
+
   build {
-    context = var.context
+    context  = "../docker"
+    platform = "linux/amd64"
+
     build_args = {
-      "PRE_RELEASE" : var.prerelease
       "COVALENT_PACKAGE_VERSION" : var.covalent_package_version
+      "PRE_RELEASE" : var.prerelease
     }
     label = {
       author = "Agnostiq Inc"
     }
-    platform = "linux/amd64"
   }
 }
 
 resource "docker_registry_image" "base_executor" {
   name          = docker_image.base_executor.name
-  keep_remotely = true
+  keep_remotely = false
 }
 
-# Create a storage bucket
 resource "google_storage_bucket" "covalent" {
-  name          = join("-", [var.prefix, "covalent", "storage", "bucket"])
-  location      = data.google_client_config.current.region
+  name          = join("-", ["covalent", "storage", local.prefix])
+  location      = local.region
   force_destroy = true
 }
 
-# Create custom service account for running the batch job
-resource "google_service_account" "covalent" {
-  account_id   = join("", [var.prefix, "covalent", "saaccount"])
-  display_name = "CovalentBatchExecutorServiceAccount"
-}
-
-resource "google_project_iam_member" "agent_reporter" {
-  project = var.project_id
-  role    = "roles/batch.agentReporter"
-  member  = google_service_account.covalent.member
-}
-
-resource "google_project_iam_member" "log_writer" {
-  project = var.project_id
-  role    = "roles/logging.logWriter"
-  member  = google_service_account.covalent.member
-}
-
-resource "google_project_iam_member" "log_viewer" {
-  project = var.project_id
-  role    = "roles/logging.viewer"
-  member  = google_service_account.covalent.member
-}
-
-resource "google_project_iam_member" "registry_writer" {
-  project = var.project_id
-  role    = "roles/artifactregistry.writer"
-  member  = google_service_account.covalent.member
-}
-
-resource "google_project_iam_member" "storage_object_creator" {
-  project = var.project_id
-  role    = "roles/storage.objectCreator"
-  member  = google_service_account.covalent.member
-}
-
-resource "google_project_iam_member" "storage_object_reader" {
-  project = var.project_id
-  role    = "roles/storage.objectViewer"
-  member  = google_service_account.covalent.member
-}
-
-data "template_file" "executor_config" {
-  template = file("${path.module}/gcpbatch.conf.tftpl")
-
-  vars = {
-    project_id               = var.project_id
-    covalent_package_version = var.covalent_package_version
-    key_path                 = var.key_path
-  }
-}
-
 resource "local_file" "executor_config" {
-  content  = data.template_file.executor_config.rendered
   filename = "${path.module}/gcpbatch.conf"
+  content = templatefile("${path.module}/gcpbatch.conf.tftpl", {
+    region                   = local.region
+    key_path                 = local.key_path
+    project_id               = local.project_id
+    bucket_name              = google_storage_bucket.covalent.name
+    container_image_uri      = docker_registry_image.base_executor.name
+    covalent_package_version = var.covalent_package_version
+  })
 }

covalent_gcpbatch_plugin/assets/infra/outputs.tf

@@ -19,7 +19,7 @@ output service_account_email {
 }
 
 output container_image_uri {
-  value = local.executor_image_tag
+  value = local.executor_image_name
 }
 
 output storage_bucket_name {
@@ -29,10 +29,10 @@ output storage_bucket_name {
 output GCPBatchExecutor {
   value = <<EOL
   GCPBatchExecutor(
-    project_id='${data.google_client_config.current.project}',
-    region='${data.google_client_config.current.region}',
+    project_id='${local.project_id}',
+    region='${local.region}',
     bucket_name='${google_storage_bucket.covalent.name}',
-    container_image_uri='${local.executor_image_tag}',
+    container_image_uri='${local.executor_image_name}',
     service_account_email='${google_service_account.covalent.email}'
   )
 EOL

covalent_gcpbatch_plugin/assets/infra/variables.tf

@@ -20,19 +20,6 @@ variable "project_id" {
   default = "covalenttesting"
 }
 
-variable "access_token" {
-  type        = string
-  default     = ""
-  sensitive   = true
-  description = "Google cloud access token for authenticating to the artifact registry"
-}
-
-variable "context" {
-  type        = string
-  description = "Path to the build context. Defaults to the root directory up three levels"
-  default     = "../../.."
-}
-
 variable "prerelease" {
   type        = string
   description = "Specify if the latest pre-release version of Covalent is to be installed when building the docker container"
@@ -47,11 +34,18 @@ variable "covalent_package_version" {
 
 variable "prefix" {
   type    = string
-  default = "covalent"
+  default = ""
 }
 
-variable "key_path"{
-  type = string
+
+variable "key_path" {
+  type        = string
   description = "JSON file containing the credentials to connect to google provider"
-  default = ""
+  default     = ""
+}
+
+variable "region" {
+  type        = string
+  description = "Region to deploy the infrastructure to"
+  default     = "us-east1"
 }