minor fix for gcp plugin (#31)

* minor fix for gcp plugin * minor modifications for MANIFEST.in file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
AgnostiqHQ · Nov 27, 2023 · c99852c · c99852c
1 parent e0f72e1
commit c99852c
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 46 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Fix
+
+- Added `key_path` & `access_token` to ExecutorInfraDefaults to accept arguments from **deploy** command
+
+### Added
+
+- Added tftpl file to main.tf to generate conf for GCP batch plugin
+
+### Changed
+
+- Modified the `key_path` variable to have a default value
+
 ## [0.12.0] - 2023-11-21
 
 ### Authors

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,5 +1,3 @@
 include VERSION
 include requirements.txt
-include covalent_gcpbatch_plugin/assets/infra/main.tf
-include covalent_gcpbatch_plugin/assets/infra/variables.tf
-include covalent_gcpbatch_plugin/assets/infra/outputs.tf
+include covalent_gcpbatch_plugin/assets/infra/*
diff --git a/covalent_gcpbatch_plugin/assets/infra/gcpbatch.conf.tftpl b/covalent_gcpbatch_plugin/assets/infra/gcpbatch.conf.tftpl
@@ -0,0 +1,4 @@
+[gcpbatch]
+project_id = ${project_id}
+covalent_package_version = ${covalent_package_version}
+key_path = ${key_path}
diff --git a/covalent_gcpbatch_plugin/assets/infra/main.tf b/covalent_gcpbatch_plugin/assets/infra/main.tf
@@ -17,55 +17,55 @@
 terraform {
   required_providers {
     docker = {
-      source = "kreuzwerker/docker"
+      source  = "kreuzwerker/docker"
       version = "3.0.1"
     }
   }
 }
 
-provider google {
-  project = var.project_id
-  region  = "us-east1"
+provider "google" {
+  project     = var.project_id
+  region      = "us-east1"
   credentials = file(var.key_path)
 }
 
-provider docker {
+provider "docker" {
   host = "unix:///var/run/docker.sock"
   registry_auth {
-    address = "https://${data.google_client_config.current.region}-docker.pkg.dev"
+    address  = "https://${data.google_client_config.current.region}-docker.pkg.dev"
     username = "oauth2accesstoken"
     password = var.access_token
   }
 }
 
-data google_client_config current {}
+data "google_client_config" "current" {}
 
 locals {
   executor_image_tag = join("/", [join("-", [data.google_client_config.current.region, "docker.pkg.dev"]), var.project_id, "covalent", "covalent-gcpbatch-executor"])
 }
 
-resource random_string sasuffix {
-  length = 16
-  lower = false
+resource "random_string" "sasuffix" {
+  length  = 16
+  lower   = false
   special = false
 }
 
 # Create the docker artifact registry
-resource google_artifact_registry_repository covalent {
+resource "google_artifact_registry_repository" "covalent" {
   location      = data.google_client_config.current.region
   repository_id = "covalent"
   description   = "Covalent Batch executor base images"
   format        = "DOCKER"
 }
 
 
-resource docker_image base_executor {
+resource "docker_image" "base_executor" {
   name = local.executor_image_tag
   build {
     context = var.context
     build_args = {
-      "PRE_RELEASE": var.prerelease
-      "COVALENT_PACKAGE_VERSION": var.covalent_package_version
+      "PRE_RELEASE" : var.prerelease
+      "COVALENT_PACKAGE_VERSION" : var.covalent_package_version
     }
     label = {
       author = "Agnostiq Inc"
@@ -74,56 +74,71 @@ resource docker_image base_executor {
   }
 }
 
-resource docker_registry_image base_executor {
-  name = docker_image.base_executor.name
+resource "docker_registry_image" "base_executor" {
+  name          = docker_image.base_executor.name
   keep_remotely = true
 }
 
 # Create a storage bucket
-resource google_storage_bucket covalent {
-  name            = join("-", [var.prefix, "covalent", "storage", "bucket"])
-  location        = data.google_client_config.current.region
-  force_destroy   = true
+resource "google_storage_bucket" "covalent" {
+  name          = join("-", [var.prefix, "covalent", "storage", "bucket"])
+  location      = data.google_client_config.current.region
+  force_destroy = true
 }
 
 # Create custom service account for running the batch job
-resource google_service_account covalent {
-  account_id      = join("", [var.prefix, "covalent", "saaccount"])
-  display_name    = "CovalentBatchExecutorServiceAccount"
+resource "google_service_account" "covalent" {
+  account_id   = join("", [var.prefix, "covalent", "saaccount"])
+  display_name = "CovalentBatchExecutorServiceAccount"
 }
 
-resource google_project_iam_member agent_reporter {
+resource "google_project_iam_member" "agent_reporter" {
   project = var.project_id
-  role = "roles/batch.agentReporter"
-  member = google_service_account.covalent.member
+  role    = "roles/batch.agentReporter"
+  member  = google_service_account.covalent.member
 }
 
-resource google_project_iam_member log_writer {
+resource "google_project_iam_member" "log_writer" {
   project = var.project_id
-  role = "roles/logging.logWriter"
-  member = google_service_account.covalent.member
+  role    = "roles/logging.logWriter"
+  member  = google_service_account.covalent.member
 }
 
-resource google_project_iam_member log_viewer {
+resource "google_project_iam_member" "log_viewer" {
   project = var.project_id
-  role = "roles/logging.viewer"
-  member = google_service_account.covalent.member
+  role    = "roles/logging.viewer"
+  member  = google_service_account.covalent.member
 }
 
-resource google_project_iam_member registry_writer {
+resource "google_project_iam_member" "registry_writer" {
   project = var.project_id
-  role = "roles/artifactregistry.writer"
-  member = google_service_account.covalent.member
+  role    = "roles/artifactregistry.writer"
+  member  = google_service_account.covalent.member
 }
 
-resource google_project_iam_member storage_object_creator {
+resource "google_project_iam_member" "storage_object_creator" {
   project = var.project_id
-  role = "roles/storage.objectCreator"
-  member = google_service_account.covalent.member
+  role    = "roles/storage.objectCreator"
+  member  = google_service_account.covalent.member
 }
 
-resource google_project_iam_member storage_object_reader {
+resource "google_project_iam_member" "storage_object_reader" {
   project = var.project_id
-  role = "roles/storage.objectViewer"
-  member = google_service_account.covalent.member
+  role    = "roles/storage.objectViewer"
+  member  = google_service_account.covalent.member
+}
+
+data "template_file" "executor_config" {
+  template = file("${path.module}/gcpbatch.conf.tftpl")
+
+  vars = {
+    project_id               = var.project_id
+    covalent_package_version = var.covalent_package_version
+    key_path                 = var.key_path
+  }
+}
+
+resource "local_file" "executor_config" {
+  content  = data.template_file.executor_config.rendered
+  filename = "${path.module}/gcpbatch.conf"
 }
diff --git a/covalent_gcpbatch_plugin/assets/infra/variables.tf b/covalent_gcpbatch_plugin/assets/infra/variables.tf
@@ -53,4 +53,5 @@ variable "prefix" {
 variable "key_path"{
   type = string
   description = "JSON file containing the credentials to connect to google provider"
+  default = ""
 }
diff --git a/covalent_gcpbatch_plugin/gcpbatch.py b/covalent_gcpbatch_plugin/gcpbatch.py
@@ -54,14 +54,16 @@ class ExecutorInfraDefaults(BaseModel):
     Executor configuration values for deploying infrastructure
     """
 
-    prefix: str
-    project_id: str
+    prefix: str = ""
+    project_id: str = "covalenttesting"
+    access_token: str = ""
     vcpus: Optional[int] = 2
     memory: Optional[float] = 512
     time_limit: Optional[int] = 300
     poll_freq: Optional[int] = 5
     retries: Optional[int] = 3
     cache_dir: Optional[str] = "/tmp/covalent"
+    key_path: str
 
 
 _EXECUTOR_PLUGIN_DEFAULTS = ExecutorPluginDefaults().dict()