Merge pull request #28 from patoarvizu/add-etcdv3-backend

Add etcdv3 backend

.circleci/config.yml

@@ -100,18 +100,20 @@ jobs:
         command: |
           export KUBECONFIG=~/.k3d/k3s-default-config
           export KUBE_CONFIG_PATH=~/.k3d/k3s-default-config
-          k3d cluster create --image rancher/k3s:v1.21.8-k3s1 --port 8500:30058@server[0] --port 5432:32345@server[0] --port 8082:32082@server[0] # --k3s-server-arg "--kube-apiserver-arg=feature-gates=ServerSideApply=false"
+          k3d cluster create --image rancher/k3s:v1.21.8-k3s1 --port 8500:30058@server[0] --port 5432:32345@server[0] --port 8082:32082@server[0] --port 2379:30732@server[0] # --k3s-server-arg "--kube-apiserver-arg=feature-gates=ServerSideApply=false"
           k3d image import patoarvizu/amphibian:latest
           cd test/consul && helmfile sync
           cd ../consul-service && helmfile sync
           export CONSUL_HTTP_TOKEN=$(kubectl -n consul get secret consul-bootstrap-acl-token -o json | jq -r '.data.token' | base64 -d)
           cd ../postgres && helmfile sync
           cd ../artifactory && helmfile sync
+          cd ../etcdv3 && helmfile sync
           sleep 30
           cd ../consul-state && terraform init && terraform apply -auto-approve
           cd ../kubernetes-state && terraform init && terraform apply -auto-approve
           cd ../postgres-state && terraform init && terraform apply -auto-approve
           cd ../artifactory-state && terraform init && terraform apply -auto-approve
+          cd ../etcdv3-state && terraform init && terraform apply -auto-approve
           export AMP_CONSUL_TOKEN=${CONSUL_HTTP_TOKEN}
           cd ../secrets && helmfile sync
           cd ../amphibian && helmfile sync

README.md

@@ -15,6 +15,7 @@
     - [Kubernetes](#kubernetes)
     - [Postgres](#postgres)
     - [Artifactory](#artifactory)
+    - [Etcd V3](#etcd-v3)
   - [Target](#target)
     - [Values](#values)
 - [For security nerds](#for-security-nerds)
@@ -158,6 +159,17 @@ The following fields can be alternatively be set as environment variables (as do
 - `password` (`ARTIFACTORY_PASSWORD`)
 - `url` (`ARTIFACTORY_URL`)
 
+#### Etcd V3
+
+- [Documentation](https://www.terraform.io/language/settings/backends/etcdv3)
+- `type: etcdv3`
+- Configuration block name: `etcdv3Config`
+
+The following fields can be alternatively be set as environment variables (as documented in the link above):
+
+- `username` (`ETCDV3_USERNAME`)
+- `password` (`ETCDV3_PASSWORD`)
+
 ### Target
 
 The `target` field represents the location and type of object where the outputs from the upstream state will be projected.

api/v1/terraformstate_types.go

@@ -110,6 +110,19 @@ type ArtifactoryConfig struct {
 	Subpath  string `json:"subpath"`
 }
 
+type EtcdV3Config struct {
+	// +nullable
+	Endpoints       []string `json:"endpoints"`
+	Username        string   `json:"username,omitempty"`
+	Password        string   `json:"password,omitempty"`
+	Prefix          string   `json:"prefix,omitempty"`
+	Lock            bool     `json:"lock,omitempty"`
+	CACertPath      string   `json:"caCertPath,omitempty"`
+	CertPath        string   `json:"certPath,omitempty"`
+	KeyPath         string   `json:"keyPath,omitempty"`
+	MaxRequestBytes string   `json:"maxRequestBytes,omitempty"`
+}
+
 type TerraformStateSpec struct {
 	Type              string            `json:"type"`
 	RemoteConfig      RemoteConfig      `json:"remoteConfig,omitempty"`
@@ -119,6 +132,7 @@ type TerraformStateSpec struct {
 	GCSConfig         GCSConfig         `json:"gcsConfig,omitempty"`
 	PostgresConfig    PostgresConfig    `json:"postgresConfig,omitempty"`
 	ArtifactoryConfig ArtifactoryConfig `json:"artifactoryConfig,omitempty"`
+	EtcdV3Config      EtcdV3Config      `json:"etcdv3Config,omitempty"`
 	Target            Target            `json:"target"`
 }
 

config/crd/bases/terraform.patoarvizu.dev_terraformstates.yaml

@@ -75,6 +75,32 @@ spec:
               required:
               - path
               type: object
+            etcdv3Config:
+              properties:
+                caCertPath:
+                  type: string
+                certPath:
+                  type: string
+                endpoints:
+                  items:
+                    type: string
+                  nullable: true
+                  type: array
+                keyPath:
+                  type: string
+                lock:
+                  type: boolean
+                maxRequestBytes:
+                  type: string
+                password:
+                  type: string
+                prefix:
+                  type: string
+                username:
+                  type: string
+              required:
+              - endpoints
+              type: object
             gcsConfig:
               properties:
                 accessToken:

controllers/terraformstate_controller.go

@@ -111,6 +111,8 @@ func (r *TerraformStateReconciler) Reconcile(req ctrl.Request) (ctrl.Result, err
 		dataBody.SetAttributeValue("config", createPostgresBackendBody(state.Spec.PostgresConfig))
 	case "artifactory":
 		dataBody.SetAttributeValue("config", createArtifactoryBackendBody(state.Spec.ArtifactoryConfig))
+	case "etcdv3":
+		dataBody.SetAttributeValue("config", createEtcdV3BackendBody(state.Spec.EtcdV3Config))
 	}
 	dataFile, err := os.Create(fmt.Sprintf("%s/data.tf", stateDir))
 	if err != nil {
@@ -454,6 +456,34 @@ func createArtifactoryBackendBody(config terraformv1.ArtifactoryConfig) cty.Valu
 	return cty.ObjectVal(c)
 }
 
+func createEtcdV3BackendBody(config terraformv1.EtcdV3Config) cty.Value {
+	c := make(map[string]cty.Value)
+	c["endpoints"] = cty.ListVal(createValueList(config.Endpoints))
+	if len(config.Username) > 0 {
+		c["username"] = cty.StringVal(config.Username)
+	}
+	if len(config.Password) > 0 {
+		c["password"] = cty.StringVal(config.Password)
+	}
+	if len(config.Prefix) > 0 {
+		c["prefix"] = cty.StringVal(config.Prefix)
+	}
+	if len(config.Prefix) > 0 {
+		c["ca_cert_path"] = cty.StringVal(config.CACertPath)
+	}
+	if len(config.Prefix) > 0 {
+		c["cert_path"] = cty.StringVal(config.CertPath)
+	}
+	if len(config.Prefix) > 0 {
+		c["key_path"] = cty.StringVal(config.KeyPath)
+	}
+	if len(config.Prefix) > 0 {
+		c["max_request_bytes"] = cty.StringVal(config.MaxRequestBytes)
+	}
+	c["lock"] = cty.BoolVal(config.Lock)
+	return cty.ObjectVal(c)
+}
+
 func createValueList(l []string) []cty.Value {
 	valueList := []cty.Value{}
 	for _, v := range l {

helm/amphibian/templates/crds/terraformstate.yaml

@@ -75,6 +75,32 @@ spec:
               required:
               - path
               type: object
+            etcdv3Config:
+              properties:
+                caCertPath:
+                  type: string
+                certPath:
+                  type: string
+                endpoints:
+                  items:
+                    type: string
+                  nullable: true
+                  type: array
+                keyPath:
+                  type: string
+                lock:
+                  type: boolean
+                maxRequestBytes:
+                  type: string
+                password:
+                  type: string
+                prefix:
+                  type: string
+                username:
+                  type: string
+              required:
+              - endpoints
+              type: object
             gcsConfig:
               properties:
                 accessToken:

test/e2e/terraformstate_test.go

@@ -249,6 +249,36 @@ func createArtifactoryStateConfig(targetType string) (*terraformv1.TerraformStat
 	return s, nil
 }
 
+func createEtcdV3StateConfig(targetType string) (*terraformv1.TerraformState, error) {
+	s := &terraformv1.TerraformState{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "TerraformState",
+			APIVersion: "terraform.patoarvizu.dev/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-etcdv3",
+			Namespace: "default",
+		},
+		Spec: terraformv1.TerraformStateSpec{
+			Type: "etcdv3",
+			EtcdV3Config: terraformv1.EtcdV3Config{
+				Endpoints: []string{"etcdv3.etcdv3:2379"},
+				Username:  "root",
+				Password:  "root123",
+			},
+			Target: terraformv1.Target{
+				Type: targetType,
+				Name: "test-etcdv3",
+			},
+		},
+	}
+	err := k8sClient.Create(context.TODO(), s)
+	if err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
@@ -427,6 +457,28 @@ var _ = Describe("With the controller running", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})
+	When("Deploying a TerraformState object with 'etcdv3' config and target type 'configmap'", func() {
+		It("Should create the target ConfigMap", func() {
+			state, err = createEtcdV3StateConfig("configmap")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(state).ToNot(BeNil())
+			err = validateStateTargetConfigMap(state)
+			Expect(err).ToNot(HaveOccurred())
+			err = k8sClient.Delete(context.TODO(), state)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+	When("Deploying a TerraformState object with 'etcdv3' config and target type 'secret'", func() {
+		It("Should create the target Secret", func() {
+			state, err = createEtcdV3StateConfig("secret")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(state).ToNot(BeNil())
+			err = validateStateTargetSecret(state)
+			Expect(err).ToNot(HaveOccurred())
+			err = k8sClient.Delete(context.TODO(), state)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
 })
 
 var _ = AfterSuite(func() {

test/etcdv3-state/outputs.tf

@@ -0,0 +1,14 @@
+output hello {
+  value = "world"
+}
+
+output map {
+  value = {
+    a = "b"
+    x = "y"
+  }
+}
+
+output list {
+  value = ["a", "b", "c"]
+}

test/etcdv3-state/remote_state.tf

@@ -0,0 +1,7 @@
+terraform {
+  backend "etcdv3" {
+    endpoints = ["localhost:2379"]
+    username = "root"
+    password = "root123"
+  }
+}

test/etcdv3/helmfile.yaml

@@ -0,0 +1,18 @@
+repositories:
+- name: bitnami
+  url: https://charts.bitnami.com/bitnami
+
+releases:
+- name: etcdv3
+  namespace: etcdv3
+  chart: bitnami/etcd
+  version: 6.13.3
+  wait: true
+  values:
+  - ./values.yaml
+
+helmDefaults:
+  kubeContext: k3d-k3s-default
+  args:
+  - --kubeconfig
+  - {{ requiredEnv "HOME" }}/.k3d/k3s-default-config

test/etcdv3/values.yaml

@@ -0,0 +1,8 @@
+auth:
+  rbac:
+    rootPassword: root123
+
+service:
+  type: NodePort
+  nodePorts:
+    client: 30732