Enabled TLS for Trident's etcd client and the etcd-copy utility

Closes #45

.gitignore

@@ -16,6 +16,8 @@ trident-installer.tar.gz
 trident-installer/*.yaml
 trident-installer/setup
 trident
+*.pem
+*.csr
 
 ###############################################################################
 # Go

CHANGELOG.md

@@ -2,7 +2,20 @@
 
 [Releases](https://github.com/NetApp/trident/releases)
 
-## Changes since v17.07.0
+## Changes since v17.10.0
+
+**Fixes:**
+- Volume deletion is an idempotent operation with the ontap-nas-economy driver (Issue [#65](https://github.com/NetApp/trident/issues/65)).
+
+**Enhancements:**
+- Added CHAP support for SolidFire (Issue [#42](https://github.com/NetApp/trident/issues/42)).
+- Added support for the etcdv3 API (Issue [#45](https://github.com/NetApp/trident/issues/45)).
+- Added the etcd-copy utility to migrate data between etcd clusters.
+- Enabled TLS for Trident's etcd client and the etcd-copy utility.
+- Added scripts and instructions for setting up an external etcd cluster using
+  etcd operator.
+
+## v17.10.0
 
 **Fixes:**
 - tridentctl correctly handles larger payloads using chunked encoding.

Dockerfile

@@ -27,8 +27,6 @@ ARG K8S=""
 ENV K8S $K8S
 ENV TRIDENT_IP localhost
 
-COPY ./scripts/* /usr/local/sbin/
-COPY $BIN /usr/local/bin
-COPY $CLI_BIN /usr/local/bin
+COPY ./scripts/* $BIN $CLI_BIN ./extras/external-etcd/bin/etcd-copy /usr/local/bin/
 
 CMD ["/usr/local/bin/$BIN -port $PORT -etcd_v3 $ETCDV3 -k8s_api_server $K8S"]

Makefile

@@ -23,7 +23,7 @@ CLI_PKG ?= github.com/netapp/trident/cli
 
 DIST_REGISTRY?=netapp
 
-TRIDENT_VERSION ?= 17.10.0
+TRIDENT_VERSION ?= 18.01.0
 
 ifeq ($(BUILD_TYPE),custom)
 TRIDENT_VERSION := ${TRIDENT_VERSION}-custom
@@ -60,7 +60,7 @@ DR=docker run --rm \
 
 GO=${DR} go
 
-.PHONY=default get build docker_get docker_build docker_image clean fmt install test test_core vet launcher_build launcher_start launch pod_launch prep_pod_template clear_trident
+.PHONY=default get build docker_get docker_build docker_image clean fmt install test test_core vet launcher_build launcher_start pod_launch prep_pod_template clear_trident
 
 SRCS = $(shell find . -name "*.go")
 
@@ -89,7 +89,7 @@ build:
 	@mkdir -p ${BIN_DIR}
 	@go ${BUILD} -ldflags $(BUILD_FLAGS) -o ${BIN_DIR}/${BIN}
 	@go ${BUILD} -ldflags $(BUILD_FLAGS) -o ${BIN_DIR}/${CLI_BIN} ${CLI_PKG}
-	@go ${BUILD} -ldflags $(BUILD_FLAGS) -o trident-installer/etcd/bin/etcd-copy github.com/netapp/trident/trident-installer/etcd/etcd-copy
+	@go ${BUILD} -ldflags $(BUILD_FLAGS) -o ${ROOT}/extras/external-etcd/bin/etcd-copy github.com/netapp/trident/extras/external-etcd/etcd-copy
 
 vendor:
 	@mkdir -p vendor
@@ -103,7 +103,7 @@ docker_build: vendor *.go
 	@chmod 777 ${BIN_DIR}
 	@${GO} ${BUILD} -ldflags $(BUILD_FLAGS) -o ${TRIDENT_VOLUME_PATH}/bin/${BIN}
 	@${GO} ${BUILD} -ldflags $(BUILD_FLAGS) -o ${TRIDENT_VOLUME_PATH}/bin/${CLI_BIN} ${CLI_PKG}
-	@${GO} ${BUILD} -ldflags $(BUILD_FLAGS) -o ${TRIDENT_VOLUME_PATH}/trident-installer/etcd/bin/etcd-copy github.com/netapp/trident/trident-installer/etcd/etcd-copy
+	@${GO} ${BUILD} -ldflags $(BUILD_FLAGS) -o ${TRIDENT_VOLUME_PATH}/extras/external-etcd/bin/etcd-copy github.com/netapp/trident/extras/external-etcd/etcd-copy
 
 docker_image: docker_retag docker_build
 	cp ${BIN_DIR}/${BIN} .
@@ -212,14 +212,18 @@ dist_tar:
 	-rm -rf /tmp/trident-installer
 	@cp -a trident-installer /tmp/
 	@cp ${BIN_DIR}/${CLI_BIN} /tmp/trident-installer/
-	@mkdir -p /tmp/trident-installer/bin
-	@cp ${BIN_DIR}/${BIN} /tmp/trident-installer/bin
-	@cp launcher/docker-build/launcher /tmp/trident-installer/bin
-	-rm -f /tmp/trident-installer/setup/backend.json
-	@rm -rf /tmp/trident-installer/etcd/etcd-copy
+	@cp -a extras /tmp/trident-installer/
+	@mkdir -p /tmp/trident-installer/extras/bin
+	@cp ${BIN_DIR}/${BIN} /tmp/trident-installer/extras/bin
+	@cp launcher/docker-build/launcher /tmp/trident-installer/extras/bin
+	-rm -rf /tmp/trident-installer/setup/backend.json /tmp/trident-installer/extras/container-tools
+	@rm -rf /tmp/trident-installer/extras/external-etcd/etcd-copy
+	-find /tmp/trident-installer -name \*.swp | xargs rm
 	@mkdir -p /tmp/trident-installer/setup
 	@sed "s|__LAUNCHER_TAG__|${LAUNCHER_DIST_TAG}|g" ./launcher/kubernetes-yaml/launcher-pod.yaml.templ > /tmp/trident-installer/launcher-pod.yaml
 	@sed "s|__TRIDENT_IMAGE__|${TRIDENT_DIST_TAG}|g" kubernetes-yaml/trident-deployment.yaml.templ > /tmp/trident-installer/setup/trident-deployment.yaml
+	@sed "s|__TRIDENT_IMAGE__|${TRIDENT_DIST_TAG}|g" kubernetes-yaml/trident-deployment-external-etcd.yaml.templ > /tmp/trident-installer/extras/external-etcd/trident/trident-deployment-external-etcd.yaml
+	@sed "s|__TRIDENT_IMAGE__|${TRIDENT_DIST_TAG}|g" kubernetes-yaml/etcdcopy-job.yaml.templ > /tmp/trident-installer/extras/external-etcd/trident/etcdcopy-job.yaml
 	@cp kubernetes-yaml/trident-namespace.yaml /tmp/trident-installer/
 	@cp kubernetes-yaml/trident-serviceaccounts.yaml /tmp/trident-installer/
 	@cp kubernetes-yaml/trident-clusterrole* /tmp/trident-installer/
@@ -240,20 +244,8 @@ build_all: pod docker_launcher_build
 
 build_and_launch: clear_trident pod launcher
 
-dist: build_all dist_tar dist_tag
+build_container_tools:
+	make -C extras/container-tools container_tools
 
-launch: 
-ifndef LAUNCHER_BACKEND
-	$(error Must define LAUNCHER_BACKEND to start the launcher.)
-endif
-	-kubectl delete --ignore-not-found=true configmap trident-launcher-config
-	-kubectl delete --ignore-not-found=true pod trident-launcher
-	@mkdir -p ${LAUNCHER_CONFIG_DIR}
-	@cp ${LAUNCHER_BACKEND} ${LAUNCHER_CONFIG_DIR}/backend.json
-	@sed "s|__TRIDENT_IMAGE__|netapp/trident:latest|g" kubernetes-yaml/trident-deployment.yaml.templ > ${LAUNCHER_CONFIG_DIR}/trident-deployment.yaml
-	@echo "Usable Trident pod definition available at ./launcher/kubernetes-yaml/trident-deployment.yaml"
-	@kubectl create configmap trident-launcher-config --from-file=${LAUNCHER_CONFIG_DIR}
-	@sed "s|__LAUNCHER_TAG__|netapp/trident-launcher:latest|g" ./launcher/kubernetes-yaml/launcher-pod.yaml.templ > ${LAUNCHER_POD_FILE}
-	@kubectl create -f ${LAUNCHER_POD_FILE}
-	@echo "Trident Launcher started; pod definition in ${LAUNCHER_POD_FILE}"
+dist: build_all dist_tar dist_tag build_container_tools
 

README.md

@@ -147,6 +147,10 @@ Kubernetes.
 	section for some advice on how to deal with some of the pitfalls that can
 	occur during this step.
 
+	If you are planning to deploy Trident with an external etcd cluster, please
+	follow the instructions in [Configuring Trident with an External etcd Cluster](https://github.com/NetApp/trident/tree/master/extras/external-etcd/README.md#configuring-trident-with-an-external-etcd-cluster)
+	instead of performing this step.
+
 7.  Make sure the `tridentctl` CLI tool, which is included in the installer
     bundle, is accessible through the `$PATH` environment variable.
 
@@ -219,7 +223,7 @@ does have the following requirements, however:
   Trident manages.  The standard deployment definition distributed as part of
   the installer includes an etcd container, so there is no need to install etcd
   separately; however, it is possible to deploy Trident with an external etcd
-  cluster.
+  cluster: [Configuring Trident with an External etcd Cluster](https://github.com/NetApp/trident/tree/master/extras/external-etcd/README.md#configuring-trident-with-an-external-etcd-cluster).
 * Kubernetes 1.4/OpenShift Enterprise 3.4/OpenShift Origin 1.4 or later:
   Optional, but necessary for the integrations with Kubernetes.  While Trident
   can be run independently and managed via its CLI or REST API, users
@@ -502,6 +506,9 @@ Trident exposes several command line options.  These are as follows:
 
 * `-etcd_v3 <address>` or `-etcd_v2 <address>`:  Required; use this to specify
   the etcd deployment that Trident should use.
+* `-etcd_v3_cert <file>`: Optional, etcdV3 client certificate.
+* `-etcd_v3_cacert <file>`: Optional, etcdV3 client CA certificate.
+* `-etcd_v3_key <file>`: Optional, etcdV3 client private key.
 * `-k8s_pod`:  Optional; however, either this or `-k8s_api_server` must be set
   to enable Kubernetes support.  Setting this will cause Trident to use its
   containing pod's Kubernetes service account credentials to contact the API

config/config.go

@@ -16,7 +16,7 @@ type VolumeType string
 const (
 	/* Misc. orchestrator constants */
 	OrchestratorName                 = "trident"
-	orchestratorVersion              = "17.10.0"
+	orchestratorVersion              = "18.01.0"
 	OrchestratorAPIVersion           = "1"
 	PersistentStoreBootstrapAttempts = 30
 	PersistentStoreBootstrapTimeout  = PersistentStoreBootstrapAttempts * time.Second

core/orchestrator_core.go

@@ -45,17 +45,12 @@ func NewTridentOrchestrator(client persistent_store.Client) *tridentOrchestrator
 }
 
 func (o *tridentOrchestrator) transformPersistentState() error {
-	// Determine if Trident needs to transform the persistent state before bootstrapping:
 	// Transforming persistent state happens under two scenarios:
 	// 1) Change in the persistent store version (e.g., from etcdv2 to etcdv3)
 	// 2) Change in the Trident API version (e.g., from Trident API v1 to v2)
-	if o.storeClient.GetType() == persistent_store.MemoryStore {
-		// No persistence so no transformation
-		return nil
-	}
 	version, err := o.storeClient.GetVersion()
 	if err != nil && persistent_store.MatchKeyNotFoundErr(err) {
-		// Version should be etcdv2 and Trident API v1
+		// Persistent store and Trident API versions should be etcdv2 and v1 respectively.
 		version = &persistent_store.PersistentStateVersion{
 			string(persistent_store.EtcdV2Store),
 			config.OrchestratorAPIVersion,
@@ -64,68 +59,25 @@ func (o *tridentOrchestrator) transformPersistentState() error {
 		return fmt.Errorf("Couldn't determine the orchestrator persistent state version: %v",
 			err)
 	}
-	if config.OrchestratorAPIVersion != version.OrchestratorVersion {
+	if config.OrchestratorAPIVersion != version.OrchestratorAPIVersion {
 		log.WithFields(log.Fields{
-			"current_api_version": version.OrchestratorVersion,
+			"current_api_version": version.OrchestratorAPIVersion,
 			"desired_api_version": config.OrchestratorAPIVersion,
 		}).Info("Transforming Trident API objects on the persistent store.")
 		//TODO: transform Trident API objects
 	}
-	clientVersion := string(o.storeClient.GetType())
-	if clientVersion != version.PersistentStoreVersion {
-		log.WithFields(log.Fields{
-			"current_store_version": version.PersistentStoreVersion,
-			"desired_store_version": clientVersion,
-		}).Info("Transforming persistent state.")
-		var srcClient, destinationClient persistent_store.EtcdClient
-		switch o.storeClient.GetType() {
-		case persistent_store.EtcdV2Store:
-			destinationClient, err = persistent_store.NewEtcdClientV2(
-				o.storeClient.GetEndpoints())
-		case persistent_store.EtcdV3Store:
-			destinationClient, err = persistent_store.NewEtcdClientV3(
-				o.storeClient.GetEndpoints())
-		default:
-			return fmt.Errorf("Didn't recognize %v as a valid persistent store version!",
-				o.storeClient.GetType())
-		}
-		if err != nil {
-			return fmt.Errorf("Failed in creating the destination etcd client for data migration: %v",
-				err)
-		}
-		switch version.PersistentStoreVersion {
-		case string(persistent_store.EtcdV2Store):
-			srcClient, err = persistent_store.NewEtcdClientV2(
-				o.storeClient.GetEndpoints())
-		case string(persistent_store.EtcdV3Store):
-			srcClient, err = persistent_store.NewEtcdClientV3(
-				o.storeClient.GetEndpoints())
-		default:
-			return fmt.Errorf("Didn't recognize %v as a valid persistent store version!",
-				version.PersistentStoreVersion)
-		}
-		if err != nil {
-			return fmt.Errorf("Failed in creating the source etcd client for data migration: %v",
-				err)
-		}
-		dataMigrator := persistent_store.NewEtcdDataMigrator(srcClient,
-			destinationClient)
-		// Moving all Trident objects for all API versions
-		if err = dataMigrator.Start("/"+config.OrchestratorName, false); err != nil {
-			return fmt.Errorf("etcd data migration failed: %v", err)
-		}
-		if err = dataMigrator.Stop(); err != nil {
-			return fmt.Errorf("Failed to shut down the etcd data migrator: %v",
-				err)
-		}
+	dataMigrator := persistent_store.NewDataMigrator(o.storeClient,
+		persistent_store.StoreType(version.PersistentStoreVersion))
+	if err = dataMigrator.Run("/"+config.OrchestratorName, false); err != nil {
+		return fmt.Errorf("Data migration failed: %v", err)
+	}
 
-		// Store the persistent store and API versions
-		version.OrchestratorVersion = config.OrchestratorAPIVersion
-		version.PersistentStoreVersion = string(o.storeClient.GetType())
-		if err = o.storeClient.SetVersion(version); err != nil {
-			return fmt.Errorf("Failed to set the persistent state version after migration: %v",
-				err)
-		}
+	// Store the persistent store and API versions
+	version.OrchestratorAPIVersion = config.OrchestratorAPIVersion
+	version.PersistentStoreVersion = string(o.storeClient.GetType())
+	if err = o.storeClient.SetVersion(version); err != nil {
+		return fmt.Errorf("Failed to set the persistent state version after migration: %v",
+			err)
 	}
 	return nil
 }

core/orchestrator_core_test.go

@@ -1538,126 +1538,6 @@ func TestBootstrapEtcdV2ToEtcdV3Migration(t *testing.T) {
 	cleanupStoreVersion(t, *etcdV3)
 }
 
-func TestBootstrapEtcdV3ToEtcdV2Migration(t *testing.T) {
-	const (
-		backendName = "bootstrapV3toV2_backend"
-		scName      = "bootstrapV3toV2_class"
-		volumeName  = "bootstrapV3toV2_vol"
-	)
-	if *etcdV2 == "" {
-		t.SkipNow()
-	}
-	cleanupStoreVersion(t, *etcdV2)
-
-	// Populate the etcdv3 cluster with one backend, one storage class, and one volume
-	empty := ""
-	etcdV3 = etcdV2
-	etcdV2 = &empty
-
-	// Set the persistent state version to etcdv3
-	etcdv3Client, err := persistent_store.NewEtcdClientV3(*etcdV3)
-	if err != nil {
-		t.Fatalf("Creating etcdv3 client failed: %v", err)
-	}
-	version := &persistent_store.PersistentStateVersion{
-		string(persistent_store.EtcdV3Store),
-		config.OrchestratorAPIVersion,
-	}
-	versionJSON, err := json.Marshal(version)
-	if err != nil {
-		t.Fatalf("Marshalling persistent state version failed: %v", err)
-	}
-	if err = etcdv3Client.Set(config.StoreURL, string(versionJSON)); err != nil {
-		t.Fatalf("Setting persistent state version failed: %v", err)
-	}
-	orchestratorV3 := getOrchestrator()
-	addBackendStorageClass(t, orchestratorV3, backendName, scName)
-
-	orchestratorV3.mutex.Lock()
-	if _, ok := orchestratorV3.storageClasses[scName]; !ok {
-		t.Fatal("Storage class not found in the orchestrator map!")
-	}
-	orchestratorV3.mutex.Unlock()
-
-	v3Volume, err := orchestratorV3.AddVolume(generateVolumeConfig(volumeName, 50, scName,
-		config.File))
-	if err != nil {
-		t.Fatal("Unable to create volume: ", err)
-	}
-	orchestratorV3.mutex.Lock()
-	if _, ok := orchestratorV3.volumes[volumeName]; !ok {
-		t.Fatal("Volume name found in orchestrator map!")
-	}
-	orchestratorV3.mutex.Unlock()
-	// Set the current persistent state version to be etcdv3 and bring up the
-	// an etcdv2 orchestrator with etcdv3 data
-	etcdV2 = etcdV3
-	etcdv2Client, err := persistent_store.NewEtcdClientV2(*etcdV2)
-	if err != nil {
-		t.Fatalf("Creating etcdv2 client failed: %v", err)
-	}
-	version = &persistent_store.PersistentStateVersion{
-		string(orchestratorV3.storeClient.GetType()),
-		config.OrchestratorAPIVersion,
-	}
-	if versionJSON, err = json.Marshal(version); err != nil {
-		t.Fatalf("Marshalling persistent state version failed: %v", err)
-	}
-	if err = etcdv2Client.Set(config.StoreURL, string(versionJSON)); err != nil {
-		t.Fatalf("Setting persistent state version failed: %v", err)
-	}
-	// Bootstrap etcdv2 orchestrator with etcdv3 data
-	orchestratorV2 := getOrchestrator()
-
-	// Verify etcdv3 to etcdv2 transformation
-	if orchestratorV2.GetBackend(backendName) == nil {
-		t.Fatalf("Failed to find backend %s after bootstrapping!",
-			backendName)
-	}
-	//TODO: Optionally we can diff etcdv2 and etcdv3 backends here
-	if orchestratorV2.GetStorageClass(scName) == nil {
-		t.Fatalf("Failed to find storage class %s after bootstrapping!",
-			scName)
-	}
-	//TODO: Optionally we can diff etcdv2 and etcdv3 storage classes here
-	v2Volume := orchestratorV2.GetVolume(volumeName)
-	if v2Volume == nil {
-		t.Fatalf("Failed to find storage class %s after bootstrapping!",
-			volumeName)
-	}
-	// Verifying the same volume exists on both clusters
-	if !reflect.DeepEqual(v2Volume, v3Volume) {
-		t.Fatalf("etcdv2 volume (%v) doesn't match the etcdv3 volume (%v)!",
-			v2Volume, v3Volume)
-	}
-	versionJSONString, err := etcdv2Client.Read(config.StoreURL)
-	if err != nil {
-		t.Fatalf("Couldn't determine the orchestrator persistent state version: %v",
-			err)
-	}
-	err = json.Unmarshal([]byte(versionJSONString), version)
-	if err != nil {
-		t.Fatalf("Couldn't unmarshall the orchestrator persistent state version: %v",
-			err)
-	}
-	if config.OrchestratorAPIVersion != version.OrchestratorVersion ||
-		string(orchestratorV2.storeClient.GetType()) != version.PersistentStoreVersion {
-		t.Fatalf("Failed to set the orchestrator persistent state version after bootsrapping: %v",
-			err)
-	}
-
-	// cleanup
-	if err = etcdv3Client.Stop(); err != nil {
-		t.Fatalf("Couldn't shut down etcdv3 client: %v", err)
-	}
-	if err = etcdv2Client.Stop(); err != nil {
-		t.Fatalf("Couldn't shut down etcdv2 client: %v", err)
-	}
-	cleanup(t, orchestratorV2)
-	cleanup(t, orchestratorV3)
-	cleanupStoreVersion(t, *etcdV2)
-}
-
 // The next series of tests test that bootstrap doesn't exit early if it
 // encounters a key error for one of the main types of entries.
 func TestStorageClassOnlyBootstrap(t *testing.T) {