diff --git a/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml b/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml index 0c4ec33ae..20f8a2330 100644 --- a/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml +++ b/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml @@ -939,6 +939,8 @@ spec: type: string secret-namespace: type: string + strictTLSMode: + type: boolean token: type: string url: diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 13b27ce6d..b7fd5e61f 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -112,6 +112,8 @@ type Registration struct { } type SystemAgent struct { + // +optional + StrictTLSMode bool `json:"strictTLSMode,omitempty" yaml:"strictTLSMode,omitempty"` // +optional URL string `json:"url,omitempty" yaml:"url,omitempty"` // +optional diff --git a/config/crd/bases/elemental.cattle.io_machineregistrations.yaml b/config/crd/bases/elemental.cattle.io_machineregistrations.yaml index 930b6ec86..daf3faf57 100644 --- a/config/crd/bases/elemental.cattle.io_machineregistrations.yaml +++ b/config/crd/bases/elemental.cattle.io_machineregistrations.yaml @@ -173,6 +173,8 @@ spec: type: string secret-namespace: type: string + strictTLSMode: + type: boolean token: type: string url: diff --git a/pkg/install/_testdata/after-hook-config-install.yaml b/pkg/install/_testdata/after-hook-config-install.yaml index d0fdefc91..491558901 100644 --- a/pkg/install/_testdata/after-hook-config-install.yaml +++ b/pkg/install/_testdata/after-hook-config-install.yaml @@ -106,6 +106,13 @@ stages: connectionInfoFile: /var/lib/elemental/agent/elemental_connection.json encoding: "" ownerstring: "" + - path: /etc/rancher/elemental/agent/envs + permissions: 384 + owner: 0 + group: 0 + content: CATTLE_AGENT_STRICT_VERIFY="true" + encoding: "" + ownerstring: "" encoding: "" ownerstring: "" name: Elemental System Agent Config diff --git a/pkg/install/_testdata/after-hook-config-reset.yaml b/pkg/install/_testdata/after-hook-config-reset.yaml index a1a5de52f..127ca1ef4 100644 --- a/pkg/install/_testdata/after-hook-config-reset.yaml +++ b/pkg/install/_testdata/after-hook-config-reset.yaml @@ -106,6 +106,13 @@ stages: connectionInfoFile: /var/lib/elemental/agent/elemental_connection.json encoding: "" ownerstring: "" + - path: /etc/rancher/elemental/agent/envs + permissions: 384 + owner: 0 + group: 0 + content: CATTLE_AGENT_STRICT_VERIFY="true" + encoding: "" + ownerstring: "" encoding: "" ownerstring: "" name: Elemental System Agent Config diff --git a/pkg/install/install.go b/pkg/install/install.go index 3faa0e881..e154b3b1c 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -476,6 +476,10 @@ func (i *installer) getAgentConfigBytes() ([]byte, error) { return agentConfigBytes, nil } +func (i *installer) getAgentConfigEnvs(config elementalv1.Elemental) []byte { + return []byte(fmt.Sprintf("CATTLE_AGENT_STRICT_VERIFY=\"%t\"", config.SystemAgent.StrictTLSMode)) +} + // Write system agent config files to local filesystem func (i *installer) WriteLocalSystemAgentConfig(config elementalv1.Elemental) error { connectionInfoBytes, err := i.getConnectionInfoBytes(config) @@ -495,6 +499,13 @@ func (i *installer) WriteLocalSystemAgentConfig(config elementalv1.Elemental) er } log.Infof("connection info file '%s' written.", connectionInfoFile) + elementalAgentEnvsFile := filepath.Join(agentConfDir, "envs") + err = os.WriteFile(elementalAgentEnvsFile, i.getAgentConfigEnvs(config), 0600) + if err != nil { + return fmt.Errorf("writing agent envs file: %w", err) + } + log.Infof("agent envs file '%s' written.", elementalAgentEnvsFile) + agentConfigBytes, err := i.getAgentConfigBytes() if err != nil { return fmt.Errorf("getting agent config: %w", err) @@ -523,6 +534,7 @@ func (i *installer) elementalSystemAgentYip(config elementalv1.Elemental) ([]byt if err != nil { return nil, fmt.Errorf("getting agent config: %w", err) } + agentEnvsBytes := i.getAgentConfigEnvs(config) var stages []schema.Stage @@ -538,6 +550,11 @@ func (i *installer) elementalSystemAgentYip(config elementalv1.Elemental) ([]byt Content: string(agentConfigBytes), Permissions: 0600, }, + { + Path: filepath.Join(agentConfDir, "envs"), + Content: string(agentEnvsBytes), + Permissions: 0600, + }, }, }) diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go index a978d7c34..4758cc449 100644 --- a/pkg/install/install_test.go +++ b/pkg/install/install_test.go @@ -76,6 +76,7 @@ var ( Reboot: true, }, SystemAgent: elementalv1.SystemAgent{ + StrictTLSMode: true, URL: "https://127.0.0.1.sslip.io/test/control/plane/endpoint", Token: "a test token", SecretName: "a test secret name", diff --git a/pkg/server/api_registration.go b/pkg/server/api_registration.go index 848527ffa..85a4cebde 100644 --- a/pkg/server/api_registration.go +++ b/pkg/server/api_registration.go @@ -162,6 +162,7 @@ func (i *InventoryServer) writeMachineInventoryCloudConfig(conn *websocket.Conn, return err } config.Elemental.SystemAgent = elementalv1.SystemAgent{ + StrictTLSMode: i.isAgentTLSModeStrict(), URL: fmt.Sprintf("%s/k8s/clusters/local", serverURL), Token: string(secret.Data["token"]), SecretName: inventory.Name, @@ -221,6 +222,23 @@ func (i *InventoryServer) getRancherCACert() string { return cacert } +// Support for agent-tls-mode +func (i *InventoryServer) isAgentTLSModeStrict() bool { + agentTLSMode, err := i.getValue("agent-tls-mode") + if err != nil { + log.Errorf("Error getting agent-tls-mode: %s", err.Error()) + } + switch agentTLSMode { + case "strict": + return true + case "system-store": + return false + default: + // Historically the default has been strict TLS verification + return true + } +} + func (i *InventoryServer) serveLoop(conn *websocket.Conn, inventory *elementalv1.MachineInventory, registration *elementalv1.MachineRegistration) error { //nolint: gocyclo protoVersion := register.MsgUndefined tmpl := templater.NewTemplater() diff --git a/pkg/server/api_registration_test.go b/pkg/server/api_registration_test.go index 538903de4..1c143f679 100644 --- a/pkg/server/api_registration_test.go +++ b/pkg/server/api_registration_test.go @@ -38,6 +38,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/fake" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" @@ -687,6 +688,51 @@ func TestRegistrationDynamicLabels(t *testing.T) { }) } +func TestAgentTLSMode(t *testing.T) { + type test struct { + name string + agentTLSModeValue *string + wantStrictTLSMode bool + } + + tests := []test{ + { + name: "missing agent-tls-mode", + agentTLSModeValue: nil, + wantStrictTLSMode: true, + }, + { + name: "strict agent-tls-mode", + agentTLSModeValue: ptr.To("strict"), + wantStrictTLSMode: true, + }, + { + name: "system-store agent-tls-mode", + agentTLSModeValue: ptr.To("system-store"), + wantStrictTLSMode: false, + }, + } + + for _, tt := range tests { + server := NewInventoryServer(&FakeAuthServer{}) + + t.Run(tt.name, func(t *testing.T) { + if tt.agentTLSModeValue != nil { + server.Client.Create(context.Background(), &managementv3.Setting{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-tls-mode", + }, + + Value: *tt.agentTLSModeValue, + }) + } + + assert.Equal(t, server.isAgentTLSModeStrict(), tt.wantStrictTLSMode) + }) + } + +} + func NewInventoryServer(auth authenticator) *InventoryServer { scheme := runtime.NewScheme() elementalv1.AddToScheme(scheme) diff --git a/tests/e2e/config/config.yaml b/tests/e2e/config/config.yaml index 2ae6a0e47..a8a65b047 100644 --- a/tests/e2e/config/config.yaml +++ b/tests/e2e/config/config.yaml @@ -12,7 +12,7 @@ nginxURL: https://raw.githubusercontent.com/kubernetes/ingress-nginx/${NGINX_VER certManagerVersion: v1.14.2 certManagerChartURL: https://charts.jetstack.io/charts/cert-manager-${CERT_MANAGER_VERSION}.tgz -rancherVersion: 2.8.2 +rancherVersion: 2.9.2 rancherChartURL: https://releases.rancher.com/server-charts/latest/rancher-${RANCHER_VERSION}.tgz systemUpgradeControllerVersion: v0.13.4 diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index e74d305d7..03353dd2c 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -23,6 +23,7 @@ import ( "io" "net/http" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -199,6 +200,48 @@ var _ = BeforeSuite(func() { }, 5*time.Minute, 2*time.Second).Should(BeTrue()) }) + By("creating cattle-system namespace", func() { + Expect(kubectl.Apply(testRegistryNamespace, "../manifests/cattle-system-namespace.yaml")).To(Succeed()) + }) + + By("installing a self-signed CA", func() { + Expect(kubectl.Apply("cattle-system", "../manifests/test-private-ca.yaml")).To(Succeed()) + + Eventually(func() bool { + return doesSecretExist("cattle-system", "tls-ca") + }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + Eventually(func() bool { + return doesSecretExist("cattle-system", "tls-rancher-ingress") + }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + + // We need to cope with the arbitrary and hardcoded `cacerts.pem` secret key + // See https://github.com/rancher/rancher/issues/36994 + + // For this reason we fetch the cert-manager generated data['tls.crt'] from the tls-ca secret, + // and we copy its value to data['cacerts.pem'] where Rancher expects it. + // See the rancher Deployment in cattle-system namespace for more info on how this is mounted. + + printCA := "-n cattle-system get secret tls-ca -o jsonpath=\"{.data['tls\\.crt']}\"" + caCert, err := kubectl.Run(strings.Split(printCA, " ")...) + caCert = strings.ReplaceAll(caCert, `"`, "") + Expect(err).ShouldNot(HaveOccurred()) + + //patch := fmt.Sprintf(`-n cattle-system patch secret tls-ca -p "{\"data\":{\"cacerts.pem\":\"%s\"}}"`, caCert) + //_, err = kubectl.Run(strings.Split(patch, " ")...) + //Expect(err).ShouldNot(HaveOccurred()) + + // If you wonder what the heck is happening here with the bash script, uncomment the lines above and knock yourself out. + // It has been a long day. + patchScript := fmt.Sprintf(`kubectl -n cattle-system patch secret tls-ca -p '{"data":{"cacerts.pem":"%s"}}'`, caCert) + Expect(os.WriteFile("/tmp/kubectl-patch-tls-ca.sh", []byte(patchScript), os.ModePerm)).Should(Succeed()) + cmd := exec.Command("bash", "/tmp/kubectl-patch-tls-ca.sh") + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Failed to patch tls-ca: %s\n", string(output)) + } + Expect(err).ShouldNot(HaveOccurred()) + }) + By("installing rancher: "+e2eCfg.RancherVersion, func() { if isAlreadyInstalled(cattleSystemNamespace) { By("already installed") @@ -218,25 +261,36 @@ var _ = BeforeSuite(func() { "--set", "extraEnv[0].value=https://"+hostname, "--set", "extraEnv[1].name=CATTLE_BOOTSTRAP_PASSWORD", "--set", "extraEnv[1].value="+password, + "--set", "privateCA=true", + "--set", "agentTLSMode=system-store", "--namespace", cattleSystemNamespace, - "--create-namespace", )).To(Succeed()) Eventually(func() bool { return isDeploymentReady(cattleSystemNamespace, rancherName) }, 5*time.Minute, 2*time.Second).Should(BeTrue()) - Eventually(func() bool { - return isDeploymentReady(cattleFleetNamespace, fleetAgent) - }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + rancherVer, err := checkver.NewVersion(e2eCfg.RancherVersion) + Expect(err).ToNot(HaveOccurred()) - // capi-controller exists only since Rancher Manager v2.7.8 - refVer, err := checkver.NewVersion("2.7.8") + // fleet is deployed as statefulSet since 2.9 + fleetStatefulSetVer, err := checkver.NewVersion("2.9.0") Expect(err).ToNot(HaveOccurred()) - rancherVer, err := checkver.NewVersion(e2eCfg.RancherVersion) + if rancherVer.GreaterThanOrEqual(fleetStatefulSetVer) { + Eventually(func() bool { + return isStatefulSetReady(cattleFleetNamespace, fleetAgent) + }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + } else { + Eventually(func() bool { + return isDeploymentReady(cattleFleetNamespace, fleetAgent) + }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + } + + // capi-controller exists only since Rancher Manager v2.7.8 + nonCapiVer, err := checkver.NewVersion("2.7.8") Expect(err).ToNot(HaveOccurred()) - if rancherVer.GreaterThanOrEqual(refVer) { + if rancherVer.GreaterThanOrEqual(nonCapiVer) { Eventually(func() bool { return isDeploymentReady(cattleCapiNamespace, capiController) }, 5*time.Minute, 2*time.Second).Should(BeTrue()) @@ -380,6 +434,39 @@ func isDeploymentReady(namespace, name string) bool { return false } +func isStatefulSetReady(namespace, name string) bool { + statefulSet := &appsv1.StatefulSet{} + if err := cl.Get(ctx, + runtimeclient.ObjectKey{ + Namespace: namespace, + Name: name, + }, + statefulSet, + ); err != nil { + return false + } + + if statefulSet.Status.AvailableReplicas == *statefulSet.Spec.Replicas { + return true + } + + return false +} + +func doesSecretExist(namespace, name string) bool { + secret := &corev1.Secret{} + if err := cl.Get(ctx, + runtimeclient.ObjectKey{ + Namespace: namespace, + Name: name, + }, + secret, + ); err != nil { + return false + } + return true +} + func collectArtifacts() { By("Creating artifact directory") if _, err := os.Stat(e2eCfg.ArtifactsDir); os.IsNotExist(err) { diff --git a/tests/manifests/cattle-system-namespace.yaml b/tests/manifests/cattle-system-namespace.yaml new file mode 100644 index 000000000..35597d851 --- /dev/null +++ b/tests/manifests/cattle-system-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cattle-system diff --git a/tests/manifests/test-private-ca.yaml b/tests/manifests/test-private-ca.yaml new file mode 100644 index 000000000..48b3d1db5 --- /dev/null +++ b/tests/manifests/test-private-ca.yaml @@ -0,0 +1,46 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: tls-ca + namespace: cattle-system +spec: + commonName: elemental-selfsigned-ca + duration: 94800h + isCA: true + issuerRef: + kind: Issuer + name: elemental-selfsigned + renewBefore: 360h + secretName: tls-ca +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: tls-rancher-ingress + namespace: cattle-system +spec: + dnsNames: + - 172.18.0.2.sslip.io + duration: 9480h + issuerRef: + kind: Issuer + name: elemental-ca + renewBefore: 360h + secretName: tls-rancher-ingress +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: elemental-ca + namespace: cattle-system +spec: + ca: + secretName: tls-ca +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: elemental-selfsigned + namespace: cattle-system +spec: + selfSigned: {}