diff --git a/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml b/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml index 0c4ec33a..20f8a233 100644 --- a/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml +++ b/.obs/chartfile/elemental-operator-crds-helm/templates/crds.yaml @@ -939,6 +939,8 @@ spec: type: string secret-namespace: type: string + strictTLSMode: + type: boolean token: type: string url: diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 13b27ce6..b7fd5e61 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -112,6 +112,8 @@ type Registration struct { } type SystemAgent struct { + // +optional + StrictTLSMode bool `json:"strictTLSMode,omitempty" yaml:"strictTLSMode,omitempty"` // +optional URL string `json:"url,omitempty" yaml:"url,omitempty"` // +optional diff --git a/config/crd/bases/elemental.cattle.io_machineregistrations.yaml b/config/crd/bases/elemental.cattle.io_machineregistrations.yaml index 930b6ec8..daf3faf5 100644 --- a/config/crd/bases/elemental.cattle.io_machineregistrations.yaml +++ b/config/crd/bases/elemental.cattle.io_machineregistrations.yaml @@ -173,6 +173,8 @@ spec: type: string secret-namespace: type: string + strictTLSMode: + type: boolean token: type: string url: diff --git a/pkg/install/_testdata/after-hook-config-install.yaml b/pkg/install/_testdata/after-hook-config-install.yaml index d0fdefc9..49155890 100644 --- a/pkg/install/_testdata/after-hook-config-install.yaml +++ b/pkg/install/_testdata/after-hook-config-install.yaml @@ -106,6 +106,13 @@ stages: connectionInfoFile: /var/lib/elemental/agent/elemental_connection.json encoding: "" ownerstring: "" + - path: /etc/rancher/elemental/agent/envs + permissions: 384 + owner: 0 + group: 0 + content: CATTLE_AGENT_STRICT_VERIFY="true" + encoding: "" + ownerstring: "" encoding: "" ownerstring: "" name: Elemental System Agent Config diff --git a/pkg/install/_testdata/after-hook-config-reset.yaml b/pkg/install/_testdata/after-hook-config-reset.yaml index a1a5de52..127ca1ef 100644 --- a/pkg/install/_testdata/after-hook-config-reset.yaml +++ b/pkg/install/_testdata/after-hook-config-reset.yaml @@ -106,6 +106,13 @@ stages: connectionInfoFile: /var/lib/elemental/agent/elemental_connection.json encoding: "" ownerstring: "" + - path: /etc/rancher/elemental/agent/envs + permissions: 384 + owner: 0 + group: 0 + content: CATTLE_AGENT_STRICT_VERIFY="true" + encoding: "" + ownerstring: "" encoding: "" ownerstring: "" name: Elemental System Agent Config diff --git a/pkg/install/install.go b/pkg/install/install.go index 3faa0e88..e154b3b1 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -476,6 +476,10 @@ func (i *installer) getAgentConfigBytes() ([]byte, error) { return agentConfigBytes, nil } +func (i *installer) getAgentConfigEnvs(config elementalv1.Elemental) []byte { + return []byte(fmt.Sprintf("CATTLE_AGENT_STRICT_VERIFY=\"%t\"", config.SystemAgent.StrictTLSMode)) +} + // Write system agent config files to local filesystem func (i *installer) WriteLocalSystemAgentConfig(config elementalv1.Elemental) error { connectionInfoBytes, err := i.getConnectionInfoBytes(config) @@ -495,6 +499,13 @@ func (i *installer) WriteLocalSystemAgentConfig(config elementalv1.Elemental) er } log.Infof("connection info file '%s' written.", connectionInfoFile) + elementalAgentEnvsFile := filepath.Join(agentConfDir, "envs") + err = os.WriteFile(elementalAgentEnvsFile, i.getAgentConfigEnvs(config), 0600) + if err != nil { + return fmt.Errorf("writing agent envs file: %w", err) + } + log.Infof("agent envs file '%s' written.", elementalAgentEnvsFile) + agentConfigBytes, err := i.getAgentConfigBytes() if err != nil { return fmt.Errorf("getting agent config: %w", err) @@ -523,6 +534,7 @@ func (i *installer) elementalSystemAgentYip(config elementalv1.Elemental) ([]byt if err != nil { return nil, fmt.Errorf("getting agent config: %w", err) } + agentEnvsBytes := i.getAgentConfigEnvs(config) var stages []schema.Stage @@ -538,6 +550,11 @@ func (i *installer) elementalSystemAgentYip(config elementalv1.Elemental) ([]byt Content: string(agentConfigBytes), Permissions: 0600, }, + { + Path: filepath.Join(agentConfDir, "envs"), + Content: string(agentEnvsBytes), + Permissions: 0600, + }, }, }) diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go index a978d7c3..4758cc44 100644 --- a/pkg/install/install_test.go +++ b/pkg/install/install_test.go @@ -76,6 +76,7 @@ var ( Reboot: true, }, SystemAgent: elementalv1.SystemAgent{ + StrictTLSMode: true, URL: "https://127.0.0.1.sslip.io/test/control/plane/endpoint", Token: "a test token", SecretName: "a test secret name", diff --git a/pkg/server/api_registration.go b/pkg/server/api_registration.go index 848527ff..85a4cebd 100644 --- a/pkg/server/api_registration.go +++ b/pkg/server/api_registration.go @@ -162,6 +162,7 @@ func (i *InventoryServer) writeMachineInventoryCloudConfig(conn *websocket.Conn, return err } config.Elemental.SystemAgent = elementalv1.SystemAgent{ + StrictTLSMode: i.isAgentTLSModeStrict(), URL: fmt.Sprintf("%s/k8s/clusters/local", serverURL), Token: string(secret.Data["token"]), SecretName: inventory.Name, @@ -221,6 +222,23 @@ func (i *InventoryServer) getRancherCACert() string { return cacert } +// Support for agent-tls-mode +func (i *InventoryServer) isAgentTLSModeStrict() bool { + agentTLSMode, err := i.getValue("agent-tls-mode") + if err != nil { + log.Errorf("Error getting agent-tls-mode: %s", err.Error()) + } + switch agentTLSMode { + case "strict": + return true + case "system-store": + return false + default: + // Historically the default has been strict TLS verification + return true + } +} + func (i *InventoryServer) serveLoop(conn *websocket.Conn, inventory *elementalv1.MachineInventory, registration *elementalv1.MachineRegistration) error { //nolint: gocyclo protoVersion := register.MsgUndefined tmpl := templater.NewTemplater() diff --git a/pkg/server/api_registration_test.go b/pkg/server/api_registration_test.go index 538903de..1c143f67 100644 --- a/pkg/server/api_registration_test.go +++ b/pkg/server/api_registration_test.go @@ -38,6 +38,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/fake" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" @@ -687,6 +688,51 @@ func TestRegistrationDynamicLabels(t *testing.T) { }) } +func TestAgentTLSMode(t *testing.T) { + type test struct { + name string + agentTLSModeValue *string + wantStrictTLSMode bool + } + + tests := []test{ + { + name: "missing agent-tls-mode", + agentTLSModeValue: nil, + wantStrictTLSMode: true, + }, + { + name: "strict agent-tls-mode", + agentTLSModeValue: ptr.To("strict"), + wantStrictTLSMode: true, + }, + { + name: "system-store agent-tls-mode", + agentTLSModeValue: ptr.To("system-store"), + wantStrictTLSMode: false, + }, + } + + for _, tt := range tests { + server := NewInventoryServer(&FakeAuthServer{}) + + t.Run(tt.name, func(t *testing.T) { + if tt.agentTLSModeValue != nil { + server.Client.Create(context.Background(), &managementv3.Setting{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-tls-mode", + }, + + Value: *tt.agentTLSModeValue, + }) + } + + assert.Equal(t, server.isAgentTLSModeStrict(), tt.wantStrictTLSMode) + }) + } + +} + func NewInventoryServer(auth authenticator) *InventoryServer { scheme := runtime.NewScheme() elementalv1.AddToScheme(scheme) diff --git a/tests/e2e/config/config.yaml b/tests/e2e/config/config.yaml index 2ae6a0e4..a8a65b04 100644 --- a/tests/e2e/config/config.yaml +++ b/tests/e2e/config/config.yaml @@ -12,7 +12,7 @@ nginxURL: https://raw.githubusercontent.com/kubernetes/ingress-nginx/${NGINX_VER certManagerVersion: v1.14.2 certManagerChartURL: https://charts.jetstack.io/charts/cert-manager-${CERT_MANAGER_VERSION}.tgz -rancherVersion: 2.8.2 +rancherVersion: 2.9.2 rancherChartURL: https://releases.rancher.com/server-charts/latest/rancher-${RANCHER_VERSION}.tgz systemUpgradeControllerVersion: v0.13.4 diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index cb2893f3..03353dd2 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -242,7 +242,7 @@ var _ = BeforeSuite(func() { Expect(err).ShouldNot(HaveOccurred()) }) - By("installing rancher"+e2eCfg.RancherVersion, func() { + By("installing rancher: "+e2eCfg.RancherVersion, func() { if isAlreadyInstalled(cattleSystemNamespace) { By("already installed") return @@ -262,6 +262,7 @@ var _ = BeforeSuite(func() { "--set", "extraEnv[1].name=CATTLE_BOOTSTRAP_PASSWORD", "--set", "extraEnv[1].value="+password, "--set", "privateCA=true", + "--set", "agentTLSMode=system-store", "--namespace", cattleSystemNamespace, )).To(Succeed()) @@ -269,17 +270,27 @@ var _ = BeforeSuite(func() { return isDeploymentReady(cattleSystemNamespace, rancherName) }, 5*time.Minute, 2*time.Second).Should(BeTrue()) - Eventually(func() bool { - return isDeploymentReady(cattleFleetNamespace, fleetAgent) - }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + rancherVer, err := checkver.NewVersion(e2eCfg.RancherVersion) + Expect(err).ToNot(HaveOccurred()) - // capi-controller exists only since Rancher Manager v2.7.8 - refVer, err := checkver.NewVersion("2.7.8") + // fleet is deployed as statefulSet since 2.9 + fleetStatefulSetVer, err := checkver.NewVersion("2.9.0") Expect(err).ToNot(HaveOccurred()) - rancherVer, err := checkver.NewVersion(e2eCfg.RancherVersion) + if rancherVer.GreaterThanOrEqual(fleetStatefulSetVer) { + Eventually(func() bool { + return isStatefulSetReady(cattleFleetNamespace, fleetAgent) + }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + } else { + Eventually(func() bool { + return isDeploymentReady(cattleFleetNamespace, fleetAgent) + }, 5*time.Minute, 2*time.Second).Should(BeTrue()) + } + + // capi-controller exists only since Rancher Manager v2.7.8 + nonCapiVer, err := checkver.NewVersion("2.7.8") Expect(err).ToNot(HaveOccurred()) - if rancherVer.GreaterThanOrEqual(refVer) { + if rancherVer.GreaterThanOrEqual(nonCapiVer) { Eventually(func() bool { return isDeploymentReady(cattleCapiNamespace, capiController) }, 5*time.Minute, 2*time.Second).Should(BeTrue()) @@ -423,6 +434,25 @@ func isDeploymentReady(namespace, name string) bool { return false } +func isStatefulSetReady(namespace, name string) bool { + statefulSet := &appsv1.StatefulSet{} + if err := cl.Get(ctx, + runtimeclient.ObjectKey{ + Namespace: namespace, + Name: name, + }, + statefulSet, + ); err != nil { + return false + } + + if statefulSet.Status.AvailableReplicas == *statefulSet.Spec.Replicas { + return true + } + + return false +} + func doesSecretExist(namespace, name string) bool { secret := &corev1.Secret{} if err := cl.Get(ctx,