diff --git a/bootstrap/api/v1beta1/conversion.go b/bootstrap/api/v1beta1/conversion.go index 941f23db..07155d94 100644 --- a/bootstrap/api/v1beta1/conversion.go +++ b/bootstrap/api/v1beta1/conversion.go @@ -43,6 +43,8 @@ func (c *KThreesConfig) ConvertTo(dstRaw ctrlconversion.Hub) error { dst.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider dst.Spec.ServerConfig.DisableCloudController = restored.Spec.ServerConfig.DisableCloudController dst.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.ServerConfig.SystemDefaultRegistry + dst.Spec.ServerConfig.EtcdProxyImage = restored.Spec.ServerConfig.EtcdProxyImage + dst.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.AgentConfig.AirGappedInstallScriptPath return nil } @@ -96,6 +98,8 @@ func (r *KThreesConfigTemplate) ConvertTo(dstRaw ctrlconversion.Hub) error { dst.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider dst.Spec.Template.Spec.ServerConfig.DisableCloudController = restored.Spec.Template.Spec.ServerConfig.DisableCloudController dst.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry + dst.Spec.Template.Spec.ServerConfig.EtcdProxyImage = restored.Spec.Template.Spec.ServerConfig.EtcdProxyImage + dst.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath return nil } @@ -159,3 +163,8 @@ func Convert_v1beta2_KThreesServerConfig_To_v1beta1_KThreesServerConfig(in *boot return nil } + +// Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig is an autogenerated conversion function. +func Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in *bootstrapv1beta2.KThreesAgentConfig, out *KThreesAgentConfig, s conversion.Scope) error { //nolint: stylecheck + return autoConvert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in, out, s) +} diff --git a/bootstrap/api/v1beta1/zz_generated.conversion.go b/bootstrap/api/v1beta1/zz_generated.conversion.go index 6a2dec08..963e96a9 100644 --- a/bootstrap/api/v1beta1/zz_generated.conversion.go +++ b/bootstrap/api/v1beta1/zz_generated.conversion.go @@ -61,11 +61,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta2.KThreesAgentConfig)(nil), (*KThreesAgentConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(a.(*v1beta2.KThreesAgentConfig), b.(*KThreesAgentConfig), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*KThreesConfig)(nil), (*v1beta2.KThreesConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_KThreesConfig_To_v1beta2_KThreesConfig(a.(*KThreesConfig), b.(*v1beta2.KThreesConfig), scope) }); err != nil { @@ -161,6 +156,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta2.KThreesAgentConfig)(nil), (*KThreesAgentConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(a.(*v1beta2.KThreesAgentConfig), b.(*KThreesAgentConfig), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta2.KThreesServerConfig)(nil), (*KThreesServerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_KThreesServerConfig_To_v1beta1_KThreesServerConfig(a.(*v1beta2.KThreesServerConfig), b.(*KThreesServerConfig), scope) }); err != nil { @@ -247,14 +247,10 @@ func autoConvert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in *v1 out.KubeProxyArgs = *(*[]string)(unsafe.Pointer(&in.KubeProxyArgs)) out.NodeName = in.NodeName out.AirGapped = in.AirGapped + // WARNING: in.AirGappedInstallScriptPath requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig is an autogenerated conversion function. -func Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in *v1beta2.KThreesAgentConfig, out *KThreesAgentConfig, s conversion.Scope) error { - return autoConvert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in, out, s) -} - func autoConvert_v1beta1_KThreesConfig_To_v1beta2_KThreesConfig(in *KThreesConfig, out *v1beta2.KThreesConfig, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1beta1_KThreesConfigSpec_To_v1beta2_KThreesConfigSpec(&in.Spec, &out.Spec, s); err != nil { @@ -551,6 +547,7 @@ func autoConvert_v1beta2_KThreesServerConfig_To_v1beta1_KThreesServerConfig(in * // WARNING: in.DisableCloudController requires manual conversion: does not exist in peer-type // WARNING: in.CloudProviderName requires manual conversion: does not exist in peer-type // WARNING: in.SystemDefaultRegistry requires manual conversion: does not exist in peer-type + // WARNING: in.EtcdProxyImage requires manual conversion: does not exist in peer-type return nil } diff --git a/bootstrap/api/v1beta2/kthreesconfig_types.go b/bootstrap/api/v1beta2/kthreesconfig_types.go index 3611d660..9fb711d9 100644 --- a/bootstrap/api/v1beta2/kthreesconfig_types.go +++ b/bootstrap/api/v1beta2/kthreesconfig_types.go @@ -124,6 +124,10 @@ type KThreesServerConfig struct { // SystemDefaultRegistry defines private registry to be used for all system images // +optional SystemDefaultRegistry string `json:"systemDefaultRegistry,omitempty"` + + // Customized etcd proxy image for management cluster to communicate with workload cluster etcd (default: "alpine/socat") + // +optional + EtcdProxyImage string `json:"etcdProxyImage,omitempty"` } type KThreesAgentConfig struct { @@ -154,10 +158,16 @@ type KThreesAgentConfig struct { // AirGapped is a boolean value to define if the bootstrapping should be air-gapped, // basically supposing that online container registries and k3s install scripts are not reachable. - // User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh` + // User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh") // on all nodes in the air-gap environment. // +optional AirGapped bool `json:"airGapped,omitempty"` + + // AirGappedInstallScriptPath is the path to the install script in the air-gapped environment. + // The install script should be prepared by the user. The value is only + // used when AirGapped is set to true (default: "/opt/install.sh"). + // +optional + AirGappedInstallScriptPath string `json:"airGappedInstallScriptPath,omitempty"` } // KThreesConfigStatus defines the observed state of KThreesConfig. diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml index 05e5781f..1e6327b9 100644 --- a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml +++ b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml @@ -326,9 +326,15 @@ spec: description: |- AirGapped is a boolean value to define if the bootstrapping should be air-gapped, basically supposing that online container registries and k3s install scripts are not reachable. - User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh` + User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh") on all nodes in the air-gap environment. type: boolean + airGappedInstallScriptPath: + description: |- + AirGappedInstallScriptPath is the path to the install script in the air-gapped environment. + The install script should be prepared by the user. The value is only + used when AirGapped is set to true (default: "/opt/install.sh"). + type: string kubeProxyArgs: description: KubeProxyArgs Customized flag for kube-proxy process items: @@ -471,6 +477,10 @@ spec: the ''cloud-provider=external'' kubelet argument. (default: false)' type: boolean + etcdProxyImage: + description: 'Customized etcd proxy image for management cluster + to communicate with workload cluster etcd (default: "alpine/socat")' + type: string httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' type: string diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml index 90a3b64a..f01ba1d4 100644 --- a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml +++ b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml @@ -281,9 +281,15 @@ spec: description: |- AirGapped is a boolean value to define if the bootstrapping should be air-gapped, basically supposing that online container registries and k3s install scripts are not reachable. - User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh` + User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh") on all nodes in the air-gap environment. type: boolean + airGappedInstallScriptPath: + description: |- + AirGappedInstallScriptPath is the path to the install script in the air-gapped environment. + The install script should be prepared by the user. The value is only + used when AirGapped is set to true (default: "/opt/install.sh"). + type: string kubeProxyArgs: description: KubeProxyArgs Customized flag for kube-proxy process @@ -432,6 +438,11 @@ spec: the ''cloud-provider=external'' kubelet argument. (default: false)' type: boolean + etcdProxyImage: + description: 'Customized etcd proxy image for management + cluster to communicate with workload cluster etcd (default: + "alpine/socat")' + type: string httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' diff --git a/bootstrap/controllers/kthreesconfig_controller.go b/bootstrap/controllers/kthreesconfig_controller.go index deb33559..ebd5e2ee 100644 --- a/bootstrap/controllers/kthreesconfig_controller.go +++ b/bootstrap/controllers/kthreesconfig_controller.go @@ -17,9 +17,11 @@ limitations under the License. package controllers import ( + "bytes" "context" "errors" "fmt" + "html/template" "time" "github.com/go-logr/logr" @@ -249,23 +251,24 @@ func (r *KThreesConfigReconciler) joinControlplane(ctx context.Context, scope *S } if scope.Config.Spec.IsEtcdEmbedded() { - etcdProxyFile := bootstrapv1.File{ - Path: etcd.EtcdProxyDaemonsetYamlLocation, - Content: etcd.EtcdProxyDaemonsetYaml, - Owner: "root:root", - Permissions: "0640", + etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config) + if err != nil { + conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) + return fmt.Errorf("failed to resolve etcd proxy file: %w", err) } - files = append(files, etcdProxyFile) + + files = append(files, *etcdProxyFile) } cpInput := &cloudinit.ControlPlaneInput{ BaseUserData: cloudinit.BaseUserData{ - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - AdditionalFiles: files, - ConfigFile: workerConfigFile, - K3sVersion: scope.Config.Spec.Version, - AirGapped: scope.Config.Spec.AgentConfig.AirGapped, + PreK3sCommands: scope.Config.Spec.PreK3sCommands, + PostK3sCommands: scope.Config.Spec.PostK3sCommands, + AdditionalFiles: files, + ConfigFile: workerConfigFile, + K3sVersion: scope.Config.Spec.Version, + AirGapped: scope.Config.Spec.AgentConfig.AirGapped, + AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath, }, } @@ -320,12 +323,13 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope) winput := &cloudinit.WorkerInput{ BaseUserData: cloudinit.BaseUserData{ - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - AdditionalFiles: files, - ConfigFile: workerConfigFile, - K3sVersion: scope.Config.Spec.Version, - AirGapped: scope.Config.Spec.AgentConfig.AirGapped, + PreK3sCommands: scope.Config.Spec.PreK3sCommands, + PostK3sCommands: scope.Config.Spec.PostK3sCommands, + AdditionalFiles: files, + ConfigFile: workerConfigFile, + K3sVersion: scope.Config.Spec.Version, + AirGapped: scope.Config.Spec.AgentConfig.AirGapped, + AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath, }, } @@ -380,6 +384,38 @@ func (r *KThreesConfigReconciler) resolveSecretFileContent(ctx context.Context, return data, nil } +func (r *KThreesConfigReconciler) resolveEtcdProxyFile(cfg *bootstrapv1.KThreesConfig) (*bootstrapv1.File, error) { + // Parse the template + tpl, err := template.New("etcd-proxy").Parse(etcd.EtcdProxyDaemonsetYamlTemplate) + if err != nil { + return nil, fmt.Errorf("failed to parse etcd-proxy template: %w", err) + } + + // If user has set the systemDefaultRegistry, will prefix the image with it. + systemDefaultRegistry := cfg.Spec.ServerConfig.SystemDefaultRegistry + if systemDefaultRegistry != "" { + systemDefaultRegistry = fmt.Sprintf("%s/", systemDefaultRegistry) + } + + // Render the template, the image name will be ${EtcdProxyImage} if the user + // has set it, otherwise it will be ${SystemDefaultRegistry}alpine/socat + var buf bytes.Buffer + err = tpl.Execute(&buf, map[string]string{ + "EtcdProxyImage": cfg.Spec.ServerConfig.EtcdProxyImage, + "SystemDefaultRegistry": systemDefaultRegistry, + }) + if err != nil { + return nil, fmt.Errorf("failed to render etcd-proxy template: %w", err) + } + + return &bootstrapv1.File{ + Path: etcd.EtcdProxyDaemonsetYamlLocation, + Content: buf.String(), + Owner: "root:root", + Permissions: "0640", + }, nil +} + func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) { // initialize the DataSecretAvailableCondition if missing. // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing @@ -465,23 +501,23 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex } if scope.Config.Spec.IsEtcdEmbedded() { - etcdProxyFile := bootstrapv1.File{ - Path: etcd.EtcdProxyDaemonsetYamlLocation, - Content: etcd.EtcdProxyDaemonsetYaml, - Owner: "root:root", - Permissions: "0640", + etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config) + if err != nil { + conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) + return ctrl.Result{}, fmt.Errorf("failed to resolve etcd proxy file: %w", err) } - files = append(files, etcdProxyFile) + files = append(files, *etcdProxyFile) } cpinput := &cloudinit.ControlPlaneInput{ BaseUserData: cloudinit.BaseUserData{ - PreK3sCommands: scope.Config.Spec.PreK3sCommands, - PostK3sCommands: scope.Config.Spec.PostK3sCommands, - AdditionalFiles: files, - ConfigFile: initConfigFile, - K3sVersion: scope.Config.Spec.Version, - AirGapped: scope.Config.Spec.AgentConfig.AirGapped, + PreK3sCommands: scope.Config.Spec.PreK3sCommands, + PostK3sCommands: scope.Config.Spec.PostK3sCommands, + AdditionalFiles: files, + ConfigFile: initConfigFile, + K3sVersion: scope.Config.Spec.Version, + AirGapped: scope.Config.Spec.AgentConfig.AirGapped, + AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath, }, Certificates: certificates, } diff --git a/bootstrap/controllers/kthreesconfig_controller_test.go b/bootstrap/controllers/kthreesconfig_controller_test.go new file mode 100644 index 00000000..91286a9a --- /dev/null +++ b/bootstrap/controllers/kthreesconfig_controller_test.go @@ -0,0 +1,63 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "strings" + "testing" + + bootstrapv1 "github.com/k3s-io/cluster-api-k3s/bootstrap/api/v1beta2" +) + +func TestKThreesConfigReconciler_ResolveEtcdProxyFile(t *testing.T) { + // If EtcdProxyImage is set, it should override the system default registry + config := &bootstrapv1.KThreesConfig{ + Spec: bootstrapv1.KThreesConfigSpec{ + ServerConfig: bootstrapv1.KThreesServerConfig{ + EtcdProxyImage: "etcd-proxy-image", + SystemDefaultRegistry: "system-default-registry", + }, + }, + } + r := &KThreesConfigReconciler{} + etcdProxyFile, err := r.resolveEtcdProxyFile(config) + if err != nil { + t.Fatalf("failed to resolve etcd proxy file %v", err) + } + if !strings.Contains(etcdProxyFile.Content, "etcd-proxy-image") { + t.Fatalf("generated etcd proxy image should contain EtcdProxyImage") + } + if strings.Contains(etcdProxyFile.Content, "system-default-registry") { + t.Fatalf("system-default-registry should be overwritten by EtcdProxyImage") + } + + // If EtcdProxyImage is not set, the system default registry should be used + config2 := &bootstrapv1.KThreesConfig{ + Spec: bootstrapv1.KThreesConfigSpec{ + ServerConfig: bootstrapv1.KThreesServerConfig{ + SystemDefaultRegistry: "system-default-registry2", + }, + }, + } + etcdProxyFile, err = r.resolveEtcdProxyFile(config2) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(etcdProxyFile.Content, "system-default-registry2/") { + t.Fatalf("generated etcd proxy image should be prefixed with SystemDefaultRegistry") + } +} diff --git a/controlplane/api/v1beta1/conversion.go b/controlplane/api/v1beta1/conversion.go index eb0030ce..93a1ecce 100644 --- a/controlplane/api/v1beta1/conversion.go +++ b/controlplane/api/v1beta1/conversion.go @@ -94,9 +94,11 @@ func (in *KThreesControlPlane) ConvertTo(dstRaw ctrlconversion.Hub) error { dst.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider dst.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController = restored.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController dst.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry = restored.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry + dst.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage = restored.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage dst.Spec.MachineTemplate.NodeVolumeDetachTimeout = restored.Spec.MachineTemplate.NodeVolumeDetachTimeout dst.Spec.MachineTemplate.NodeDeletionTimeout = restored.Spec.MachineTemplate.NodeDeletionTimeout dst.Status.Version = restored.Status.Version + dst.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath return nil } diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml index 1b8e8dc9..a46809ee 100644 --- a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml +++ b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml @@ -629,9 +629,15 @@ spec: description: |- AirGapped is a boolean value to define if the bootstrapping should be air-gapped, basically supposing that online container registries and k3s install scripts are not reachable. - User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh` + User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh") on all nodes in the air-gap environment. type: boolean + airGappedInstallScriptPath: + description: |- + AirGappedInstallScriptPath is the path to the install script in the air-gapped environment. + The install script should be prepared by the user. The value is only + used when AirGapped is set to true (default: "/opt/install.sh"). + type: string kubeProxyArgs: description: KubeProxyArgs Customized flag for kube-proxy process @@ -778,6 +784,10 @@ spec: the ''cloud-provider=external'' kubelet argument. (default: false)' type: boolean + etcdProxyImage: + description: 'Customized etcd proxy image for management cluster + to communicate with workload cluster etcd (default: "alpine/socat")' + type: string httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanetemplates.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanetemplates.yaml index b9c3529e..d46ff91b 100644 --- a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanetemplates.yaml +++ b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanetemplates.yaml @@ -63,9 +63,15 @@ spec: description: |- AirGapped is a boolean value to define if the bootstrapping should be air-gapped, basically supposing that online container registries and k3s install scripts are not reachable. - User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh` + User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh") on all nodes in the air-gap environment. type: boolean + airGappedInstallScriptPath: + description: |- + AirGappedInstallScriptPath is the path to the install script in the air-gapped environment. + The install script should be prepared by the user. The value is only + used when AirGapped is set to true (default: "/opt/install.sh"). + type: string kubeProxyArgs: description: KubeProxyArgs Customized flag for kube-proxy process @@ -218,6 +224,11 @@ spec: suppresses the ''cloud-provider=external'' kubelet argument. (default: false)' type: boolean + etcdProxyImage: + description: 'Customized etcd proxy image for management + cluster to communicate with workload cluster etcd + (default: "alpine/socat")' + type: string httpsListenPort: description: 'HTTPSListenPort HTTPS listen port (default: 6443)' diff --git a/pkg/cloudinit/cloudinit.go b/pkg/cloudinit/cloudinit.go index 079c165e..e4f86472 100644 --- a/pkg/cloudinit/cloudinit.go +++ b/pkg/cloudinit/cloudinit.go @@ -69,20 +69,33 @@ write_files:{{ range . }} {{- end -}} {{- end -}} ` - sentinelFileCommand = "mkdir -p /run/cluster-api && echo success > /run/cluster-api/bootstrap-success.complete" + sentinelFileCommand = "mkdir -p /run/cluster-api && echo success > /run/cluster-api/bootstrap-success.complete" + defaultAirGappedInstallScriptPath = "/opt/install.sh" ) // BaseUserData is shared across all the various types of files written to disk. type BaseUserData struct { - Header string - PreK3sCommands []string - PostK3sCommands []string - AdditionalFiles []bootstrapv1.File - WriteFiles []bootstrapv1.File - ConfigFile bootstrapv1.File - K3sVersion string - AirGapped bool - SentinelFileCommand string + Header string + PreK3sCommands []string + PostK3sCommands []string + AdditionalFiles []bootstrapv1.File + WriteFiles []bootstrapv1.File + ConfigFile bootstrapv1.File + K3sVersion string + AirGapped bool + AirGappedInstallScriptPath string + SentinelFileCommand string +} + +func (input *BaseUserData) prepare() { + input.Header = cloudConfigHeader + input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) + input.WriteFiles = append(input.WriteFiles, input.ConfigFile) + if input.AirGappedInstallScriptPath == "" { + input.AirGappedInstallScriptPath = defaultAirGappedInstallScriptPath + } + + input.SentinelFileCommand = sentinelFileCommand } func generate(kind string, tpl string, data interface{}) ([]byte, error) { diff --git a/pkg/cloudinit/controlplane_init.go b/pkg/cloudinit/controlplane_init.go index 516ed697..c068d1d4 100644 --- a/pkg/cloudinit/controlplane_init.go +++ b/pkg/cloudinit/controlplane_init.go @@ -27,7 +27,7 @@ const ( {{template "files" .WriteFiles}} runcmd: {{- template "commands" .PreK3sCommands }} - - {{ if .AirGapped }} INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='server' sh /opt/install.sh {{ else }} curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s sh -s - server {{ end }} && {{ .SentinelFileCommand }} + - {{ if .AirGapped }} INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='server' sh {{ .AirGappedInstallScriptPath }} {{ else }} curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s sh -s - server {{ end }} && {{ .SentinelFileCommand }} {{- template "commands" .PostK3sCommands }} ` ) @@ -40,11 +40,8 @@ type ControlPlaneInput struct { // NewInitControlPlane returns the user data string to be used on a controlplane instance. func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) { - input.Header = cloudConfigHeader input.WriteFiles = input.Certificates.AsFiles() - input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - input.WriteFiles = append(input.WriteFiles, input.ConfigFile) - input.SentinelFileCommand = sentinelFileCommand + input.BaseUserData.prepare() controlPlaneCloudJoinWithVersion := fmt.Sprintf(controlPlaneCloudInit, input.K3sVersion) userData, err := generate("InitControlplane", controlPlaneCloudJoinWithVersion, input) diff --git a/pkg/cloudinit/controlplane_join.go b/pkg/cloudinit/controlplane_join.go index 28deeb3a..477ad6fc 100644 --- a/pkg/cloudinit/controlplane_join.go +++ b/pkg/cloudinit/controlplane_join.go @@ -20,11 +20,7 @@ import "fmt" // NewInitControlPlane returns the user data string to be used on a controlplane instance. func NewJoinControlPlane(input *ControlPlaneInput) ([]byte, error) { - input.Header = cloudConfigHeader - input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - input.WriteFiles = append(input.WriteFiles, input.ConfigFile) - input.SentinelFileCommand = sentinelFileCommand - + input.BaseUserData.prepare() // As controlPlaneCloudJoin template is the same as the controlPlaneCloudInit template, will reuse the controlPlaneCloudInit template controlPlaneCloudJoinWithVersion := fmt.Sprintf(controlPlaneCloudInit, input.K3sVersion) userData, err := generate("JoinControlplane", controlPlaneCloudJoinWithVersion, input) diff --git a/pkg/cloudinit/worker_join.go b/pkg/cloudinit/worker_join.go index 9821d7ca..7a282808 100644 --- a/pkg/cloudinit/worker_join.go +++ b/pkg/cloudinit/worker_join.go @@ -23,7 +23,7 @@ const ( {{template "files" .WriteFiles}} runcmd: {{- template "commands" .PreK3sCommands }} - - {{ if .AirGapped }} INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='agent' sh /opt/install.sh {{ else }} curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s sh -s - agent {{ end }} && {{ .SentinelFileCommand }} + - {{ if .AirGapped }} INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC='agent' sh {{ .AirGappedInstallScriptPath }}{{ else }} curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s sh -s - agent {{ end }} && {{ .SentinelFileCommand }} {{- template "commands" .PostK3sCommands }} ` ) @@ -35,10 +35,7 @@ type WorkerInput struct { // NewInitControlPlane returns the user data string to be used on a controlplane instance. func NewWorker(input *WorkerInput) ([]byte, error) { - input.Header = cloudConfigHeader - input.WriteFiles = append(input.WriteFiles, input.AdditionalFiles...) - input.WriteFiles = append(input.WriteFiles, input.ConfigFile) - input.SentinelFileCommand = sentinelFileCommand + input.BaseUserData.prepare() workerCloudInitWithVersion := fmt.Sprintf(workerCloudInit, input.K3sVersion) userData, err := generate("Worker", workerCloudInitWithVersion, input) diff --git a/pkg/cloudinit/worker_join_test.go b/pkg/cloudinit/worker_join_test.go new file mode 100644 index 00000000..7a199f31 --- /dev/null +++ b/pkg/cloudinit/worker_join_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudinit + +import ( + "testing" + + . "github.com/onsi/gomega" + + infrav1 "github.com/k3s-io/cluster-api-k3s/bootstrap/api/v1beta2" +) + +func TestWorkerJoin(t *testing.T) { + g := NewWithT(t) + + cpinput := &WorkerInput{ + BaseUserData: BaseUserData{ + PreK3sCommands: nil, + PostK3sCommands: nil, + AdditionalFiles: []infrav1.File{ + { + Path: "/tmp/my-path", + Encoding: infrav1.Base64, + Content: "aGk=", + }, + { + Path: "/tmp/my-other-path", + Content: "hi", + }, + }, + }, + } + + out, err := NewWorker(cpinput) + g.Expect(err).NotTo(HaveOccurred()) + t.Log(string(out)) +} + +func TestWorkerJoinAirGapped(t *testing.T) { + g := NewWithT(t) + + // Test setting the install script path with worker + workerInput := &WorkerInput{ + BaseUserData: BaseUserData{ + PreK3sCommands: nil, + PostK3sCommands: nil, + AdditionalFiles: []infrav1.File{ + { + Path: "/tmp/my-path", + Encoding: infrav1.Base64, + Content: "aGk=", + }, + { + Path: "/tmp/my-other-path", + Content: "hi", + }, + }, + AirGapped: true, + AirGappedInstallScriptPath: "/test/install.sh", + }, + } + out, err := NewWorker(workerInput) + g.Expect(err).NotTo(HaveOccurred()) + result := string(out) + g.Expect(result).To(ContainSubstring("sh /test/install.sh")) + g.Expect(result).NotTo(ContainSubstring("get.k3s.io")) +} diff --git a/pkg/etcd/etcd-proxy.go b/pkg/etcd/etcd-proxy.go index 67c096b6..31a5742b 100644 --- a/pkg/etcd/etcd-proxy.go +++ b/pkg/etcd/etcd-proxy.go @@ -7,4 +7,4 @@ import ( const EtcdProxyDaemonsetYamlLocation = "/var/lib/rancher/k3s/server/manifests/etcd-proxy.yaml" //go:embed etcd-proxy.yaml -var EtcdProxyDaemonsetYaml string +var EtcdProxyDaemonsetYamlTemplate string diff --git a/pkg/etcd/etcd-proxy.yaml b/pkg/etcd/etcd-proxy.yaml index 00fa4ae9..89fdb0d3 100644 --- a/pkg/etcd/etcd-proxy.yaml +++ b/pkg/etcd/etcd-proxy.yaml @@ -25,7 +25,11 @@ spec: effect: NoSchedule containers: - name: etcd-proxy - image: alpine/socat + {{ if ne .EtcdProxyImage "" }} + image: {{ .EtcdProxyImage }} + {{ else }} + image: {{ .SystemDefaultRegistry }}alpine/socat:1.8.0.0 + {{ end }} env: - name: HOSTIP valueFrom: diff --git a/samples/docker/air-gapped/Dockerfile b/samples/docker/air-gapped/Dockerfile index 9341f0b8..27552b0d 100644 --- a/samples/docker/air-gapped/Dockerfile +++ b/samples/docker/air-gapped/Dockerfile @@ -1,14 +1,18 @@ FROM kindest/node:v1.28.0 ARG K3S_VERSION=v1.28.6+k3s2 +ARG INSTALL_SCRIPT_PATH=/opt/install.sh +ARG ALPINE_SOCAT_TARBALL_PATH=./alpine-socat.tar # Load docker images # Note that the flow follows the manually deploy image steps, but private registry method should also be supported RUN mkdir -p /var/lib/rancher/k3s/agent/images/ RUN curl -L -o /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar.zst "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s-airgap-images-amd64.tar.zst" +# Copy alpine-socat.tar to /var/lib/rancher/k3s/agent/images/ +COPY $ALPINE_SOCAT_TARBALL_PATH /var/lib/rancher/k3s/agent/images/ -# Download install script to /opt/install.sh -RUN curl -L -o /opt/install.sh https://get.k3s.io -RUN chmod +x /opt/install.sh +# Download install script to $INSTALL_SCRIPT_PATH +RUN curl -L -o $INSTALL_SCRIPT_PATH https://get.k3s.io +RUN chmod +x $INSTALL_SCRIPT_PATH # Download k3s binary RUN curl -L -o /usr/local/bin/k3s "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s" diff --git a/samples/docker/air-gapped/README.md b/samples/docker/air-gapped/README.md index d7c306e8..d39cc588 100644 --- a/samples/docker/air-gapped/README.md +++ b/samples/docker/air-gapped/README.md @@ -4,15 +4,27 @@ K3s is supporting air-gapped installations. This sample demonstrates how to crea It will first build a kind node docker image with the K3s binary, the required images and scripts, following [k3s Air-Gap Install](https://docs.k3s.io/installation/airgap). Then it will create a K3s cluster with this kind node image. +K3s CAPI depends on the `alpine/socat` image to communicate with the etcd server on the node. A daemonset is created to deploy the `alpine/socat` image to all the etcd nodes. You should also download the `alpine/socat` image in the air-gapped node. + ```shell export AIRGAPPED_KIND_IMAGE=kindnode:airgapped +export AIRGAPPED_INSTALL_SCRIPT_PATH=/k3s-airgapped-install.sh export CLUSTER_NAME=k3s-airgapped +export NAMESPACE=default export CONTROL_PLANE_MACHINE_COUNT=1 export KUBERNETES_VERSION=v1.28.6+k3s2 export WORKER_MACHINE_COUNT=3 +export AIRGAPPED_ALPINE_SOCAT_IMAGE=alpine/socat:1.8.0.0 + +# Prepare the alpine/socat image +docker pull $AIRGAPPED_ALPINE_SOCAT_IMAGE +docker save $AIRGAPPED_ALPINE_SOCAT_IMAGE -o alpine-socat.tar # Build the kind node image -docker build -t $AIRGAPPED_KIND_IMAGE . --build-arg="K3S_VERSION=$KUBERNETES_VERSION" +docker build -t $AIRGAPPED_KIND_IMAGE . \ +--build-arg="K3S_VERSION=$KUBERNETES_VERSION" \ +--build-arg="INSTALL_SCRIPT_PATH=$AIRGAPPED_INSTALL_SCRIPT_PATH" \ +--build-arg="ALPINE_SOCAT_TARBALL_PATH=./alpine-socat.tar" # Generate the cluster yaml # Note that `airGapped` is set to true in `agentConfig` diff --git a/samples/docker/air-gapped/k3s-template.yaml b/samples/docker/air-gapped/k3s-template.yaml index a8893d76..39a71464 100644 --- a/samples/docker/air-gapped/k3s-template.yaml +++ b/samples/docker/air-gapped/k3s-template.yaml @@ -1,18 +1,19 @@ apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster +kind: Cluster metadata: - name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} spec: clusterNetwork: pods: cidrBlocks: - 10.45.0.0/16 + serviceDomain: cluster.local services: cidrBlocks: - 10.46.0.0/16 - serviceDomain: cluster.local controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: KThreesControlPlane name: ${CLUSTER_NAME}-control-plane infrastructureRef: @@ -20,76 +21,91 @@ spec: kind: DockerCluster name: ${CLUSTER_NAME} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: DockerCluster +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment metadata: - name: ${CLUSTER_NAME} -spec: {} + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: ${CLUSTER_NAME}-md-0 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KThreesConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} --- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: KThreesControlPlane metadata: name: ${CLUSTER_NAME}-control-plane - namespace: default + namespace: ${NAMESPACE} spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-control-plane kthreesConfigSpec: + serverConfig: + tlsSan: + - 0.0.0.0 + etcdProxyImage: ${AIRGAPPED_ALPINE_SOCAT_IMAGE} agentConfig: airGapped: true + airGappedInstallScriptPath: ${AIRGAPPED_INSTALL_SCRIPT_PATH} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: ${CLUSTER_NAME} +spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate metadata: name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} spec: template: spec: customImage: ${AIRGAPPED_KIND_IMAGE} --- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: worker-md-0 -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} - template: - spec: - version: ${KUBERNETES_VERSION} - clusterName: ${CLUSTER_NAME} - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KThreesConfigTemplate - name: ${CLUSTER_NAME}-md-0 - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - name: ${CLUSTER_NAME}-md-0 ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate metadata: name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} spec: template: spec: customImage: ${AIRGAPPED_KIND_IMAGE} --- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: KThreesConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} spec: template: spec: agentConfig: - airGapped: true \ No newline at end of file + airGapped: true + airGappedInstallScriptPath: ${AIRGAPPED_INSTALL_SCRIPT_PATH} \ No newline at end of file