From 72fa8af4d6498d2795a3940d82aac2b47a3563c8 Mon Sep 17 00:00:00 2001 From: Dan Norris Date: Sat, 23 Mar 2024 19:17:57 +0100 Subject: [PATCH] feat: add option to start hosts as a DaemonSet Add a `daemonset` option for managing hosts as a DaemonSet instead of just a Deployment. Signed-off-by: Dan Norris --- Cargo.lock | 4 +- Cargo.toml | 2 +- README.md | 8 +- crates/types/Cargo.toml | 2 +- .../src/v1alpha1/wasmcloud_host_config.rs | 7 + deploy/base/deployment.yaml | 1 + sample.yaml | 2 + src/controller.rs | 163 ++++++++++++------ 8 files changed, 128 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 639c658..0347ff2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4302,7 +4302,7 @@ dependencies = [ [[package]] name = "wasmcloud-operator" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "async-nats", @@ -4338,7 +4338,7 @@ dependencies = [ [[package]] name = "wasmcloud-operator-types" -version = "0.1.0" +version = "0.1.1" dependencies = [ "k8s-openapi", "kube", diff --git a/Cargo.toml b/Cargo.toml index 7acc36a..67927f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wasmcloud-operator" -version = "0.1.0" +version = "0.1.1" edition = "2021" [[bin]] diff --git a/README.md b/README.md index 910200f..33fbc41 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,9 @@ spec: hostReplicas: 2 # The cluster issuers to use for each host issuers: - - CDKF6OKPOBQKAX57UOXO7SCHURTOZWKWIVPC2HFJTGFXY5VJX44ECEHH + # This needs to be the public key of a cluster seed generated by + # `wash keys gen cluster` + - CDK... # The lattice to connect the hosts to lattice: 83a5b52e-17cf-4080-bac8-f844099f142e # Additional labels to apply to the host other than the defaults set in the operator @@ -29,6 +31,8 @@ spec: version: 0.81.0 # The name of a secret in the same namespace that provides the required secrets. secretName: cluster-secrets + # Enable the following to run the wasmCloud hosts as a DaemonSet + #daemonset: true ``` The CRD requires a Kubernetes Secret with the following keys: @@ -40,7 +44,7 @@ metadata: name: my-wasmcloud-cluster data: # You can generate this with wash: - # wash keys gen cluster + # `wash keys gen cluster` WASMCLOUD_CLUSTER_SEED: # Only required if using a NATS creds file # nats.creds: diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index fdd974b..f09f073 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wasmcloud-operator-types" -version = "0.1.0" +version = "0.1.1" edition = "2021" [dependencies] diff --git a/crates/types/src/v1alpha1/wasmcloud_host_config.rs b/crates/types/src/v1alpha1/wasmcloud_host_config.rs index 8eeac7d..b9d7253 100644 --- a/crates/types/src/v1alpha1/wasmcloud_host_config.rs +++ b/crates/types/src/v1alpha1/wasmcloud_host_config.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; )] #[serde(rename_all = "camelCase")] pub struct WasmCloudHostConfigSpec { + #[serde(default = "default_host_replicas")] pub host_replicas: u32, pub issuers: Vec, pub lattice: String, @@ -37,6 +38,12 @@ pub struct WasmCloudHostConfigSpec { pub jetstream_domain: String, #[serde(default = "default_log_level")] pub log_level: String, + #[serde(default)] + pub daemonset: bool, +} + +fn default_host_replicas() -> u32 { + 1 } fn default_jetstream_domain() -> String { diff --git a/deploy/base/deployment.yaml b/deploy/base/deployment.yaml index 0e4e477..9388c9c 100644 --- a/deploy/base/deployment.yaml +++ b/deploy/base/deployment.yaml @@ -61,6 +61,7 @@ rules: - apps resources: - deployments + - daemonsets verbs: - get - list diff --git a/sample.yaml b/sample.yaml index da4fd67..3cae419 100644 --- a/sample.yaml +++ b/sample.yaml @@ -18,3 +18,5 @@ spec: secretName: cluster-secrets logLevel: INFO natsAddress: nats://nats-cluster.default.svc.cluster.local + # Enable the following to run the wasmCloud hosts as a DaemonSet + #daemonset: true diff --git a/src/controller.rs b/src/controller.rs index 0792c25..c11a005 100644 --- a/src/controller.rs +++ b/src/controller.rs @@ -3,7 +3,7 @@ use crate::{Error, Result}; use anyhow::bail; use futures::StreamExt; use handlebars::Handlebars; -use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec}; +use k8s_openapi::api::apps::v1::{DaemonSet, DaemonSetSpec, Deployment, DeploymentSpec}; use k8s_openapi::api::core::v1::{ ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EnvVar, EnvVarSource, ExecAction, Lifecycle, LifecycleHandler, PodSpec, PodTemplateSpec, Secret, SecretKeySelector, @@ -148,7 +148,7 @@ async fn reconcile_crd(config: &WasmCloudHostConfig, ctx: Arc) -> Resul return Err(e); }; - if let Err(e) = configure_deployment(&cfg, ctx.clone()).await { + if let Err(e) = configure_hosts(&cfg, ctx.clone()).await { warn!("Failed to configure deployment: {}", e); return Err(e); }; @@ -210,14 +210,10 @@ async fn cleanup(config: &WasmCloudHostConfig, ctx: Arc) -> Result) -> DeploymentSpec { +fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc) -> PodTemplateSpec { let mut labels = BTreeMap::new(); labels.insert("app.kubernetes.io/name".to_string(), config.name_any()); labels.insert("app.kubernetes.io/instance".to_string(), config.name_any()); - let ls = LabelSelector { - match_labels: Some(labels.clone()), - ..Default::default() - }; let mut wasmcloud_env = vec![ EnvVar { @@ -375,45 +371,74 @@ fn deployment_spec(config: &WasmCloudHostConfig, _ctx: Arc) -> Deployme ..Default::default() }, ]; + PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(labels), + ..Default::default() + }), + spec: Some(PodSpec { + service_account: Some(config.name_any()), + containers, + volumes: Some(vec![ + Volume { + name: "nats-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(config.name_any()), + ..Default::default() + }), + ..Default::default() + }, + Volume { + name: "nats-creds".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(config.spec.secret_name.clone()), + ..Default::default() + }), + ..Default::default() + }, + ]), + ..Default::default() + }), + } +} + +fn deployment_spec(config: &WasmCloudHostConfig, ctx: Arc) -> DeploymentSpec { + let pod_template = pod_template(config, ctx); + let mut labels = BTreeMap::new(); + labels.insert("app.kubernetes.io/name".to_string(), config.name_any()); + labels.insert("app.kubernetes.io/instance".to_string(), config.name_any()); + let ls = LabelSelector { + match_labels: Some(labels.clone()), + ..Default::default() + }; + DeploymentSpec { replicas: Some(config.spec.host_replicas as i32), selector: ls, - template: PodTemplateSpec { - metadata: Some(ObjectMeta { - labels: Some(labels), - ..Default::default() - }), - spec: Some(PodSpec { - service_account: Some(config.name_any()), - containers, - volumes: Some(vec![ - Volume { - name: "nats-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: Some(config.name_any()), - ..Default::default() - }), - ..Default::default() - }, - Volume { - name: "nats-creds".to_string(), - secret: Some(SecretVolumeSource { - secret_name: Some(config.spec.secret_name.clone()), - ..Default::default() - }), - ..Default::default() - }, - ]), - ..Default::default() - }), - }, + template: pod_template, ..Default::default() } } -async fn configure_deployment(config: &WasmCloudHostConfig, ctx: Arc) -> Result<()> { - let mut spec = deployment_spec(config, ctx.clone()); +fn daemonset_spec(config: &WasmCloudHostConfig, ctx: Arc) -> DaemonSetSpec { + let pod_template = pod_template(config, ctx); + let mut labels = BTreeMap::new(); + labels.insert("app.kubernetes.io/name".to_string(), config.name_any()); + labels.insert("app.kubernetes.io/instance".to_string(), config.name_any()); + let ls = LabelSelector { + match_labels: Some(labels.clone()), + ..Default::default() + }; + + DaemonSetSpec { + selector: ls, + template: pod_template, + ..Default::default() + } +} +async fn configure_hosts(config: &WasmCloudHostConfig, ctx: Arc) -> Result<()> { + let mut env_vars = vec![]; if let Some(registry_credentials) = &config.spec.registry_credentials_secret { let secrets_client = Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); @@ -443,7 +468,7 @@ async fn configure_deployment(config: &WasmCloudHostConfig, ctx: Arc) - ))); } - let mut env_vars = vec![ + env_vars = vec![ EnvVar { name: "OCI_REGISTRY_USER".to_string(), value: Some( @@ -477,32 +502,60 @@ async fn configure_deployment(config: &WasmCloudHostConfig, ctx: Arc) - ..Default::default() }, ]; + } + if config.spec.daemonset { + let mut spec = daemonset_spec(config, ctx.clone()); spec.template.spec.as_mut().unwrap().containers[1] .env .as_mut() .unwrap() .append(&mut env_vars); - } + let ds = DaemonSet { + metadata: ObjectMeta { + name: Some(config.name_any()), + namespace: Some(config.namespace().unwrap()), + owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]), + ..Default::default() + }, + spec: Some(spec), + ..Default::default() + }; - let deployment = Deployment { - metadata: ObjectMeta { - name: Some(config.name_any()), - namespace: Some(config.namespace().unwrap()), - owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]), + let api = Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); + api.patch( + &config.name_any(), + &PatchParams::apply(CLUSTER_CONFIG_FINALIZER), + &Patch::Apply(ds), + ) + .await?; + } else { + let mut spec = deployment_spec(config, ctx.clone()); + spec.template.spec.as_mut().unwrap().containers[1] + .env + .as_mut() + .unwrap() + .append(&mut env_vars); + let deployment = Deployment { + metadata: ObjectMeta { + name: Some(config.name_any()), + namespace: Some(config.namespace().unwrap()), + owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]), + ..Default::default() + }, + spec: Some(spec), ..Default::default() - }, - spec: Some(spec), - ..Default::default() + }; + + let api = Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); + api.patch( + &config.name_any(), + &PatchParams::apply(CLUSTER_CONFIG_FINALIZER), + &Patch::Apply(deployment), + ) + .await?; }; - let api = Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); - api.patch( - &config.name_any(), - &PatchParams::apply(CLUSTER_CONFIG_FINALIZER), - &Patch::Apply(deployment), - ) - .await?; Ok(()) }