Fixed e2e (#53)

- Add support for arm64
- Separate e2e for agent initiated scenario
- Update cluster CIDR to not overlap with the host network
- Collect cluster logs via crust gather
- Allow justfile to use pre-installed binaries from the env, making nix environment useful for starters.
- Update generated API: a diff in the CRDs in main.

Signed-off-by: Danil-Grigorev <danil.grigorev@suse.com>

.github/workflows/ci.yaml

@@ -3,6 +3,7 @@ name: Run CI checks
 on:
   pull_request:
     types: [opened, edited, synchronize, reopened, labeled, unlabeled]
+  merge_group:
   workflow_dispatch:
 
 jobs:
@@ -34,3 +35,12 @@ jobs:
       - uses: actions/checkout@v4
       - name: Test
         run: just test-cluster-class-import
+      - name: Collect artifacts
+        if: always()
+        run: just collect-test-cluster-class-import
+      - name: Store run artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: artifacts
+          path: _out/gather

justfile

@@ -5,14 +5,13 @@ TAG := "dev"
 HOME_DIR := env_var('HOME')
 YQ_VERSION := "v4.43.1"
 CLUSTERCTL_VERSION := "v1.7.1"
-YQ_BIN := "_out/yq"
 OUT_DIR := "_out"
-KOPIUM_BIN := "_out/bin/kopium"
 KUSTOMIZE_VERSION := "v5.4.1"
-KUSTOMIZE_BIN := "_out/kustomize"
-CLUSTERCTL_BIN := "_out/clusterctl"
 ARCH := if arch() == "aarch64" { "arm64"} else { "amd64" }
 DIST := os()
+REFRESH_BIN := env_var_or_default('REFRESH_BIN', '1')
+
+export PATH := "_out:_out/bin:" + env_var('PATH')
 
 [private]
 default:
@@ -25,15 +24,15 @@ generate features="":
 
 # generates files for CRDS
 generate-crds: _create-out-dir _install-kopium _download-yq
-    just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusters.yaml" "src/api/capi_cluster.rs" ""
-    just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusterclasses.yaml" "src/api/capi_clusterclass.rs" ""
-    just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster.rs" "select(.spec.names.singular==\"cluster\")" "--no-condition"
-    just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_clustergroup.rs" "select(.spec.names.singular==\"clustergroup\")" "--no-condition"
-    just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster_registration_token.rs" "select(.spec.names.singular==\"clusterregistrationtoken\")" ""
+    just _generate-kopium-url kopium "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusters.yaml" "src/api/capi_cluster.rs" ""
+    just _generate-kopium-url kopium "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusterclasses.yaml" "src/api/capi_clusterclass.rs" ""
+    just _generate-kopium-url kopium "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster.rs" "select(.spec.names.singular==\"cluster\")" "--no-condition"
+    just _generate-kopium-url kopium "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_clustergroup.rs" "select(.spec.names.singular==\"clustergroup\")" "--no-condition"
+    just _generate-kopium-url kopium "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster_registration_token.rs" "select(.spec.names.singular==\"clusterregistrationtoken\")" ""
 
 [private]
 _generate-kopium-url kpath="" source="" dest="" yqexp="." condition="":
-    curl -sSL {{source}} | {{YQ_BIN}} '{{yqexp}}' | {{kpath}} -D Default {{condition}} -f - > {{dest}}
+    curl -sSL {{source}} | yq '{{yqexp}}' | {{kpath}} -D Default {{condition}} -f - > {{dest}}
 
 generate-addon-crds features="":
     cargo run --features={{features}} --bin crdgen > config/crds/fleet-addon-config.yaml
@@ -63,7 +62,7 @@ compile features="":  _create-out-dir
     -w /volume \
     -t clux/muslrust:stable \
     cargo build --release --features={{features}} --bin controller
-  cp target/x86_64-unknown-linux-musl/release/controller _out/controller
+  cp target/{{arch()}}-unknown-linux-musl/release/controller {{OUT_DIR}}/controller
 
 [private]
 _build features="":
@@ -91,12 +90,10 @@ load-base features="":
 start-dev: _cleanup-out-dir _create-out-dir
     just update-helm-repos
     kind delete cluster --name dev || true
-    export LOCAL_IP=$(ip -4 -j route list default | jq -r .[0].prefsrc)
-    envsubst < testdata/kind-config.yaml > _out/kind-config.yaml
-    kind create cluster --config --image=kindest/node:v{{KUBE_VERSION}} --config _out/kind-config.yaml
+    kind create cluster --config --image=kindest/node:v{{KUBE_VERSION}} --config testdata/kind-config.yaml
     just install-fleet
     just install-capi
-    kubectl wait pods --for=condition=Ready --timeout=300s --all --all-namespaces
+    kubectl wait pods --for=condition=Ready --timeout=150s --all --all-namespaces
 
 # Stop the local dev environment
 stop-dev:
@@ -112,6 +109,7 @@ deploy-child-cluster:
 
 # Deploy child cluster-call based cluster using docker & kubeadm
 deploy-child-cluster-class:
+    kind delete cluster --name capi-quickstart || true
     kubectl --context kind-dev apply -f testdata/capi-quickstart.yaml
 
 # Add and update helm repos used
@@ -127,86 +125,139 @@ update-helm-repos:
 install-fleet: _create-out-dir
     #!/usr/bin/env bash
     set -euxo pipefail
-    kubectl config view -o json --raw | jq -r '.clusters[].cluster["certificate-authority-data"]' | base64 -d > _out/ca.pem
-    API_SERVER_URL=`kubectl config view -o json --raw | jq -r '.clusters[] | select(.name=="kind-dev").cluster["server"]'`
+    kubectl config view -o json --raw | jq -r '.clusters[].cluster["certificate-authority-data"]' | base64 -d > {{OUT_DIR}}/ca.pem
+    API_SERVER_URL=`kubectl get nodes -o json | jq -r '.items[0].status.addresses[] | select(.type=="InternalIP").address'`
+    API_SERVER_URL=https://${API_SERVER_URL}:6443
     helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd
-    helm install --create-namespace -n cattle-fleet-system --set apiServerURL=$API_SERVER_URL --set-file apiServerCA=_out/ca.pem fleet fleet/fleet --wait
+    helm install --create-namespace -n cattle-fleet-system --set apiServerURL=$API_SERVER_URL --set-file apiServerCA={{OUT_DIR}}/ca.pem fleet fleet/fleet --wait
 
 # Install cluster api and any providers
 install-capi: _download-clusterctl
-    EXP_CLUSTER_RESOURCE_SET=true CLUSTER_TOPOLOGY=true {{CLUSTERCTL_BIN}} init -i docker
+    EXP_CLUSTER_RESOURCE_SET=true CLUSTER_TOPOLOGY=true clusterctl init -i docker
 
 # Deploy will deploy the operator
 deploy features="": _download-kustomize
     just generate {{features}}
     just load-base {{features}}
-    {{KUSTOMIZE_BIN}} build config/default | kubectl apply -f -
+    kustomize build config/default | kubectl apply -f -
 
 undeploy: _download-kustomize
-    {{KUSTOMIZE_BIN}} build config/default | kubectl delete --ignore-not-found=true -f -
+    kustomize build config/default | kubectl delete --ignore-not-found=true -f -
 
 release-manifests: _create-out-dir _download-kustomize
-    {{KUSTOMIZE_BIN}} build config/default > _out/addon-components.yaml
+    kustomize build config/default > {{OUT_DIR}}/addon-components.yaml
 
 # Full e2e test of importing cluster in fleet
-test-import: start-dev deploy deploy-child-cluster deploy-crs
-    kubectl wait pods --for=condition=Ready --timeout=300s --all --all-namespaces
-    kubectl wait clusters.fleet.cattle.io --timeout=300s --for=condition=Ready docker-demo
+test-import: start-dev deploy deploy-child-cluster deploy-crs && collect-test-import
+    kubectl wait pods --for=condition=Ready --timeout=150s --all --all-namespaces
+    kubectl wait cluster --timeout=500s --for=condition=ControlPlaneReady=true docker-demo
+    kubectl wait clusters.fleet.cattle.io --timeout=150s --for=condition=Ready docker-demo
+
+collect-test-import:
+    -just collect-artifacts dev
+    -just collect-artifacts docker-demo
 
 # Full e2e test of importing cluster in fleet
-test-cluster-class-import: start-dev
+test-cluster-class-import: start-dev deploy _test-import-all && collect-test-cluster-class-import
+
+collect-test-cluster-class-import:
+    -just collect-artifacts dev
+    -just collect-artifacts capi-quickstart
+
+# Test e2e with agent initiated connection procedure
+test-cluster-class-import-agent-initated: start-dev && collect-test-cluster-class-import
     just deploy "agent-initiated"
+    just _test-import-all
+
+collect-artifacts cluster:
+    kind get kubeconfig --name {{cluster}} > {{OUT_DIR}}/kubeconfig
+    just crust-gather collect -f {{OUT_DIR}}/gather/{{cluster}} -k {{OUT_DIR}}/kubeconfig
+
+# Full e2e test of importing cluster and ClusterClass in fleet
+[private]
+_test-import-all:
     just deploy-child-cluster-class
     just deploy-crs
-    kubectl wait pods --for=condition=Ready --timeout=300s --all --all-namespaces
-    kubectl wait clustergroups.fleet.cattle.io --timeout=300s --for=jsonpath='{.status.clusterCount}=1' quick-start
-    kubectl wait clustergroups.fleet.cattle.io --timeout=300s --for=condition=Ready quick-start
-    kubectl wait clusters.fleet.cattle.io --timeout=300s --for=condition=Ready capi-quickstart
+    kubectl wait pods --for=condition=Ready --timeout=150s --all --all-namespaces
+    kubectl wait clustergroups.fleet.cattle.io --timeout=150s --for=jsonpath='{.status.clusterCount}=1' quick-start
+    kubectl wait clustergroups.fleet.cattle.io --timeout=150s --for=condition=Ready=true quick-start
+    kubectl wait clusters.fleet.cattle.io --timeout=150s --for=condition=Ready=false capi-quickstart
+    kubectl wait clusters.fleet.cattle.io --timeout=150s --for=condition=Ready=true capi-quickstart
 
 # Install kopium
 [private]
 _install-kopium:
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    [ -z `which kopium` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0
     cargo install --git https://github.com/kube-rs/kopium.git --tag 0.19.0 --root {{OUT_DIR}}
 
+download-kustomize: _download-kustomize
+
 # Download kustomize
 [private]
 [linux]
 [macos]
 _download-kustomize:
-    curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/{{KUSTOMIZE_VERSION}}/kustomize_{{KUSTOMIZE_VERSION}}_{{DIST}}_{{ARCH}}.tar.gz -o {{KUSTOMIZE_BIN}}.tar.gz
-    tar -xzf {{KUSTOMIZE_BIN}}.tar.gz -C _out
-    chmod +x {{KUSTOMIZE_BIN}}
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    [ -z `which kustomize` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0
+    curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/{{KUSTOMIZE_VERSION}}/kustomize_{{KUSTOMIZE_VERSION}}_{{DIST}}_{{ARCH}}.tar.gz -o {{OUT_DIR}}/kustomize.tar.gz
+    tar -xzf {{OUT_DIR}}/kustomize.tar.gz -C {{OUT_DIR}}
+    chmod +x {{OUT_DIR}}/kustomize
 
 [private]
 [linux]
 _download-clusterctl:
-    curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-linux-amd64 -o {{CLUSTERCTL_BIN}}
-    chmod +x {{CLUSTERCTL_BIN}}
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    [ -z `which clusterctl` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0
+    curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-linux-{{ARCH}} -o {{OUT_DIR}}/clusterctl
+    chmod +x {{OUT_DIR}}/clusterctl
 
 [private]
 [macos]
 _download-clusterctl:
-    curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-darwin-amd64 -o {{CLUSTERCTL_BIN}}
-    chmod +x {{CLUSTERCTL_BIN}}
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    [ -z `which clusterctl` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0
+    curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-darwin-{{ARCH}} -o {{OUT_DIR}}/clusterctl
+    chmod +x {{OUT_DIR}}/clusterctl
 
 # Download yq
 [private]
 [linux]
 _download-yq:
-    curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_linux_{{ARCH}} -o {{YQ_BIN}}
-    chmod +x {{YQ_BIN}}
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    [ -z `which yq` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0
+    curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_linux_{{ARCH}} -o {{OUT_DIR}}/yq
+    chmod +x {{OUT_DIR}}/yq
 
 [private]
 [macos]
 _download-yq:
-    curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_darwin_{{ARCH}} -o {{YQ_BIN}}
-    chmod +x {{YQ_BIN}}
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    [ -z `which yq` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0
+    curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_darwin_{{ARCH}} -o {{OUT_DIR}}/yq
+    chmod +x {{OUT_DIR}}/yq
 
 [private]
 _create-out-dir:
-    mkdir -p _out
+    mkdir -p {{OUT_DIR}}
 
 [private]
 _cleanup-out-dir:
-    rm -rf _out/ || true
+    rm -rf {{OUT_DIR}} || true
+
+crust-gather *flags: _download-crust-gather
+    crust-gather {{flags}}
+
+[private]
+_download-crust-gather: _create-out-dir
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    [ -z `which crust-gather` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0
+    curl -sSfL https://github.com/crust-gather/crust-gather/raw/main/install.sh | sh -s - -f -b {{OUT_DIR}}
 

shell.nix

@@ -8,15 +8,14 @@ pkgs.mkShell {
     rustfmt       # Formatter
     clippy        # Linter
     just
+    fzf
     kind
     kubernetes-helm
     clusterctl
     kubectl
     k9s
     jq
     yq
-    envsubst
-    iproute2
     docker-client
     kustomize
   ];

src/api/capi_cluster.rs

@@ -127,6 +127,8 @@ pub struct ClusterTopologyControlPlane {
     pub node_volume_detach_timeout: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub replicas: Option<i32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub variables: Option<ClusterTopologyControlPlaneVariables>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
@@ -179,6 +181,20 @@ pub struct ClusterTopologyControlPlaneMetadata {
     pub labels: Option<BTreeMap<String, String>>,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, Default)]
+pub struct ClusterTopologyControlPlaneVariables {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub overrides: Option<Vec<ClusterTopologyControlPlaneVariablesOverrides>>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default)]
+pub struct ClusterTopologyControlPlaneVariablesOverrides {
+    #[serde(default, skip_serializing_if = "Option::is_none", rename = "definitionFrom")]
+    pub definition_from: Option<String>,
+    pub name: String,
+    pub value: serde_json::Value,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, Default)]
 pub struct ClusterTopologyVariables {
     #[serde(default, skip_serializing_if = "Option::is_none", rename = "definitionFrom")]

testdata/capi-quickstart.yaml

@@ -316,11 +316,11 @@ spec:
   clusterNetwork:
     pods:
       cidrBlocks:
-      - 192.168.0.0/16
+      - 10.1.0.0/16
     serviceDomain: k8s.test
     services:
       cidrBlocks:
-      - 10.96.0.0/12
+      - 10.10.0.0/16
   topology:
     class: quick-start
     controlPlane:

testdata/kind-config.yaml

@@ -5,6 +5,4 @@ nodes:
 - role: control-plane
   extraMounts:
     - hostPath: /var/run/docker.sock
-      containerPath: /var/run/docker.sock
-networking:
-  apiServerAddress: "$LOCAL_IP"
+      containerPath: /var/run/docker.sock