From bf38c37b73783d89b6a7b089491162b1b440568e Mon Sep 17 00:00:00 2001 From: Danil Grigorev Date: Mon, 3 Jun 2024 16:45:52 +0200 Subject: [PATCH] Fixed e2e (#53) - Add support for arm64 - Separate e2e for agent initiated scenario - Update cluster CIDR to not overlap with the host network - Collect cluster logs via crust gather - Allow justfile to use pre-installed binaries from the env, making nix environment useful for starters. - Update generated API: a diff in the CRDs in main. Signed-off-by: Danil-Grigorev --- .github/workflows/ci.yaml | 10 +++ justfile | 137 +++++++++++++++++++++++----------- shell.nix | 3 +- src/api/capi_cluster.rs | 16 ++++ testdata/capi-quickstart.yaml | 4 +- testdata/kind-config.yaml | 4 +- 6 files changed, 124 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 449d295..04a6606 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,6 +3,7 @@ name: Run CI checks on: pull_request: types: [opened, edited, synchronize, reopened, labeled, unlabeled] + merge_group: workflow_dispatch: jobs: @@ -34,3 +35,12 @@ jobs: - uses: actions/checkout@v4 - name: Test run: just test-cluster-class-import + - name: Collect artifacts + if: always() + run: just collect-test-cluster-class-import + - name: Store run artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: artifacts + path: _out/gather diff --git a/justfile b/justfile index d558661..c285fd5 100644 --- a/justfile +++ b/justfile @@ -5,14 +5,13 @@ TAG := "dev" HOME_DIR := env_var('HOME') YQ_VERSION := "v4.43.1" CLUSTERCTL_VERSION := "v1.7.1" -YQ_BIN := "_out/yq" OUT_DIR := "_out" -KOPIUM_BIN := "_out/bin/kopium" KUSTOMIZE_VERSION := "v5.4.1" -KUSTOMIZE_BIN := "_out/kustomize" -CLUSTERCTL_BIN := "_out/clusterctl" ARCH := if arch() == "aarch64" { "arm64"} else { "amd64" } DIST := os() +REFRESH_BIN := env_var_or_default('REFRESH_BIN', '1') + +export PATH := "_out:_out/bin:" + env_var('PATH') [private] default: @@ -25,15 +24,15 @@ generate features="": # generates files for CRDS generate-crds: _create-out-dir _install-kopium _download-yq - just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusters.yaml" "src/api/capi_cluster.rs" "" - just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusterclasses.yaml" "src/api/capi_clusterclass.rs" "" - just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster.rs" "select(.spec.names.singular==\"cluster\")" "--no-condition" - just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_clustergroup.rs" "select(.spec.names.singular==\"clustergroup\")" "--no-condition" - just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster_registration_token.rs" "select(.spec.names.singular==\"clusterregistrationtoken\")" "" + just _generate-kopium-url kopium "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusters.yaml" "src/api/capi_cluster.rs" "" + just _generate-kopium-url kopium "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusterclasses.yaml" "src/api/capi_clusterclass.rs" "" + just _generate-kopium-url kopium "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster.rs" "select(.spec.names.singular==\"cluster\")" "--no-condition" + just _generate-kopium-url kopium "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_clustergroup.rs" "select(.spec.names.singular==\"clustergroup\")" "--no-condition" + just _generate-kopium-url kopium "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster_registration_token.rs" "select(.spec.names.singular==\"clusterregistrationtoken\")" "" [private] _generate-kopium-url kpath="" source="" dest="" yqexp="." condition="": - curl -sSL {{source}} | {{YQ_BIN}} '{{yqexp}}' | {{kpath}} -D Default {{condition}} -f - > {{dest}} + curl -sSL {{source}} | yq '{{yqexp}}' | {{kpath}} -D Default {{condition}} -f - > {{dest}} generate-addon-crds features="": cargo run --features={{features}} --bin crdgen > config/crds/fleet-addon-config.yaml @@ -63,7 +62,7 @@ compile features="": _create-out-dir -w /volume \ -t clux/muslrust:stable \ cargo build --release --features={{features}} --bin controller - cp target/x86_64-unknown-linux-musl/release/controller _out/controller + cp target/{{arch()}}-unknown-linux-musl/release/controller {{OUT_DIR}}/controller [private] _build features="": @@ -91,12 +90,10 @@ load-base features="": start-dev: _cleanup-out-dir _create-out-dir just update-helm-repos kind delete cluster --name dev || true - export LOCAL_IP=$(ip -4 -j route list default | jq -r .[0].prefsrc) - envsubst < testdata/kind-config.yaml > _out/kind-config.yaml - kind create cluster --config --image=kindest/node:v{{KUBE_VERSION}} --config _out/kind-config.yaml + kind create cluster --config --image=kindest/node:v{{KUBE_VERSION}} --config testdata/kind-config.yaml just install-fleet just install-capi - kubectl wait pods --for=condition=Ready --timeout=300s --all --all-namespaces + kubectl wait pods --for=condition=Ready --timeout=150s --all --all-namespaces # Stop the local dev environment stop-dev: @@ -112,6 +109,7 @@ deploy-child-cluster: # Deploy child cluster-call based cluster using docker & kubeadm deploy-child-cluster-class: + kind delete cluster --name capi-quickstart || true kubectl --context kind-dev apply -f testdata/capi-quickstart.yaml # Add and update helm repos used @@ -127,86 +125,139 @@ update-helm-repos: install-fleet: _create-out-dir #!/usr/bin/env bash set -euxo pipefail - kubectl config view -o json --raw | jq -r '.clusters[].cluster["certificate-authority-data"]' | base64 -d > _out/ca.pem - API_SERVER_URL=`kubectl config view -o json --raw | jq -r '.clusters[] | select(.name=="kind-dev").cluster["server"]'` + kubectl config view -o json --raw | jq -r '.clusters[].cluster["certificate-authority-data"]' | base64 -d > {{OUT_DIR}}/ca.pem + API_SERVER_URL=`kubectl get nodes -o json | jq -r '.items[0].status.addresses[] | select(.type=="InternalIP").address'` + API_SERVER_URL=https://${API_SERVER_URL}:6443 helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd - helm install --create-namespace -n cattle-fleet-system --set apiServerURL=$API_SERVER_URL --set-file apiServerCA=_out/ca.pem fleet fleet/fleet --wait + helm install --create-namespace -n cattle-fleet-system --set apiServerURL=$API_SERVER_URL --set-file apiServerCA={{OUT_DIR}}/ca.pem fleet fleet/fleet --wait # Install cluster api and any providers install-capi: _download-clusterctl - EXP_CLUSTER_RESOURCE_SET=true CLUSTER_TOPOLOGY=true {{CLUSTERCTL_BIN}} init -i docker + EXP_CLUSTER_RESOURCE_SET=true CLUSTER_TOPOLOGY=true clusterctl init -i docker # Deploy will deploy the operator deploy features="": _download-kustomize just generate {{features}} just load-base {{features}} - {{KUSTOMIZE_BIN}} build config/default | kubectl apply -f - + kustomize build config/default | kubectl apply -f - undeploy: _download-kustomize - {{KUSTOMIZE_BIN}} build config/default | kubectl delete --ignore-not-found=true -f - + kustomize build config/default | kubectl delete --ignore-not-found=true -f - release-manifests: _create-out-dir _download-kustomize - {{KUSTOMIZE_BIN}} build config/default > _out/addon-components.yaml + kustomize build config/default > {{OUT_DIR}}/addon-components.yaml # Full e2e test of importing cluster in fleet -test-import: start-dev deploy deploy-child-cluster deploy-crs - kubectl wait pods --for=condition=Ready --timeout=300s --all --all-namespaces - kubectl wait clusters.fleet.cattle.io --timeout=300s --for=condition=Ready docker-demo +test-import: start-dev deploy deploy-child-cluster deploy-crs && collect-test-import + kubectl wait pods --for=condition=Ready --timeout=150s --all --all-namespaces + kubectl wait cluster --timeout=500s --for=condition=ControlPlaneReady=true docker-demo + kubectl wait clusters.fleet.cattle.io --timeout=150s --for=condition=Ready docker-demo + +collect-test-import: + -just collect-artifacts dev + -just collect-artifacts docker-demo # Full e2e test of importing cluster in fleet -test-cluster-class-import: start-dev +test-cluster-class-import: start-dev deploy _test-import-all && collect-test-cluster-class-import + +collect-test-cluster-class-import: + -just collect-artifacts dev + -just collect-artifacts capi-quickstart + +# Test e2e with agent initiated connection procedure +test-cluster-class-import-agent-initated: start-dev && collect-test-cluster-class-import just deploy "agent-initiated" + just _test-import-all + +collect-artifacts cluster: + kind get kubeconfig --name {{cluster}} > {{OUT_DIR}}/kubeconfig + just crust-gather collect -f {{OUT_DIR}}/gather/{{cluster}} -k {{OUT_DIR}}/kubeconfig + +# Full e2e test of importing cluster and ClusterClass in fleet +[private] +_test-import-all: just deploy-child-cluster-class just deploy-crs - kubectl wait pods --for=condition=Ready --timeout=300s --all --all-namespaces - kubectl wait clustergroups.fleet.cattle.io --timeout=300s --for=jsonpath='{.status.clusterCount}=1' quick-start - kubectl wait clustergroups.fleet.cattle.io --timeout=300s --for=condition=Ready quick-start - kubectl wait clusters.fleet.cattle.io --timeout=300s --for=condition=Ready capi-quickstart + kubectl wait pods --for=condition=Ready --timeout=150s --all --all-namespaces + kubectl wait clustergroups.fleet.cattle.io --timeout=150s --for=jsonpath='{.status.clusterCount}=1' quick-start + kubectl wait clustergroups.fleet.cattle.io --timeout=150s --for=condition=Ready=true quick-start + kubectl wait clusters.fleet.cattle.io --timeout=150s --for=condition=Ready=false capi-quickstart + kubectl wait clusters.fleet.cattle.io --timeout=150s --for=condition=Ready=true capi-quickstart # Install kopium [private] _install-kopium: + #!/usr/bin/env bash + set -euxo pipefail + [ -z `which kopium` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0 cargo install --git https://github.com/kube-rs/kopium.git --tag 0.19.0 --root {{OUT_DIR}} +download-kustomize: _download-kustomize + # Download kustomize [private] [linux] [macos] _download-kustomize: - curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/{{KUSTOMIZE_VERSION}}/kustomize_{{KUSTOMIZE_VERSION}}_{{DIST}}_{{ARCH}}.tar.gz -o {{KUSTOMIZE_BIN}}.tar.gz - tar -xzf {{KUSTOMIZE_BIN}}.tar.gz -C _out - chmod +x {{KUSTOMIZE_BIN}} + #!/usr/bin/env bash + set -euxo pipefail + [ -z `which kustomize` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0 + curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/{{KUSTOMIZE_VERSION}}/kustomize_{{KUSTOMIZE_VERSION}}_{{DIST}}_{{ARCH}}.tar.gz -o {{OUT_DIR}}/kustomize.tar.gz + tar -xzf {{OUT_DIR}}/kustomize.tar.gz -C {{OUT_DIR}} + chmod +x {{OUT_DIR}}/kustomize [private] [linux] _download-clusterctl: - curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-linux-amd64 -o {{CLUSTERCTL_BIN}} - chmod +x {{CLUSTERCTL_BIN}} + #!/usr/bin/env bash + set -euxo pipefail + [ -z `which clusterctl` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0 + curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-linux-{{ARCH}} -o {{OUT_DIR}}/clusterctl + chmod +x {{OUT_DIR}}/clusterctl [private] [macos] _download-clusterctl: - curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-darwin-amd64 -o {{CLUSTERCTL_BIN}} - chmod +x {{CLUSTERCTL_BIN}} + #!/usr/bin/env bash + set -euxo pipefail + [ -z `which clusterctl` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0 + curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/{{CLUSTERCTL_VERSION}}/clusterctl-darwin-{{ARCH}} -o {{OUT_DIR}}/clusterctl + chmod +x {{OUT_DIR}}/clusterctl # Download yq [private] [linux] _download-yq: - curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_linux_{{ARCH}} -o {{YQ_BIN}} - chmod +x {{YQ_BIN}} + #!/usr/bin/env bash + set -euxo pipefail + [ -z `which yq` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0 + curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_linux_{{ARCH}} -o {{OUT_DIR}}/yq + chmod +x {{OUT_DIR}}/yq [private] [macos] _download-yq: - curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_darwin_{{ARCH}} -o {{YQ_BIN}} - chmod +x {{YQ_BIN}} + #!/usr/bin/env bash + set -euxo pipefail + [ -z `which yq` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0 + curl -sSL https://github.com/mikefarah/yq/releases/download/{{YQ_VERSION}}/yq_darwin_{{ARCH}} -o {{OUT_DIR}}/yq + chmod +x {{OUT_DIR}}/yq [private] _create-out-dir: - mkdir -p _out + mkdir -p {{OUT_DIR}} [private] _cleanup-out-dir: - rm -rf _out/ || true + rm -rf {{OUT_DIR}} || true + +crust-gather *flags: _download-crust-gather + crust-gather {{flags}} + +[private] +_download-crust-gather: _create-out-dir + #!/usr/bin/env bash + set -euxo pipefail + [ -z `which crust-gather` ] || [ {{REFRESH_BIN}} != "0" ] || exit 0 + curl -sSfL https://github.com/crust-gather/crust-gather/raw/main/install.sh | sh -s - -f -b {{OUT_DIR}} diff --git a/shell.nix b/shell.nix index 6fb0856..c7c6522 100644 --- a/shell.nix +++ b/shell.nix @@ -8,6 +8,7 @@ pkgs.mkShell { rustfmt # Formatter clippy # Linter just + fzf kind kubernetes-helm clusterctl @@ -15,8 +16,6 @@ pkgs.mkShell { k9s jq yq - envsubst - iproute2 docker-client kustomize ]; diff --git a/src/api/capi_cluster.rs b/src/api/capi_cluster.rs index d9e8777..3f7f963 100644 --- a/src/api/capi_cluster.rs +++ b/src/api/capi_cluster.rs @@ -127,6 +127,8 @@ pub struct ClusterTopologyControlPlane { pub node_volume_detach_timeout: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub replicas: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub variables: Option, } #[derive(Serialize, Deserialize, Clone, Debug, Default)] @@ -179,6 +181,20 @@ pub struct ClusterTopologyControlPlaneMetadata { pub labels: Option>, } +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct ClusterTopologyControlPlaneVariables { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overrides: Option>, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct ClusterTopologyControlPlaneVariablesOverrides { + #[serde(default, skip_serializing_if = "Option::is_none", rename = "definitionFrom")] + pub definition_from: Option, + pub name: String, + pub value: serde_json::Value, +} + #[derive(Serialize, Deserialize, Clone, Debug, Default)] pub struct ClusterTopologyVariables { #[serde(default, skip_serializing_if = "Option::is_none", rename = "definitionFrom")] diff --git a/testdata/capi-quickstart.yaml b/testdata/capi-quickstart.yaml index 41dd7c8..e21b78a 100644 --- a/testdata/capi-quickstart.yaml +++ b/testdata/capi-quickstart.yaml @@ -316,11 +316,11 @@ spec: clusterNetwork: pods: cidrBlocks: - - 192.168.0.0/16 + - 10.1.0.0/16 serviceDomain: k8s.test services: cidrBlocks: - - 10.96.0.0/12 + - 10.10.0.0/16 topology: class: quick-start controlPlane: diff --git a/testdata/kind-config.yaml b/testdata/kind-config.yaml index 009b6fe..fb8472a 100644 --- a/testdata/kind-config.yaml +++ b/testdata/kind-config.yaml @@ -5,6 +5,4 @@ nodes: - role: control-plane extraMounts: - hostPath: /var/run/docker.sock - containerPath: /var/run/docker.sock -networking: - apiServerAddress: "$LOCAL_IP" \ No newline at end of file + containerPath: /var/run/docker.sock \ No newline at end of file