diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c70f6d1525..96a4fc0a0f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,6 +27,7 @@ env: IMAGE_REGISTRY: "quay.io" IMAGE_TAG: "ci" DOCKERCMD: "podman" + DRIVER: "container" defaults: run: shell: bash @@ -116,7 +117,6 @@ jobs: curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 sudo install minikube-linux-amd64 /usr/local/bin/minikube minikube version - mkdir "$HOME/.minikube/profiles" - name: Install kubectl run: | @@ -145,6 +145,10 @@ jobs: run: make black working-directory: test + - name: Start test cluster + run: make cluster + working-directory: test + - name: Run tests run: make test working-directory: test @@ -153,6 +157,10 @@ jobs: run: make coverage working-directory: test + - name: Clean up + run: make clean + working-directory: test + ramenctl-test: name: ramenctl tests runs-on: ubuntu-22.04 diff --git a/.gitignore b/.gitignore index 2035b56bfb..467b23ed24 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,8 @@ *.dll *.so *.dylib -bin -testbin/* +/bin +/testbin/* # Test binary, build with `go test -c` *.test @@ -37,9 +37,14 @@ venv # Test enviromemnt generated files test/.coverage test/.coverage.* +test/addons/kubevirt/vm/id_rsa.pub # Python generated files *.egg-info/ __pycache__/ test/build ramenctl/build + +# Generated disk images +*.qcow2 +*.iso diff --git a/.golangci.yaml b/.golangci.yaml index 9889328dc9..57002227f0 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -86,6 +86,9 @@ issues: linters: - revive text: "should not use dot imports" + - source: "^func Test" + linters: + - funlen linters: diff --git a/controllers/drcluster_controller.go b/controllers/drcluster_controller.go index a89af4607d..a03a6d3a25 100644 --- a/controllers/drcluster_controller.go +++ b/controllers/drcluster_controller.go @@ -294,7 +294,7 @@ func (r *DRClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( u.initializeStatus() - if !u.object.ObjectMeta.DeletionTimestamp.IsZero() { + if drClusterIsDeleted(drcluster) { return processDeletion(u) } @@ -360,6 +360,11 @@ func (r DRClusterReconciler) processCreateOrUpdate(u *drclusterInstance) (ctrl.R return ctrl.Result{Requeue: requeue || u.requeue}, reconcileError } +// Return true if dr cluster was marked for deletion. +func drClusterIsDeleted(c *ramen.DRCluster) bool { + return !c.GetDeletionTimestamp().IsZero() +} + func (u *drclusterInstance) initializeStatus() { // Save a copy of the instance status to be used for the DRCluster status update comparison u.object.Status.DeepCopyInto(&u.savedInstanceStatus) diff --git a/controllers/drcluster_mmode.go b/controllers/drcluster_mmode.go index 4b08084072..b151edfd39 100644 --- a/controllers/drcluster_mmode.go +++ b/controllers/drcluster_mmode.go @@ -100,6 +100,11 @@ func (u *drclusterInstance) mModeActivationsRequired() (map[string]ramen.Storage // getVRGs is a helper function to get the VRGs for the passed in DRPC and DRPolicy association func (u *drclusterInstance) getVRGs(drpcCollection DRPCAndPolicy) (map[string]*ramen.VolumeReplicationGroup, error) { + drClusters, err := getDRClusters(u.ctx, u.client, drpcCollection.drPolicy) + if err != nil { + return nil, err + } + placementObj, err := getPlacementOrPlacementRule(u.ctx, u.client, drpcCollection.drpc, u.log) if err != nil { return nil, err @@ -113,7 +118,7 @@ func (u *drclusterInstance) getVRGs(drpcCollection DRPCAndPolicy) (map[string]*r vrgs, failedToQueryCluster, err := getVRGsFromManagedClusters( u.reconciler.MCVGetter, drpcCollection.drpc, - drpcCollection.drPolicy, + drClusters, vrgNamespace, u.log) if err != nil { diff --git a/controllers/drplacementcontrol.go b/controllers/drplacementcontrol.go index b58271a334..db25f2c3a3 100644 --- a/controllers/drplacementcontrol.go +++ b/controllers/drplacementcontrol.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/yaml" rmn "github.com/ramendr/ramen/api/v1alpha1" @@ -533,20 +534,23 @@ func (d *DRPCInstance) checkMetroFailoverPrerequisites(curHomeCluster string) (b func (d *DRPCInstance) checkRegionalFailoverPrerequisites() bool { d.setProgression(rmn.ProgressionWaitForStorageMaintenanceActivation) - if required, activationsRequired := requiresRegionalFailoverPrerequisites( - d.ctx, - d.reconciler.APIReader, - rmnutil.DRPolicyS3Profiles(d.drPolicy, d.drClusters).List(), - d.instance.GetName(), d.instance.GetNamespace(), - d.vrgs, d.instance.Spec.FailoverCluster, - d.reconciler.ObjStoreGetter, d.log); required { - for _, drCluster := range d.drClusters { - if drCluster.Name != d.instance.Spec.FailoverCluster { - continue - } + for _, drCluster := range d.drClusters { + if drCluster.Name != d.instance.Spec.FailoverCluster { + continue + } + // we want to work with failover cluster only, because the previous primary cluster might be unreachable + if required, activationsRequired := requiresRegionalFailoverPrerequisites( + d.ctx, + d.reconciler.APIReader, + []string{drCluster.Spec.S3ProfileName}, + d.instance.GetName(), d.instance.GetNamespace(), + d.vrgs, d.instance.Spec.FailoverCluster, + d.reconciler.ObjStoreGetter, d.log); required { return checkFailoverMaintenanceActivations(drCluster, activationsRequired, d.log) } + + break } return true @@ -1448,16 +1452,9 @@ func (d *DRPCInstance) createVRGManifestWork(homeCluster string, repState rmn.Re return nil } +// ensureVRGManifestWork ensures that the VRG ManifestWork exists and matches the current VRG state. +// TODO: This may be safe only when the VRG is primary - check if callers use this correctly. func (d *DRPCInstance) ensureVRGManifestWork(homeCluster string) error { - mw, mwErr := d.mwu.FindManifestWorkByType(rmnutil.MWTypeVRG, homeCluster) - if mwErr != nil { - d.log.Info("Ensure VRG ManifestWork", "Error", mwErr) - } - - if mw != nil { - return nil - } - d.log.Info("Ensure VRG ManifestWork", "Last State:", d.getLastDRState(), "cluster", homeCluster) @@ -1496,7 +1493,7 @@ func (d *DRPCInstance) generateVRG(repState rmn.ReplicationState) rmn.VolumeRepl Spec: rmn.VolumeReplicationGroupSpec{ PVCSelector: d.instance.Spec.PVCSelector, ReplicationState: repState, - S3Profiles: rmnutil.DRPolicyS3Profiles(d.drPolicy, d.drClusters).List(), + S3Profiles: d.availableS3Profiles(), KubeObjectProtection: d.instance.Spec.KubeObjectProtection, }, } @@ -1508,6 +1505,21 @@ func (d *DRPCInstance) generateVRG(repState rmn.ReplicationState) rmn.VolumeRepl return vrg } +func (d *DRPCInstance) availableS3Profiles() []string { + profiles := sets.New[string]() + + for i := range d.drClusters { + drCluster := &d.drClusters[i] + if drClusterIsDeleted(drCluster) { + continue + } + + profiles.Insert(drCluster.Spec.S3ProfileName) + } + + return sets.List(profiles) +} + func (d *DRPCInstance) generateVRGSpecAsync() *rmn.VRGAsyncSpec { if dRPolicySupportsRegional(d.drPolicy, d.drClusters) { return &rmn.VRGAsyncSpec{ diff --git a/controllers/drplacementcontrol_controller.go b/controllers/drplacementcontrol_controller.go index a4c6f072f6..1e6353f91e 100644 --- a/controllers/drplacementcontrol_controller.go +++ b/controllers/drplacementcontrol_controller.go @@ -776,7 +776,7 @@ func (r *DRPlacementControlReconciler) createDRPCInstance( return nil, err } - vrgs, err := updateVRGsFromManagedClusters(r.MCVGetter, drpc, drPolicy, vrgNamespace, log) + vrgs, err := updateVRGsFromManagedClusters(r.MCVGetter, drpc, drClusters, vrgNamespace, log) if err != nil { return nil, err } @@ -1065,8 +1065,13 @@ func (r *DRPlacementControlReconciler) finalizeDRPC(ctx context.Context, drpc *r } } + drClusters, err := getDRClusters(ctx, r.Client, drPolicy) + if err != nil { + return fmt.Errorf("failed to get drclusters. Error (%w)", err) + } + // Verify VRGs have been deleted - vrgs, _, err := getVRGsFromManagedClusters(r.MCVGetter, drpc, drPolicy, vrgNamespace, log) + vrgs, _, err := getVRGsFromManagedClusters(r.MCVGetter, drpc, drClusters, vrgNamespace, log) if err != nil { return fmt.Errorf("failed to retrieve VRGs. We'll retry later. Error (%w)", err) } @@ -1422,9 +1427,9 @@ func (r *DRPlacementControlReconciler) clonePlacementRule(ctx context.Context, } func updateVRGsFromManagedClusters(mcvGetter rmnutil.ManagedClusterViewGetter, drpc *rmn.DRPlacementControl, - drPolicy *rmn.DRPolicy, vrgNamespace string, log logr.Logger, + drClusters []rmn.DRCluster, vrgNamespace string, log logr.Logger, ) (map[string]*rmn.VolumeReplicationGroup, error) { - vrgs, failedClusterToQuery, err := getVRGsFromManagedClusters(mcvGetter, drpc, drPolicy, vrgNamespace, log) + vrgs, failedClusterToQuery, err := getVRGsFromManagedClusters(mcvGetter, drpc, drClusters, vrgNamespace, log) if err != nil { return nil, err } @@ -1443,7 +1448,7 @@ func updateVRGsFromManagedClusters(mcvGetter rmnutil.ManagedClusterViewGetter, d } func getVRGsFromManagedClusters(mcvGetter rmnutil.ManagedClusterViewGetter, drpc *rmn.DRPlacementControl, - drPolicy *rmn.DRPolicy, vrgNamespace string, log logr.Logger, + drClusters []rmn.DRCluster, vrgNamespace string, log logr.Logger, ) (map[string]*rmn.VolumeReplicationGroup, string, error) { vrgs := map[string]*rmn.VolumeReplicationGroup{} @@ -1456,33 +1461,41 @@ func getVRGsFromManagedClusters(mcvGetter rmnutil.ManagedClusterViewGetter, drpc var clustersQueriedSuccessfully int - for _, drCluster := range rmnutil.DrpolicyClusterNames(drPolicy) { - vrg, err := mcvGetter.GetVRGFromManagedCluster(drpc.Name, vrgNamespace, drCluster, annotations) + for i := range drClusters { + drCluster := &drClusters[i] + + vrg, err := mcvGetter.GetVRGFromManagedCluster(drpc.Name, vrgNamespace, drCluster.Name, annotations) if err != nil { // Only NotFound error is accepted if errors.IsNotFound(err) { - log.Info(fmt.Sprintf("VRG not found on %q", drCluster)) + log.Info(fmt.Sprintf("VRG not found on %q", drCluster.Name)) clustersQueriedSuccessfully++ continue } - failedClusterToQuery = drCluster + failedClusterToQuery = drCluster.Name - log.Info(fmt.Sprintf("failed to retrieve VRG from %s. err (%v)", drCluster, err)) + log.Info(fmt.Sprintf("failed to retrieve VRG from %s. err (%v)", drCluster.Name, err)) continue } clustersQueriedSuccessfully++ - vrgs[drCluster] = vrg + if drClusterIsDeleted(drCluster) { + log.Info("Skipping VRG on deleted drcluster", "drcluster", drCluster.Name, "vrg", vrg.Name) + + continue + } + + vrgs[drCluster.Name] = vrg - log.Info("VRG location", "VRG on", drCluster) + log.Info("VRG location", "VRG on", drCluster.Name) } // We are done if we successfully queried all drClusters - if clustersQueriedSuccessfully == len(rmnutil.DrpolicyClusterNames(drPolicy)) { + if clustersQueriedSuccessfully == len(drClusters) { return vrgs, "", nil } diff --git a/controllers/util/mw_util.go b/controllers/util/mw_util.go index d6ac4418a4..8cb5fe614f 100644 --- a/controllers/util/mw_util.go +++ b/controllers/util/mw_util.go @@ -481,50 +481,32 @@ func (mwu *MWUtil) createOrUpdateManifestWork( mw *ocmworkv1.ManifestWork, managedClusternamespace string, ) error { + key := types.NamespacedName{Name: mw.Name, Namespace: managedClusternamespace} foundMW := &ocmworkv1.ManifestWork{} - err := mwu.Client.Get(mwu.Ctx, - types.NamespacedName{Name: mw.Name, Namespace: managedClusternamespace}, - foundMW) + err := mwu.Client.Get(mwu.Ctx, key, foundMW) if err != nil { if !errors.IsNotFound(err) { - return errorswrapper.Wrap(err, fmt.Sprintf("failed to fetch ManifestWork %s", mw.Name)) + return errorswrapper.Wrap(err, fmt.Sprintf("failed to fetch ManifestWork %s", key)) } - // Let DRPC receive notification for any changes to ManifestWork CR created by it. - // if err := ctrl.SetControllerReference(d.instance, mw, d.reconciler.Scheme); err != nil { - // return fmt.Errorf("failed to set owner reference to ManifestWork resource (%s/%s) (%v)", - // mw.Name, mw.Namespace, err) - // } - mwu.Log.Info("Creating ManifestWork", "cluster", managedClusternamespace, "MW", mw) return mwu.Client.Create(mwu.Ctx, mw) } if !reflect.DeepEqual(foundMW.Spec, mw.Spec) { - mwu.Log.Info("ManifestWork exists.", "name", mw.Name, "namespace", foundMW.Namespace) - - retryErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - var err error + mwu.Log.Info("Updating ManifestWork", "name", mw.Name, "namespace", foundMW.Namespace) - err = mwu.Client.Get(mwu.Ctx, - types.NamespacedName{Name: mw.Name, Namespace: managedClusternamespace}, - foundMW) - if err != nil { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := mwu.Client.Get(mwu.Ctx, key, foundMW); err != nil { return err } mw.Spec.DeepCopyInto(&foundMW.Spec) - err = mwu.Client.Update(mwu.Ctx, foundMW) - - return err + return mwu.Client.Update(mwu.Ctx, foundMW) }) - - if retryErr != nil { - return retryErr - } } return nil diff --git a/docs/user-quick-start.md b/docs/user-quick-start.md index ded0bb7310..b524329477 100644 --- a/docs/user-quick-start.md +++ b/docs/user-quick-start.md @@ -99,32 +99,39 @@ enough resources: Tested with version v1.31. - Verify you can create a Kubernetes cluster with minikube. +1. Validate the installation - ``` - minikube start -p testcluster - ``` - - Wait for `testcluster` to complete and then issue this command: + Run the drenv-selftest to validate that we can create a test cluster: ``` - minikube profile list + test/scripts/drenv-selftest ``` Example output: ``` - |-------------|-----------|------------|-----------------|------|---------|---------|-------|--------| - | Profile | VM Driver | Runtime | IP | Port | Version | Status | Nodes | Active | - |-------------|-----------|------------|-----------------|------|---------|---------|-------|--------| - | testcluster | kvm2 | docker | 192.168.39.211 | 8443 | v1.27.4 | Running | 1 | | - |-------------|-----------|------------|-----------------|------|---------|---------|-------|--------| - ``` + 1. Activating the ramen virtual environment ... - After verifying minikube `testcluster` created, delete the cluster. - ``` - minikube delete -p testcluster + 2. Creating a test cluster ... + + 2023-11-12 14:53:43,321 INFO [drenv-selftest-vm] Starting environment + 2023-11-12 14:53:43,367 INFO [drenv-selftest-cluster] Starting minikube cluster + 2023-11-12 14:54:15,331 INFO [drenv-selftest-cluster] Cluster started in 31.96 seconds + 2023-11-12 14:54:15,332 INFO [drenv-selftest-cluster/0] Running addons/example/start + 2023-11-12 14:54:33,181 INFO [drenv-selftest-cluster/0] addons/example/start completed in 17.85 seconds + 2023-11-12 14:54:33,181 INFO [drenv-selftest-cluster/0] Running addons/example/test + 2023-11-12 14:54:33,381 INFO [drenv-selftest-cluster/0] addons/example/test completed in 0.20 seconds + 2023-11-12 14:54:33,381 INFO [drenv-selftest-vm] Environment started in 50.06 seconds + + 3. Deleting the test cluster ... + + 2023-11-12 14:54:33,490 INFO [drenv-selftest-vm] Deleting environment + 2023-11-12 14:54:33,492 INFO [drenv-selftest-cluster] Deleting cluster + 2023-11-12 14:54:34,106 INFO [drenv-selftest-cluster] Cluster deleted in 0.61 seconds + 2023-11-12 14:54:34,106 INFO [drenv-selftest-vm] Environment deleted in 0.62 seconds + + drenv is set up properly ``` 1. Install `clusteradm` tool. See diff --git a/hack/pre-commit.sh b/hack/pre-commit.sh index 142d199226..f13cb199bc 100755 --- a/hack/pre-commit.sh +++ b/hack/pre-commit.sh @@ -15,51 +15,97 @@ OUTPUTS_FILE="$(mktemp --tmpdir tool-errors-XXXXXX)" echo "${OUTPUTS_FILE}" -# run_check [optional args to checker...] -function run_check() { - regex="$1" - shift - exe="$1" - shift - - if [ -x "$(command -v "$exe")" ]; then - echo "===== $exe =====" - find . \ - -path ./testbin -prune -o \ - -path ./bin -prune -o \ - -regextype egrep -iregex "$regex" -print0 | \ - xargs -0r "$exe" "$@" 2>&1 | tee -a "${OUTPUTS_FILE}" - echo - echo - else - echo "FAILED: All checks required, but $exe not found!" +check_version() { + if ! [[ "$1" == "$(echo -e "$1\n$2" | sort -V | tail -n1)" ]] ; then + echo "ERROR: $3 version is too old. Expected $2, found $1" + exit 1 + fi +} + +get_files() { + git ls-files -z | grep --binary-files=without-match --null-data --null -E "$1" +} + +# check_tool +check_tool() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "ERROR: $1 is not installed" + echo "You can install it by running:" + case "$1" in + mdl) + echo " gem install mdl" + ;; + shellcheck) + echo " dnf install ShellCheck" + ;; + yamllint) + echo " dnf install yamllint" + ;; + *) + echo " unknown tool $1" + ;; + esac exit 1 fi } # markdownlint: https://github.com/markdownlint/markdownlint # https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md -# Install via: gem install mdl -mdl_version_test() { - IFS=. read -r x y _ <<-a - $(mdl --version) - a - test "$x" -gt 0 || test "$x" -eq 0 && test "$y" -gt 11 - set -- $? - unset -v x y - return "$1" +run_mdl() { + local tool="mdl" + local required_version="0.11.0" + local detected_version + + echo "===== $tool =====" + + check_tool "${tool}" + + detected_version=$("${tool}" --version) + check_version "${detected_version}" "${required_version}" "${tool}" + + get_files ".*\.md" | xargs -0 -r "${tool}" --style "${scriptdir}/mdl-style.rb" | tee -a "${OUTPUTS_FILE}" + echo + echo +} + +run_shellcheck() { + local tool="shellcheck" + local required_version="0.7.0" + local detected_version + + echo "===== $tool =====" + + check_tool "${tool}" + + detected_version=$("${tool}" --version | grep "version:" | cut -d' ' -f2) + check_version "${detected_version}" "${required_version}" "${tool}" + + get_files '.*\.(ba)?sh' | xargs -0 -r "${tool}" | tee -a "${OUTPUTS_FILE}" + echo + echo +} + +run_yamllint() { + local tool="yamllint" + local required_version="1.33.0" + local detected_version + + echo "===== $tool =====" + + check_tool "${tool}" + + detected_version=$("${tool}" -v | cut -d' ' -f2) + check_version "${detected_version}" "${required_version}" "${tool}" + + get_files '.*\.ya?ml' | xargs -0 -r "${tool}" -s -c "${scriptdir}/yamlconfig.yaml" | tee -a "${OUTPUTS_FILE}" + echo + echo } -if ! mdl_version_test; then - echo error: mdl version precedes minimum - exit 1 -fi -unset -f mdl_version_test -run_check '.*\.md' mdl --style "${scriptdir}/mdl-style.rb" -# Install via: dnf install ShellCheck -run_check '.*\.(ba)?sh' shellcheck -# Install via: dnf install yamllint -run_check '.*\.ya?ml' yamllint -s -c "${scriptdir}/yamlconfig.yaml" +run_mdl +run_shellcheck +run_yamllint +# Fail if any of the tools reported errors (! < "${OUTPUTS_FILE}" read -r) diff --git a/test/Makefile b/test/Makefile index f95e922e09..95cb69b563 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,13 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 +# DRIVER can be overriden to allow testing in github when we don't have +# hardware acceleration for VMs. +DRIVER ?= vm + +env := envs/$(DRIVER).yaml +prefix := drenv-test- + sources := $(wildcard \ drenv \ *.py \ @@ -39,3 +46,9 @@ coverage: coverage-html: python3 -m coverage html xdg-open htmlcov/index.html + +cluster: + drenv start --name-prefix $(prefix) $(env) + +clean: + drenv delete --name-prefix $(prefix) $(env) diff --git a/test/README.md b/test/README.md index 327bb90196..7f52c62e47 100644 --- a/test/README.md +++ b/test/README.md @@ -63,30 +63,6 @@ environment. for the details. Tested with version v1.0.1. -1. Install `docker` - - ``` - sudo dnf install docker - ``` - - Add yourself to the `docker` group to allow running docker as root: - - ``` - sudo usermod -aG docker $USER && newgrp docker - ``` - - Restart docker service to fix the permissions on the docker daemon - socket: - - ``` - sudo systemctl restart docker - ``` - - For more info see [Linux post-installation steps for Docker Engine](https://docs.docker.com/engine/install/linux-postinstall/). - - docker is used only for running drenv tests locally. You can use - podman for building and running containers locally. - ### Testing that drenv is healthy Run this script to make sure `drenv` works: @@ -620,7 +596,8 @@ These environments are useful for developing the `drenv` tool and scripts. When debugging an issue or adding a new component, it is much simpler and faster to work with a minimal environment. -- `test.yaml` - for testing `drenv` +- `vm.yaml` - for testing `drenv` with the $vm driver +- `container.yaml` - for testing `drenv` with the $container driver - `example.yaml` - example for experimenting with the `drenv` tool - `demo.yaml` - interactive demo for exploring the `drenv` tool - `e2e.yaml` - example for testing integration with the e2e framework @@ -635,6 +612,23 @@ simpler and faster to work with a minimal environment. ## Testing drenv +### Preparing the test cluster + +The tests requires a small test cluster. To create it use: + +``` +make cluster +``` + +This starts the `drenv-test-cluster` minikube profile using the kvm2 +driver. + +To delete the test cluster run: + +``` +make clean +``` + ### Running the tests Run all linters and tests and report test coverage: diff --git a/test/addons/kubevirt/test b/test/addons/kubevirt/test index 2f3f96745f..6868c3d8b4 100755 --- a/test/addons/kubevirt/test +++ b/test/addons/kubevirt/test @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import shutil import sys import time @@ -14,60 +15,23 @@ NAMESPACE = "kubevirt-test" def test(cluster): - create_namespace(cluster) - create_secret(cluster) + copy_public_key() create_vm(cluster) wait_until_vm_is_ready(cluster) verify_ssh(cluster) delete_vm(cluster) - delete_namespace(cluster) -def create_namespace(clsuter): - print(f"Creating namespace '{NAMESPACE}'") - ns = kubectl.create( - "namespace", - NAMESPACE, - "--dry-run=client", - "--output=yaml", - ) - kubectl.apply("--filename=-", input=ns, context=cluster) - - -def delete_namespace(cluster): - print(f"Deletting namespace '{NAMESPACE}'") - kubectl.delete(f"ns/{NAMESPACE}", context=cluster) - - -def create_secret(cluster): - """ - Create a secret with current user public key for accessing VM via ssh using - access credentials API: - https://kubevirt.io/user-guide/virtual_machines/accessing_virtual_machines/#static-ssh-public-key-injection-via-cloud-init - """ - public_key = os.path.expanduser("~/.ssh/id_rsa.pub") - print( - f"Creating secret my-public-key from '{public_key}' in namespace '{NAMESPACE}'" - ) - secret = kubectl.create( - "secret", - "generic", - "my-public-key", - f"--from-file=key1={public_key}", - "--dry-run=client", - "--output=yaml", - ) - kubectl.apply( - "--filename=-", - f"--namespace={NAMESPACE}", - input=secret, - context=cluster, - ) +def copy_public_key(): + src = os.path.expanduser("~/.ssh/id_rsa.pub") + dst = "vm/id_rsa.pub" + print(f"Copying public key from {src} to {dst}") + shutil.copyfile(src, dst) def create_vm(cluster): print(f"Deploying test vm in namespace '{NAMESPACE}'") - kubectl.apply("--filename=vm.yaml", f"--namespace={NAMESPACE}", context=cluster) + kubectl.apply("--kustomize=vm", context=cluster) def wait_until_vm_is_ready(cluster): @@ -83,7 +47,7 @@ def wait_until_vm_is_ready(cluster): def delete_vm(cluster): print(f"Deleting test vm in namespace '{NAMESPACE}'") - kubectl.delete("--filename=vm.yaml", f"--namespace={NAMESPACE}", context=cluster) + kubectl.delete("--kustomize=vm", context=cluster) def verify_ssh(cluster): @@ -98,21 +62,20 @@ def verify_ssh(cluster): for i in range(retries): time.sleep(delay) - print("Running 'hostname' inside the VM via ssh") + print(f"Last entries in /var/log/ramen.log (attempt {i+1}/{retries})") try: out = virtctl.ssh( "testvm", - "hostname", + "tail -6 /var/log/ramen.log", username="cirros", namespace=NAMESPACE, known_hosts="", # Skip host key verification. context=cluster, ) except Exception as e: - print(f"Attempt {i+1}/{retries} failed: {e}") + print(f"{e}") print(f"Retrying in {delay} seconds...") else: - print(f"Attempt {i+1}/{retries} succeeded") print(out) break else: diff --git a/test/addons/kubevirt/vm/kustomization.yaml b/test/addons/kubevirt/vm/kustomization.yaml new file mode 100644 index 0000000000..fa8a978071 --- /dev/null +++ b/test/addons/kubevirt/vm/kustomization.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: +- vm.yaml +- namespace.yaml +namespace: kubevirt-test +commonLabels: + app: kubevirt-test +secretGenerator: +- name: my-public-key + files: + - id_rsa.pub +generatorOptions: + disableNameSuffixHash: true diff --git a/test/addons/kubevirt/vm/namespace.yaml b/test/addons/kubevirt/vm/namespace.yaml new file mode 100644 index 0000000000..38ba06da84 --- /dev/null +++ b/test/addons/kubevirt/vm/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kubevirt-test diff --git a/test/addons/kubevirt/vm.yaml b/test/addons/kubevirt/vm/vm.yaml similarity index 96% rename from test/addons/kubevirt/vm.yaml rename to test/addons/kubevirt/vm/vm.yaml index 0aaacaccc5..cf72c8a178 100644 --- a/test/addons/kubevirt/vm.yaml +++ b/test/addons/kubevirt/vm/vm.yaml @@ -44,7 +44,7 @@ spec: - name: containerdisk containerDisk: # TODO: use ramendr repo. - image: quay.io/nirsof/cirros:0.6.2 + image: quay.io/nirsof/cirros:0.6.2-1 - name: cloudinitdisk cloudInitConfigDrive: userData: | diff --git a/test/conftest.py b/test/conftest.py index b359bb25de..6400ffc702 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,39 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 import os -import secrets -import subprocess import pytest from drenv import envfile -TEST_ENV = os.path.join("envs", "test.yaml") +# DRIVER can be overriden to allow testing in github when we don't have +# hardware acceleration for VMs. +DRIVER = os.environ.get("DRIVER", "vm") + +TEST_ENV = os.path.join("envs", f"{DRIVER}.yaml") class Env: def __init__(self): - self.prefix = f"test-{secrets.token_hex(16)}-" + self.prefix = "drenv-test-" with open(TEST_ENV) as f: env = envfile.load(f, name_prefix=self.prefix) self.profile = env["profiles"][0]["name"] - def start(self): - self._run("start") - - def delete(self): - self._run("delete") - - def _run(self, cmd): - subprocess.run( - ["drenv", cmd, "--verbose", "--name-prefix", self.prefix, TEST_ENV], - check=True, - ) - @pytest.fixture(scope="session") def tmpenv(): - env = Env() - env.start() - yield env - env.delete() + return Env() diff --git a/test/drenv/minikube.py b/test/drenv/minikube.py index 0150bdb16b..411bab751c 100644 --- a/test/drenv/minikube.py +++ b/test/drenv/minikube.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import logging +import os from . import commands @@ -17,6 +18,10 @@ def profile(command, output=None): + # Workaround for https://github.com/kubernetes/minikube/pull/16900 + # TODO: remove when issue is fixed. + _create_profiles_dir() + return _run("profile", command, output=output) @@ -110,3 +115,10 @@ def _watch(command, *args, profile=None): logging.debug("[%s] Running %s", profile, cmd) for line in commands.watch(*cmd): logging.debug("[%s] %s", profile, line) + + +def _create_profiles_dir(): + minikube_home = os.environ.get("MINIKUBE_HOME", os.environ.get("HOME")) + profiles = os.path.join(minikube_home, ".minikube", "profiles") + logging.debug("Creating '%s'", profiles) + os.makedirs(profiles, exist_ok=True) diff --git a/test/envs/test.yaml b/test/envs/container.yaml similarity index 72% rename from test/envs/test.yaml rename to test/envs/container.yaml index c332801e65..e68a59f6a5 100644 --- a/test/envs/test.yaml +++ b/test/envs/container.yaml @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviromemnt for testing the drenv package in github. +# Environment for testing the drenv with the $container driver. --- -name: test +name: container profiles: - name: cluster driver: $container diff --git a/test/envs/e2e.yaml b/test/envs/e2e.yaml index a1adf1982e..5be31da297 100644 --- a/test/envs/e2e.yaml +++ b/test/envs/e2e.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing integration with e2e framework. +# Environment for testing integration with e2e framework. --- name: "e2e" diff --git a/test/envs/minio.yaml b/test/envs/minio.yaml index de04c40534..122b293fb3 100644 --- a/test/envs/minio.yaml +++ b/test/envs/minio.yaml @@ -1,23 +1,15 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing minio deployment with both container and vm drivers. +# Environment for testing minio deployment. --- name: "minio" -templates: - - name: base - memory: 2g - workers: - - addons: - - name: minio - profiles: - name: c1 - template: base - driver: $container - container_runtime: cri-o - - name: c2 - template: base driver: $vm container_runtime: containerd + memory: 2g + workers: + - addons: + - name: minio diff --git a/test/envs/ocm.yaml b/test/envs/ocm.yaml index 849d195227..c119727105 100644 --- a/test/envs/ocm.yaml +++ b/test/envs/ocm.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing OCM deployment using clusteradm. +# Environment for testing OCM deployment using clusteradm. --- name: "ocm" diff --git a/test/envs/regional-dr-hubless.yaml b/test/envs/regional-dr-hubless.yaml index 490ff7b189..fff2a57f74 100644 --- a/test/envs/regional-dr-hubless.yaml +++ b/test/envs/regional-dr-hubless.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing Regional-DR in a setup without a hub. +# Environment for testing Regional-DR in a setup without a hub. --- name: "rdr-hubless" diff --git a/test/envs/regional-dr-kubevirt.yaml b/test/envs/regional-dr-kubevirt.yaml index c6527fa3bb..eeb7fdbced 100644 --- a/test/envs/regional-dr-kubevirt.yaml +++ b/test/envs/regional-dr-kubevirt.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing kubevirt flows with Regional-DR +# Environment for testing kubevirt flows with Regional-DR --- name: "rdr-kubevirt" diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index f42fafe7f7..66ea1bfd13 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing Regional-DR. +# Environment for testing Regional-DR. --- name: "rdr" diff --git a/test/envs/rook.yaml b/test/envs/rook.yaml index c371fa0605..d496fc5cad 100644 --- a/test/envs/rook.yaml +++ b/test/envs/rook.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing rook ceph deployment. +# Environment for testing rook ceph deployment. --- name: "rook" diff --git a/test/envs/velero.yaml b/test/envs/velero.yaml index 8a43add8dc..5d99e9f78e 100644 --- a/test/envs/velero.yaml +++ b/test/envs/velero.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing velero deployment. +# Environment for testing velero deployment. --- name: "velero" diff --git a/test/envs/vm.yaml b/test/envs/vm.yaml new file mode 100644 index 0000000000..4cf7aafa0c --- /dev/null +++ b/test/envs/vm.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +# Environment for testing the drenv with the $vm driver. +--- +name: vm +profiles: + - name: cluster + driver: $vm + container_runtime: containerd + memory: 2g + workers: + - addons: + - name: example diff --git a/test/envs/volsync.yaml b/test/envs/volsync.yaml index c0ece93562..b72d55caf6 100644 --- a/test/envs/volsync.yaml +++ b/test/envs/volsync.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 -# Enviroment for testing volsync deployment. +# Environment for testing volsync deployment. --- name: volsync diff --git a/test/scripts/drenv-selftest b/test/scripts/drenv-selftest index 6004177857..4ae4158670 100755 --- a/test/scripts/drenv-selftest +++ b/test/scripts/drenv-selftest @@ -3,7 +3,8 @@ # We must run in the test directory. cd $(dirname $(dirname $0)) -prefix="test-$(uuidgen)" +prefix="drenv-selftest-" +env="envs/vm.yaml" echo echo "1. Activating the ramen virtual environment ..." @@ -15,13 +16,13 @@ echo echo "2. Creating a test cluster ..." echo -drenv start --name-prefix $prefix envs/test.yaml +drenv start --name-prefix $prefix $env echo echo "3. Deleting the test cluster ..." echo -drenv delete --name-prefix $prefix envs/test.yaml +drenv delete --name-prefix $prefix $env echo echo drenv is set up properly diff --git a/test/vms/cirros/Containerfile b/test/vms/cirros/Containerfile new file mode 100644 index 0000000000..d6c0249cc8 --- /dev/null +++ b/test/vms/cirros/Containerfile @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +FROM scratch +ARG disk +ADD ${disk} /disk/ diff --git a/test/vms/cirros/Makefile b/test/vms/cirros/Makefile new file mode 100644 index 0000000000..6e678903ed --- /dev/null +++ b/test/vms/cirros/Makefile @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +REGISTRY ?= quay.io +REPOSITORY ?= nirsof +NAME ?= cirros +VERSION ?= 0.6.2 +RELEASE ?= 1 + +work := work.qcow2 +disk := cirros-$(VERSION)-$(RELEASE).qcow2 +container := $(REGISTRY)/$(REPOSITORY)/$(NAME):$(VERSION)-$(RELEASE) + +all: $(disk) + podman build --tag $(container) --build-arg disk=$(disk) . + +push: + podman push $(container) + +$(disk): + ./download-cirros --cirros-version $(VERSION) --output $(work) + virt-customize --add $(work) \ + --copy-in ../ramen:/tmp \ + --run-command "/tmp/ramen/install" \ + --run-command "sed -i 's/^\(kernel .*\)$$/\1 quiet/' /boot/grub/menu.lst" \ + --delete "/tmp/ramen" + qemu-img convert -f qcow2 -O qcow2 -c $(work) $@ + +vm: $(disk) nocloud.iso + # Create a test overlay image top of the cirros disk. + test -f test.qcow2 || qemu-img create -f qcow2 -b $(disk) -F qcow2 test.qcow2 + # Start a vm using the disk and nocloud.iso to avoid metadata server lookup. + qemu-kvm -m 256 \ + -net nic \ + -net user \ + -drive file=test.qcow2,format=qcow2,if=virtio \ + -drive file=nocloud.iso,format=raw,if=virtio + +nocloud.iso: + cloud-localds $@ user-data meta-data + +clean: + podman image rm -f $(container) + rm -f $(disk) + rm -f $(work) + rm -f nocloud.iso + rm -f test.qcow2 diff --git a/test/vms/cirros/README.md b/test/vms/cirros/README.md new file mode 100644 index 0000000000..51b9db1591 --- /dev/null +++ b/test/vms/cirros/README.md @@ -0,0 +1,63 @@ +# Cirros test VM + +This directory provides scripts and container configuration for creating +a container with cirros vm disk image. This VM can be use to test +KubeVirt DR integration. + +## Requirements + +For Fedora install these packages: + +``` +sudo dnf install \ + cloud-utils \ + guestfs-tools \ + podman \ + qemu \ + qemu-img \ +``` + +## Configuration + +To push the image to your private registry or modify other setting use +the environment variables: + +``` +export REPOSITORY=my-quay-user +``` + +See the Makefile for available variables. + +## Create VM image + +``` +make +``` + +This builds the image `quay.io/my-repo/cirros:latest`. + +## Test the VM image + +``` +make vm +``` + +This create a test image on top of the cirros vm disk, and starts +qemu-kvm and virt-viewer. You can log in and inspect the VM contents. + +You can stop the VM and start it again using `make vm` to test shutdown +and startup behavior. + +## Pushing image to your repo + +``` +make push +``` + +This pushes the built container to your repo. + +## Cleaning up + +``` +make clean +``` diff --git a/test/vms/cirros/download-cirros b/test/vms/cirros/download-cirros new file mode 100755 index 0000000000..dc7d713791 --- /dev/null +++ b/test/vms/cirros/download-cirros @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import json +import logging +import os +import shutil +import subprocess +import tempfile +import time + + +def main(): + p = argparse.ArgumentParser("download-cirros") + p.add_argument("--cirros-version", default="0.6.2", help="Cirros version") + p.add_argument("-o", "--output", help="Output filename") + p.add_argument("-v", "--verbose", action="store_true", help="Be more verbose") + args = p.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s %(levelname)-7s %(message)s", + ) + + out_img = args.output or f"cirros-{args.cirros_version}.qcow2" + + with tempfile.TemporaryDirectory(prefix="download-cirros-") as tmp_dir: + work_img = os.path.join(tmp_dir, out_img) + nocloud_img = os.path.join(tmp_dir, "nocloud.iso") + + download_image(args.cirros_version, work_img) + make_nocloud_img(tmp_dir, nocloud_img) + unpack_rootfs(work_img, nocloud_img) + shutil.copyfile(work_img, out_img) + + +def download_image(version, out): + logging.debug("Downloading cirros version %s to %s", version, out) + url = ( + f"https://download.cirros-cloud.net/{version}/cirros-{version}-x86_64-disk.img" + ) + cmd = ["curl", "--no-progress-meter", "--location", "--output", out, url] + logging.debug("Running %s", cmd) + subprocess.run(cmd, check=True) + + +def make_nocloud_img(tmp_dir, out): + """ + Create clound-init nocloud image with the required meta-data json and empty + user-data script. This avoids the slow lookup for metadata server. When the + image is started in teh real deployment it will get a new instance-id and + will be configured again. + """ + logging.debug("Creating cloud-init image %s", out) + + meta = {"instance-id": "download-cirros"} + meta_data = os.path.join(tmp_dir, "meta-data") + with open(meta_data, "w") as f: + f.write(json.dumps(meta) + "\n") + + user_data = os.path.join(tmp_dir, "user-data") + with open(user_data, "w") as f: + f.write("#!/bin/sh\n") + os.chmod(user_data, 0o700) + + cmd = ["cloud-localds", out, user_data, meta_data] + logging.debug("Running %s", cmd) + subprocess.run(cmd, check=True) + + +def unpack_rootfs(boot_img, nocloud_img): + logging.debug("Unpacking rootfs in %s", boot_img) + cmd = [ + "qemu-kvm", + "-nodefaults", + "-machine", + "accel=kvm:tcg", + "-m", + "256", + "-drive", + f"if=virtio,file={boot_img},format=qcow2", + "-drive", + f"if=virtio,file={nocloud_img},format=raw", + "-nographic", + "-net", + "none", + "-monitor", + "none", + "-serial", + "stdio", + ] + logging.debug("Staring process %s", cmd) + p = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + logging.debug("Started qemu-kvm pid=%s", p.pid) + logging.debug("Waiting for login prompt...") + wait_for_output(p, "login: ") + finally: + if p.poll() is not None: + err = p.stderr.read().decode() + raise RuntimeError( + f"qemu-kvm terminated pid={p.pid} rc={p.returncode} err={err}" + ) + + logging.debug("Terminating qemu-kvm pid=%s", p.pid) + p.terminate() + try: + p.wait(10) + except subprocess.TimeoutExpired: + logging.debug("Killing qemu-kvm pid=%s", p.pid) + p.kill() + p.wait() + + +def wait_for_output(p, what, timeout=120): + deadline = time.monotonic() + timeout + data = what.encode("utf-8") + line = bytearray() + + while p.poll() is None: + if time.monotonic() > deadline: + raise RuntimeError(f"Timeout waiting for output '{what}'") + + line += p.stdout.read(1) + if line.endswith(data): + break + + if line.endswith(b"\r\n"): + logging.debug("%s", line.rstrip().decode("utf-8")) + del line[:] + + +if __name__ == "__main__": + main() diff --git a/test/vms/cirros/meta-data b/test/vms/cirros/meta-data new file mode 100644 index 0000000000..0dbf56d674 --- /dev/null +++ b/test/vms/cirros/meta-data @@ -0,0 +1 @@ +{"instance-id":"download-cirros"} diff --git a/test/vms/cirros/user-data b/test/vms/cirros/user-data new file mode 100755 index 0000000000..1a2485251c --- /dev/null +++ b/test/vms/cirros/user-data @@ -0,0 +1 @@ +#!/bin/sh diff --git a/test/vms/ramen/etc/init.d/ramen b/test/vms/ramen/etc/init.d/ramen new file mode 100755 index 0000000000..6cd8ad7182 --- /dev/null +++ b/test/vms/ramen/etc/init.d/ramen @@ -0,0 +1,49 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +DAEMON="ramen" +PIDFILE="/var/run/$DAEMON.pid" + +start() { + printf 'Starting %s: ' "$DAEMON" + # Must run as "/bin/sh /usr/bin/ramen" to detect if already running. + start-stop-daemon --start \ + --quiet \ + --background \ + --make-pidfile \ + --pidfile "$PIDFILE" \ + --exec /bin/sh \ + -- "/usr/bin/$DAEMON" + status=$? + if [ "$status" -eq 0 ]; then + echo "OK" + else + echo "FAIL" + fi + return "$status" +} + +stop() { + printf 'Stopping %s: ' "$DAEMON" + start-stop-daemon --stop \ + --quiet \ + --pidfile "$PIDFILE" + status=$? + if [ "$status" -eq 0 ]; then + rm -f "$PIDFILE" + echo "OK" + else + echo "FAIL" + fi + return "$status" +} + +case "$1" in + start|stop) + "$1";; + *) + echo "Usage: $0 {start|stop}" + exit 1 +esac diff --git a/test/vms/ramen/install b/test/vms/ramen/install new file mode 100755 index 0000000000..ff35244a5c --- /dev/null +++ b/test/vms/ramen/install @@ -0,0 +1,10 @@ +#!/bin/sh -e + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +chdir $(dirname $0) +cp usr/bin/ramen /usr/bin +cp etc/init.d/ramen /etc/init.d +ln -sn ../init.d/ramen /etc/rc3.d/S00-ramen +ln -sn ../init.d/ramen /etc/rc3.d/K00-ramen diff --git a/test/vms/ramen/usr/bin/ramen b/test/vms/ramen/usr/bin/ramen new file mode 100755 index 0000000000..86aec55e39 --- /dev/null +++ b/test/vms/ramen/usr/bin/ramen @@ -0,0 +1,20 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +log="/var/log/ramen.log" + +emit() { + echo "$(date) $1" >> "$log" + sync "$log" +} + +trap "emit STOP; exit" TERM INT + +emit "START uptime=$(awk '{print $1}' /proc/uptime)" + +while true; do + sleep 10 & wait + emit UPDATE +done