From a142fe458cc9e13ceaef136211c42a0ca5e4b975 Mon Sep 17 00:00:00 2001 From: Rajiv Senthilnathan Date: Mon, 16 Oct 2023 21:03:34 -0400 Subject: [PATCH] Vm webhook limits (#477) * Add webhook to add limits to VMs * Refactoring and tests * Update failure policies * Add comments * Use VM apis from toolchain-common * Update api dependency * Fix unit test * Switch to unstructured type * Fix comment * Fix lint errors --------- Co-authored-by: Francisc Munteanu Co-authored-by: Alexey Kazakov --- cmd/webhook/main.go | 3 +- deploy/webhook/member-operator-webhook.yaml | 37 +- pkg/webhook/deploy/deployment_test.go | 6 +- pkg/webhook/mutatingwebhook/mutate.go | 89 +---- pkg/webhook/mutatingwebhook/mutate_test.go | 270 ++++--------- .../mutatingwebhook/userpods_mutate.go | 74 ++++ .../mutatingwebhook/userpods_mutate_test.go | 226 +++++++++++ pkg/webhook/mutatingwebhook/vm_mutate.go | 101 +++++ pkg/webhook/mutatingwebhook/vm_mutate_test.go | 367 ++++++++++++++++++ 9 files changed, 896 insertions(+), 277 deletions(-) create mode 100644 pkg/webhook/mutatingwebhook/userpods_mutate.go create mode 100644 pkg/webhook/mutatingwebhook/userpods_mutate_test.go create mode 100644 pkg/webhook/mutatingwebhook/vm_mutate.go create mode 100644 pkg/webhook/mutatingwebhook/vm_mutate_test.go diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 9d9d7b80..a3904775 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -100,7 +100,8 @@ func main() { } mux := http.NewServeMux() - mux.HandleFunc("/mutate-users-pods", mutatingwebhook.HandleMutate) + mux.HandleFunc("/mutate-users-pods", mutatingwebhook.HandleMutateUserPods) + mux.HandleFunc("/mutate-virtual-machines", mutatingwebhook.HandleMutateVirtualMachines) mux.HandleFunc("/validate-users-rolebindings", rolebindingValidator.HandleValidate) mux.HandleFunc("/validate-users-checlusters", checlusterValidator.HandleValidate) mux.HandleFunc("/validate-spacebindingrequests", spacebindingrequestValidator.HandleValidate) diff --git a/deploy/webhook/member-operator-webhook.yaml b/deploy/webhook/member-operator-webhook.yaml index 5092a09a..ad792cb9 100644 --- a/deploy/webhook/member-operator-webhook.yaml +++ b/deploy/webhook/member-operator-webhook.yaml @@ -36,6 +36,14 @@ objects: - get - list - watch + - apiGroups: + - "kubevirt.io" + resources: + - "virtualmachines" + verbs: + - get + - list + - watch - apiVersion: v1 kind: ServiceAccount metadata: @@ -137,6 +145,33 @@ objects: namespaceSelector: matchLabels: toolchain.dev.openshift.com/provider: codeready-toolchain + # The users.virtualmachines.webhook.sandbox webhook sets resource limits on VirtualMachines prior to creation as a workaround for https://issues.redhat.com/browse/CNV-28746 (https://issues.redhat.com/browse/CNV-32069) + # This webhook should be updated to remove the workaround once https://issues.redhat.com/browse/CNV-32069 is complete. + # The webhook code is available at member-operator/pkg/webhook/mutatingwebhook/vm_mutate.go + - name: users.virtualmachines.webhook.sandbox + admissionReviewVersions: + - v1 + clientConfig: + caBundle: ${CA_BUNDLE} + service: + name: member-operator-webhook + namespace: ${NAMESPACE} + path: "/mutate-virtual-machines" + port: 443 + matchPolicy: Equivalent + rules: + - operations: ["CREATE"] + apiGroups: ["kubevirt.io"] + apiVersions: ["v1"] + resources: ["virtualmachines"] + scope: "Namespaced" + sideEffects: None + timeoutSeconds: 5 + reinvocationPolicy: Never + failurePolicy: Fail + namespaceSelector: + matchLabels: + toolchain.dev.openshift.com/provider: codeready-toolchain - apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -189,7 +224,7 @@ objects: sideEffects: None timeoutSeconds: 5 reinvocationPolicy: Never - failurePolicy: Ignore + failurePolicy: Fail namespaceSelector: matchLabels: toolchain.dev.openshift.com/provider: codeready-toolchain diff --git a/pkg/webhook/deploy/deployment_test.go b/pkg/webhook/deploy/deployment_test.go index 0164f810..deea813a 100644 --- a/pkg/webhook/deploy/deployment_test.go +++ b/pkg/webhook/deploy/deployment_test.go @@ -223,11 +223,11 @@ func deployment(namespace, sa string, image string) string { } func mutatingWebhookConfig(namespace, caBundle string) string { - return fmt.Sprintf(`{"apiVersion":"admissionregistration.k8s.io/v1","kind":"MutatingWebhookConfiguration","metadata":{"name":"member-operator-webhook","labels":{"app":"member-operator-webhook","toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"webhooks":[{"name":"users.pods.webhook.sandbox","admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%s","service":{"name":"member-operator-webhook","namespace":"%s","path":"/mutate-users-pods","port":443}},"matchPolicy":"Equivalent","rules":[{"operations":["CREATE"],"apiGroups":[""],"apiVersions":["v1"],"resources":["pods"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5,"reinvocationPolicy":"Never","failurePolicy":"Ignore","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}}}]}`, caBundle, namespace) + return fmt.Sprintf(`{"apiVersion":"admissionregistration.k8s.io/v1","kind":"MutatingWebhookConfiguration","metadata":{"name":"member-operator-webhook","labels":{"app":"member-operator-webhook","toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"webhooks":[{"name":"users.pods.webhook.sandbox","admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/mutate-users-pods","port":443}},"matchPolicy":"Equivalent","rules":[{"operations":["CREATE"],"apiGroups":[""],"apiVersions":["v1"],"resources":["pods"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5,"reinvocationPolicy":"Never","failurePolicy":"Ignore","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}}},{"name":"users.virtualmachines.webhook.sandbox","admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/mutate-virtual-machines","port":443}},"matchPolicy":"Equivalent","rules":[{"operations":["CREATE"],"apiGroups":["kubevirt.io"],"apiVersions":["v1"],"resources":["virtualmachines"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5,"reinvocationPolicy":"Never","failurePolicy":"Fail","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}}}]}`, caBundle, namespace) } func validatingWebhookConfig(namespace, caBundle string) string { - return fmt.Sprintf(`{"apiVersion":"admissionregistration.k8s.io/v1","kind":"ValidatingWebhookConfiguration","metadata":{"labels":{"app":"member-operator-webhook","toolchain.dev.openshift.com/provider":"codeready-toolchain"},"name":"member-operator-validating-webhook"},"webhooks":[{"admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/validate-users-rolebindings","port":443}},"failurePolicy":"Ignore","matchPolicy":"Equivalent","name":"users.rolebindings.webhook.sandbox","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"reinvocationPolicy":"Never","rules":[{"apiGroups":["rbac.authorization.k8s.io","authorization.openshift.io"],"apiVersions":["v1"],"operations":["CREATE","UPDATE"],"resources":["rolebindings"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5},{"admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/validate-users-checlusters","port":443}},"failurePolicy":"Ignore","matchPolicy":"Equivalent","name":"users.checlusters.webhook.sandbox","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"reinvocationPolicy":"Never","rules":[{"apiGroups":["org.eclipse.che"],"apiVersions":["v2"],"operations":["CREATE"],"resources":["checlusters"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5},{"admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/validate-spacebindingrequests","port":443}},"failurePolicy":"Fail","matchPolicy":"Equivalent","name":"users.spacebindingrequests.webhook.sandbox","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"reinvocationPolicy":"Never","rules":[{"apiGroups":["toolchain.dev.openshift.com"],"apiVersions":["v1alpha1"],"operations":["CREATE", "UPDATE"],"resources":["spacebindingrequests"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5}]}`, caBundle, namespace) + return fmt.Sprintf(`{"apiVersion":"admissionregistration.k8s.io/v1","kind":"ValidatingWebhookConfiguration","metadata":{"labels":{"app":"member-operator-webhook","toolchain.dev.openshift.com/provider":"codeready-toolchain"},"name":"member-operator-validating-webhook"},"webhooks":[{"admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/validate-users-rolebindings","port":443}},"failurePolicy":"Ignore","matchPolicy":"Equivalent","name":"users.rolebindings.webhook.sandbox","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"reinvocationPolicy":"Never","rules":[{"apiGroups":["rbac.authorization.k8s.io","authorization.openshift.io"],"apiVersions":["v1"],"operations":["CREATE","UPDATE"],"resources":["rolebindings"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5},{"admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/validate-users-checlusters","port":443}},"failurePolicy":"Fail","matchPolicy":"Equivalent","name":"users.checlusters.webhook.sandbox","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"reinvocationPolicy":"Never","rules":[{"apiGroups":["org.eclipse.che"],"apiVersions":["v2"],"operations":["CREATE"],"resources":["checlusters"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5},{"admissionReviewVersions":["v1"],"clientConfig":{"caBundle":"%[1]s","service":{"name":"member-operator-webhook","namespace":"%[2]s","path":"/validate-spacebindingrequests","port":443}},"failurePolicy":"Fail","matchPolicy":"Equivalent","name":"users.spacebindingrequests.webhook.sandbox","namespaceSelector":{"matchLabels":{"toolchain.dev.openshift.com/provider":"codeready-toolchain"}},"reinvocationPolicy":"Never","rules":[{"apiGroups":["toolchain.dev.openshift.com"],"apiVersions":["v1alpha1"],"operations":["CREATE", "UPDATE"],"resources":["spacebindingrequests"],"scope":"Namespaced"}],"sideEffects":"None","timeoutSeconds":5}]}`, caBundle, namespace) } func serviceAccount(namespace string) string { @@ -235,7 +235,7 @@ func serviceAccount(namespace string) string { } func clusterRole() string { - return `{"apiVersion": "rbac.authorization.k8s.io/v1","kind": "ClusterRole","metadata": {"creationTimestamp": null,"name": "webhook-role"}, "rules": [{"apiGroups": ["user.openshift.io"],"resources": ["identities","useridentitymappings","users"],"verbs": ["get","list","watch"]},{"apiGroups": ["toolchain.dev.openshift.com"],"resources": ["spacebindingrequests"],"verbs": ["get","list","watch"]}]}` + return `{"apiVersion": "rbac.authorization.k8s.io/v1","kind": "ClusterRole","metadata": {"creationTimestamp": null,"name": "webhook-role"}, "rules": [{"apiGroups": ["user.openshift.io"],"resources": ["identities","useridentitymappings","users"],"verbs": ["get","list","watch"]},{"apiGroups": ["toolchain.dev.openshift.com"],"resources": ["spacebindingrequests"],"verbs": ["get","list","watch"]},{"apiGroups": ["kubevirt.io"],"resources": ["virtualmachines"],"verbs": ["get","list","watch"]}]}` } func clusterRoleBinding(namespace string) string { diff --git a/pkg/webhook/mutatingwebhook/mutate.go b/pkg/webhook/mutatingwebhook/mutate.go index 3d3e079d..4d740972 100644 --- a/pkg/webhook/mutatingwebhook/mutate.go +++ b/pkg/webhook/mutatingwebhook/mutate.go @@ -5,87 +5,59 @@ import ( "fmt" "io" "net/http" - "os" - "github.com/pkg/errors" + "github.com/go-logr/logr" v1 "k8s.io/api/admission/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" - logf "sigs.k8s.io/controller-runtime/pkg/log" ) var ( runtimeScheme = runtime.NewScheme() codecs = serializer.NewCodecFactory(runtimeScheme) deserializer = codecs.UniversalDeserializer() - - log = logf.Log.WithName("users_pods_mutating_webhook") - - patchContent = patchedContent() -) - -const ( - priority = int32(-3) - priorityClassName = "sandbox-users-pods" ) -func patchedContent() []byte { - patchItems := []map[string]interface{}{ - { - "op": "replace", - "path": "/spec/priorityClassName", - "value": priorityClassName, - }, - { - "op": "replace", - "path": "/spec/priority", - "value": priority, - }, - } - - patchContent, err := json.Marshal(patchItems) - if err != nil { - log.Error(err, "unable marshal patch items") - os.Exit(1) - } - return patchContent -} +type mutateHandler func(admReview v1.AdmissionReview) *v1.AdmissionResponse -func HandleMutate(w http.ResponseWriter, r *http.Request) { +func handleMutate(logger logr.Logger, w http.ResponseWriter, r *http.Request, mutator mutateHandler) { var respBody []byte body, err := io.ReadAll(r.Body) defer func() { if err := r.Body.Close(); err != nil { - log.Error(err, "unable to close the body") + logger.Error(err, "unable to close the body") } }() if err != nil { - log.Error(err, "unable to read the body of the request") + logger.Error(err, "unable to read the body of the request") w.WriteHeader(http.StatusInternalServerError) respBody = []byte("unable to read the body of the request") } else { // mutate the request - respBody = mutate(body) + respBody = mutate(logger, body, mutator) w.WriteHeader(http.StatusOK) } if _, err := io.WriteString(w, string(respBody)); err != nil { - log.Error(err, "unable to write response") + logger.Error(err, "unable to write response") } } -func mutate(body []byte) []byte { +func mutate(logger logr.Logger, body []byte, mutator mutateHandler) []byte { admReview := v1.AdmissionReview{} if _, _, err := deserializer.Decode(body, nil, &admReview); err != nil { - log.Error(err, "unable to deserialize the admission review object", "body", string(body)) + logger.Error(err, "unable to deserialize the admission review object", "body", string(body)) + admReview.Response = responseWithError(err) + } else if admReview.Request == nil { + err := fmt.Errorf("admission review request is nil") + logger.Error(err, "cannot read the admission review request", "AdmissionReview", admReview) admReview.Response = responseWithError(err) } else { - admReview.Response = createAdmissionReviewResponse(admReview) + admReview.Response = mutator(admReview) } responseBody, err := json.Marshal(admReview) if err != nil { - log.Error(err, "unable to marshal the admission review with response", "admissionReview", admReview) + logger.Error(err, "unable to marshal the admission review with response", "admissionReview", admReview) } return responseBody } @@ -97,34 +69,3 @@ func responseWithError(err error) *v1.AdmissionResponse { }, } } - -func createAdmissionReviewResponse(admReview v1.AdmissionReview) *v1.AdmissionResponse { - if admReview.Request == nil { - err := fmt.Errorf("admission review request is nil") - log.Error(err, "cannot read the admission review request", "AdmissionReview", admReview) - return responseWithError(err) - } - - // let's unmarshal the object to be sure that it's a pod - var pod *corev1.Pod - if err := json.Unmarshal(admReview.Request.Object.Raw, &pod); err != nil { - log.Error(err, "unable unmarshal pod json object", "AdmissionReview", admReview) - return responseWithError(errors.Wrapf(err, "unable unmarshal pod json object - raw request object: %v", admReview.Request.Object.Raw)) - } - - patchType := v1.PatchTypeJSONPatch - resp := &v1.AdmissionResponse{ - Allowed: true, - UID: admReview.Request.UID, - PatchType: &patchType, - } - resp.AuditAnnotations = map[string]string{ - "users_pods_mutating_webhook": "the sandbox-users-pods PriorityClass was set", - } - - // instead of changing the pod object we need to tell K8s how to change the object - resp.Patch = patchContent - - log.Info("the sandbox-users-pods PriorityClass was set to the pod", "pod-name", pod.Name, "namespace", pod.Namespace) - return resp -} diff --git a/pkg/webhook/mutatingwebhook/mutate_test.go b/pkg/webhook/mutatingwebhook/mutate_test.go index ba20a9f5..ec87bea9 100644 --- a/pkg/webhook/mutatingwebhook/mutate_test.go +++ b/pkg/webhook/mutatingwebhook/mutate_test.go @@ -3,7 +3,7 @@ package mutatingwebhook import ( "bytes" "encoding/json" - "io" + "errors" "net/http" "net/http/httptest" "testing" @@ -11,80 +11,90 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" admissionv1 "k8s.io/api/admission/v1" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) -func TestHandleMutateSuccess(t *testing.T) { - // given - ts := httptest.NewServer(http.HandlerFunc(HandleMutate)) - defer ts.Close() - - // when - resp, err := http.Post(ts.URL, "application/json", bytes.NewBuffer(rawJSON)) - - // then - assert.NoError(t, err) - body, err := io.ReadAll(resp.Body) - defer func() { - require.NoError(t, resp.Body.Close()) - }() - assert.NoError(t, err) - verifySuccessfulResponse(t, body) -} +var testLogger = logf.Log.WithName("test_mutate") -func TestMutateSuccess(t *testing.T) { - // when - response := mutate(rawJSON) +type expectedSuccessResponse struct { + patch string + auditAnnotationKey string + auditAnnotationVal string + uid string +} - // then - verifySuccessfulResponse(t, response) +type expectedFailedResponse struct { + auditAnnotationKey string + errMsg string } -func verifySuccessfulResponse(t *testing.T, response []byte) { - reviewResponse := toReviewResponse(t, response) - assert.Equal(t, `[{"op":"replace","path":"/spec/priorityClassName","value":"sandbox-users-pods"},{"op":"replace","path":"/spec/priority","value":-3}]`, string(reviewResponse.Patch)) - assert.Contains(t, "the sandbox-users-pods PriorityClass was set", reviewResponse.AuditAnnotations["users_pods_mutating_webhook"]) - assert.True(t, reviewResponse.Allowed) - assert.Equal(t, admissionv1.PatchTypeJSONPatch, *reviewResponse.PatchType) - assert.Empty(t, reviewResponse.Result) - assert.Equal(t, "a68769e5-d817-4617-bec5-90efa2bad6f6", string(reviewResponse.UID)) +type badReader struct{} + +func (b badReader) Read(_ []byte) (n int, err error) { + return 0, errors.New("bad reader") } -func TestMutateFailsOnInvalidJson(t *testing.T) { - // given - rawJSON := []byte(`something wrong !`) +// Test handleMutate function +func TestHandleMutate(t *testing.T) { - // when - response := mutate(rawJSON) + t.Run("success", func(t *testing.T) { + // given + req, err := http.NewRequest("GET", "/mutate-whatever", bytes.NewBuffer(userPodsRawAdmissionReviewJSON)) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + expectedRespSuccess := expectedSuccessResponse{ + auditAnnotationKey: "users_pods_mutating_webhook", + auditAnnotationVal: "the sandbox-users-pods PriorityClass was set", + uid: "a68769e5-d817-4617-bec5-90efa2bad6f6", + } - // then - verifyFailedResponse(t, response, "cannot unmarshal string into Go value of type struct") -} + // when + handleMutate(testLogger, rr, req, fakeMutator()) -func TestMutateFailsOnInvalidPod(t *testing.T) { - // when - rawJSON := []byte(`{ - "request": { - "object": 111 + // then + assert.Equal(t, http.StatusOK, rr.Code) + verifySuccessfulResponse(t, rr.Body.Bytes(), expectedRespSuccess) + }) + + t.Run("fail to write response", func(t *testing.T) { + // given + req, err := http.NewRequest("GET", "/mutate-whatever", badReader{}) + if err != nil { + t.Fatal(err) } - }`) + rr := httptest.NewRecorder() - // when - response := mutate(rawJSON) + // when + handleMutate(testLogger, rr, req, fakeMutator()) - // then - verifyFailedResponse(t, response, "cannot unmarshal number into Go value of type v1.Pod") + // then + assert.Equal(t, http.StatusInternalServerError, rr.Code) + assert.Equal(t, "unable to read the body of the request", rr.Body.String()) + }) } -func verifyFailedResponse(t *testing.T, response []byte, errMsg string) { +func verifySuccessfulResponse(t *testing.T, response []byte, expectedResp expectedSuccessResponse) { + reviewResponse := toReviewResponse(t, response) + assert.Equal(t, expectedResp.patch, string(reviewResponse.Patch)) + assert.Contains(t, expectedResp.auditAnnotationVal, reviewResponse.AuditAnnotations[expectedResp.auditAnnotationKey]) + assert.True(t, reviewResponse.Allowed) + assert.Equal(t, admissionv1.PatchTypeJSONPatch, *reviewResponse.PatchType) + assert.Empty(t, reviewResponse.Result) + assert.Equal(t, expectedResp.uid, string(reviewResponse.UID)) +} + +func verifyFailedResponse(t *testing.T, response []byte, expectedResp expectedFailedResponse) { reviewResponse := toReviewResponse(t, response) assert.Empty(t, string(reviewResponse.Patch)) - assert.Empty(t, reviewResponse.AuditAnnotations["users_pods_mutating_webhook"]) + assert.Empty(t, reviewResponse.AuditAnnotations[expectedResp.auditAnnotationKey]) assert.False(t, reviewResponse.Allowed) assert.Nil(t, reviewResponse.PatchType) assert.Empty(t, string(reviewResponse.UID)) require.NotEmpty(t, reviewResponse.Result) - assert.Contains(t, reviewResponse.Result.Message, errMsg) + assert.Contains(t, reviewResponse.Result.Message, expectedResp.errMsg) } func toReviewResponse(t *testing.T, content []byte) *admissionv1.AdmissionResponse { @@ -94,149 +104,13 @@ func toReviewResponse(t *testing.T, content []byte) *admissionv1.AdmissionRespon return r.Response } -var rawJSON = []byte(`{ - "kind": "AdmissionReview", - "apiVersion": "admission.k8s.io/v1", - "request": { - "uid": "a68769e5-d817-4617-bec5-90efa2bad6f6", - "kind": { - "group": "", - "version": "v1", - "kind": "Pod" - }, - "resource": { - "group": "", - "version": "v1", - "resource": "pods" - }, - "requestKind": { - "group": "", - "version": "v1", - "kind": "Pod" - }, - "requestResource": { - "group": "", - "version": "v1", - "resource": "pods" - }, - "name": "busybox1", - "namespace": "johnsmith-dev", - "operation": "CREATE", - "userInfo": { - "username": "kube:admin", - "groups": [ - "system:cluster-admins", - "system:authenticated" - ], - "extra": { - "scopes.authorization.openshift.io": [ - "user:full" - ] - } - }, - "object": { - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "busybox1", - "namespace": "johnsmith-dev", - "creationTimestamp": null, - "labels": { - "app": "busybox1" - }, - "managedFields": [] - }, - "spec": { - "volumes": [ - { - "name": "default-token-8x9r5", - "secret": { - "secretName": "default-token-8x9r5" - } - } - ], - "containers": [ - { - "name": "busybox", - "image": "busybox", - "command": [ - "sleep", - "3600" - ], - "resources": { - "limits": { - "cpu": "150m", - "memory": "750Mi" - }, - "requests": { - "cpu": "10m", - "memory": "64Mi" - } - }, - "volumeMounts": [ - { - "name": "default-token-8x9r5", - "readOnly": true, - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" - } - ], - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "imagePullPolicy": "IfNotPresent", - "securityContext": { - "capabilities": { - "drop": [ - "MKNOD" - ] - } - } - } - ], - "restartPolicy": "Always", - "terminationGracePeriodSeconds": 30, - "dnsPolicy": "ClusterFirst", - "serviceAccountName": "default", - "serviceAccount": "default", - "securityContext": { - "seLinuxOptions": { - "level": "s0:c30,c0" - } - }, - "imagePullSecrets": [ - { - "name": "default-dockercfg-k6xlc" - } - ], - "schedulerName": "default-scheduler", - "tolerations": [ - { - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "effect": "NoExecute", - "tolerationSeconds": 300 - }, - { - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "effect": "NoExecute", - "tolerationSeconds": 300 - }, - { - "key": "node.kubernetes.io/memory-pressure", - "operator": "Exists", - "effect": "NoSchedule" - } - ], - "priority": 0, - "enableServiceLinks": true - }, - "status": {} - }, - "oldObject": null, - "dryRun": false, - "options": { - "kind": "CreateOptions", - "apiVersion": "meta.k8s.io/v1" - } - } -}`) +func fakeMutator() mutateHandler { + return func(admReview admissionv1.AdmissionReview) *admissionv1.AdmissionResponse { + patchType := admissionv1.PatchTypeJSONPatch + return &admissionv1.AdmissionResponse{ + Allowed: true, + UID: admReview.Request.UID, + PatchType: &patchType, + } + } +} diff --git a/pkg/webhook/mutatingwebhook/userpods_mutate.go b/pkg/webhook/mutatingwebhook/userpods_mutate.go new file mode 100644 index 00000000..b174087e --- /dev/null +++ b/pkg/webhook/mutatingwebhook/userpods_mutate.go @@ -0,0 +1,74 @@ +package mutatingwebhook + +import ( + "encoding/json" + "net/http" + "os" + + "github.com/pkg/errors" + v1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var ( + podLogger = logf.Log.WithName("users_pods_mutating_webhook") + + podPatchItems = []map[string]interface{}{ + { + "op": "replace", + "path": "/spec/priorityClassName", + "value": priorityClassName, + }, + { + "op": "replace", + "path": "/spec/priority", + "value": priority, + }, + } + + patchedContent []byte +) + +const ( + priority = int32(-3) + priorityClassName = "sandbox-users-pods" +) + +func init() { + var err error + patchedContent, err = json.Marshal(podPatchItems) + if err != nil { + podLogger.Error(err, "unable to marshal patch items") + os.Exit(1) + } +} + +func HandleMutateUserPods(w http.ResponseWriter, r *http.Request) { + handleMutate(podLogger, w, r, podMutator) +} + +func podMutator(admReview v1.AdmissionReview) *v1.AdmissionResponse { + // let's unmarshal the object to be sure that it's a pod + var pod *corev1.Pod + if err := json.Unmarshal(admReview.Request.Object.Raw, &pod); err != nil { + podLogger.Error(err, "unable unmarshal pod json object", "AdmissionReview", admReview) + return responseWithError(errors.Wrapf(err, "unable unmarshal pod json object - raw request object: %v", admReview.Request.Object.Raw)) + } + + patchType := v1.PatchTypeJSONPatch + resp := &v1.AdmissionResponse{ + Allowed: true, + UID: admReview.Request.UID, + PatchType: &patchType, + } + resp.AuditAnnotations = map[string]string{ + "users_pods_mutating_webhook": "the sandbox-users-pods PriorityClass was set", + } + + // instead of changing the pod object we need to tell K8s how to change the object + resp.Patch = patchedContent + + podLogger.Info("the sandbox-users-pods PriorityClass was set to the pod", "pod-name", pod.Name, "namespace", pod.Namespace) + return resp +} diff --git a/pkg/webhook/mutatingwebhook/userpods_mutate_test.go b/pkg/webhook/mutatingwebhook/userpods_mutate_test.go new file mode 100644 index 00000000..f25af597 --- /dev/null +++ b/pkg/webhook/mutatingwebhook/userpods_mutate_test.go @@ -0,0 +1,226 @@ +package mutatingwebhook + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var expectedMutatePodsRespSuccess = expectedSuccessResponse{ + patch: `[{"op":"replace","path":"/spec/priorityClassName","value":"sandbox-users-pods"},{"op":"replace","path":"/spec/priority","value":-3}]`, + auditAnnotationKey: "users_pods_mutating_webhook", + auditAnnotationVal: "the sandbox-users-pods PriorityClass was set", + uid: "a68769e5-d817-4617-bec5-90efa2bad6f6", +} + +func TestHandleMutateUserPodsSuccess(t *testing.T) { + // given + ts := httptest.NewServer(http.HandlerFunc(HandleMutateUserPods)) + defer ts.Close() + + // when + resp, err := http.Post(ts.URL, "application/json", bytes.NewBuffer(userPodsRawAdmissionReviewJSON)) + + // then + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + defer func() { + require.NoError(t, resp.Body.Close()) + }() + assert.NoError(t, err) + verifySuccessfulResponse(t, body, expectedMutatePodsRespSuccess) +} + +func TestMutateUserPodsSuccess(t *testing.T) { + // when + response := mutate(podLogger, userPodsRawAdmissionReviewJSON, podMutator) + + // then + verifySuccessfulResponse(t, response, expectedMutatePodsRespSuccess) +} + +func TestMutateUserPodsFailsOnInvalidJson(t *testing.T) { + // given + rawJSON := []byte(`something wrong !`) + var expectedResp = expectedFailedResponse{ + auditAnnotationKey: "users_pods_mutating_webhook", + errMsg: "cannot unmarshal string into Go value of type struct", + } + + // when + response := mutate(podLogger, rawJSON, podMutator) + + // then + verifyFailedResponse(t, response, expectedResp) +} + +func TestMutateUserPodsFailsOnInvalidPod(t *testing.T) { + // when + rawJSON := []byte(`{ + "request": { + "object": 111 + } + }`) + + // when + response := mutate(podLogger, rawJSON, podMutator) + + // then + var expectedResp = expectedFailedResponse{ + auditAnnotationKey: "users_pods_mutating_webhook", + errMsg: "cannot unmarshal number into Go value of type v1.Pod", + } + verifyFailedResponse(t, response, expectedResp) +} + +var userPodsRawAdmissionReviewJSON = []byte(`{ + "kind": "AdmissionReview", + "apiVersion": "admission.k8s.io/v1", + "request": { + "uid": "a68769e5-d817-4617-bec5-90efa2bad6f6", + "kind": { + "group": "", + "version": "v1", + "kind": "Pod" + }, + "resource": { + "group": "", + "version": "v1", + "resource": "pods" + }, + "requestKind": { + "group": "", + "version": "v1", + "kind": "Pod" + }, + "requestResource": { + "group": "", + "version": "v1", + "resource": "pods" + }, + "name": "busybox1", + "namespace": "johnsmith-dev", + "operation": "CREATE", + "userInfo": { + "username": "kube:admin", + "groups": [ + "system:cluster-admins", + "system:authenticated" + ], + "extra": { + "scopes.authorization.openshift.io": [ + "user:full" + ] + } + }, + "object": { + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "busybox1", + "namespace": "johnsmith-dev", + "creationTimestamp": null, + "labels": { + "app": "busybox1" + }, + "managedFields": [] + }, + "spec": { + "volumes": [ + { + "name": "default-token-8x9r5", + "secret": { + "secretName": "default-token-8x9r5" + } + } + ], + "containers": [ + { + "name": "busybox", + "image": "busybox", + "command": [ + "sleep", + "3600" + ], + "resources": { + "limits": { + "cpu": "150m", + "memory": "750Mi" + }, + "requests": { + "cpu": "10m", + "memory": "64Mi" + } + }, + "volumeMounts": [ + { + "name": "default-token-8x9r5", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "drop": [ + "MKNOD" + ] + } + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "serviceAccountName": "default", + "serviceAccount": "default", + "securityContext": { + "seLinuxOptions": { + "level": "s0:c30,c0" + } + }, + "imagePullSecrets": [ + { + "name": "default-dockercfg-k6xlc" + } + ], + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, + { + "key": "node.kubernetes.io/memory-pressure", + "operator": "Exists", + "effect": "NoSchedule" + } + ], + "priority": 0, + "enableServiceLinks": true + }, + "status": {} + }, + "oldObject": null, + "dryRun": false, + "options": { + "kind": "CreateOptions", + "apiVersion": "meta.k8s.io/v1" + } + } + }`) diff --git a/pkg/webhook/mutatingwebhook/vm_mutate.go b/pkg/webhook/mutatingwebhook/vm_mutate.go new file mode 100644 index 00000000..9e087cb9 --- /dev/null +++ b/pkg/webhook/mutatingwebhook/vm_mutate.go @@ -0,0 +1,101 @@ +package mutatingwebhook + +import ( + "encoding/json" + "net/http" + + "github.com/pkg/errors" + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var vmLogger = logf.Log.WithName("virtual_machines_mutating_webhook") + +func HandleMutateVirtualMachines(w http.ResponseWriter, r *http.Request) { + handleMutate(vmLogger, w, r, vmMutator) +} + +func vmMutator(admReview admissionv1.AdmissionReview) *admissionv1.AdmissionResponse { + + unstructuredObj := &unstructured.Unstructured{} + if err := unstructuredObj.UnmarshalJSON(admReview.Request.Object.Raw); err != nil { + vmLogger.Error(err, "unable unmarshal VirtualMachine json object", "AdmissionReview", admReview) + return responseWithError(errors.Wrapf(err, "unable unmarshal VirtualMachine json object - raw request object: %v", admReview.Request.Object.Raw)) + } + + patchType := admissionv1.PatchTypeJSONPatch + resp := &admissionv1.AdmissionResponse{ + Allowed: true, + UID: admReview.Request.UID, + PatchType: &patchType, + } + resp.AuditAnnotations = map[string]string{ + "virtual_machines_mutating_webhook": "the resource limits were set", + } + + // instead of changing the object we need to tell K8s how to change the object + vmPatchItems := []map[string]interface{}{} + + // ensure limits are set + vmPatchItems = ensureLimits(unstructuredObj, vmPatchItems) + + patchContent, err := json.Marshal(vmPatchItems) + if err != nil { + vmLogger.Error(err, "unable to marshal patch items for VirtualMachine", "AdmissionReview", admReview, "Patch-Items", vmPatchItems) + return responseWithError(errors.Wrapf(err, "unable to marshal patch items for VirtualMachine - raw request object: %v", admReview.Request.Object.Raw)) + } + resp.Patch = patchContent + + vmLogger.Info("the resource limits were set on the VirtualMachine", "vm-name", unstructuredObj.GetName(), "namespace", unstructuredObj.GetNamespace()) + return resp +} + +// ensureLimits ensures resource limits are set on the VirtualMachine if requests are set, this is a workaround for https://issues.redhat.com/browse/CNV-28746 (https://issues.redhat.com/browse/CNV-32069) +// The issue is that if the namespace has LimitRanges defined and the VirtualMachine resource does not have resource limits defined then it will use the LimitRanges which may be less than requested +// resources and the VirtualMachine will fail to start. +// This should be removed once https://issues.redhat.com/browse/CNV-32069 is complete. +func ensureLimits(unstructuredObj *unstructured.Unstructured, patchItems []map[string]interface{}) []map[string]interface{} { + + requests, reqFound, err := unstructured.NestedStringMap(unstructuredObj.Object, "spec", "template", "spec", "domain", "resources", "requests") + if err != nil { + vmLogger.Error(err, "unable to get requests from VirtualMachine", "VirtualMachine", unstructuredObj) + return patchItems + } + + if !reqFound { + return patchItems + } + + limits, limFound, err := unstructured.NestedStringMap(unstructuredObj.Object, "spec", "template", "spec", "domain", "resources", "limits") + if err != nil { + vmLogger.Error(err, "unable to get limits from VirtualMachine", "VirtualMachine", unstructuredObj) + return patchItems + } + + if limits == nil || !limFound { + limits = map[string]string{} + } + + // if the limit is not defined but the request is, then set the limit to the same value as the request + anyChanges := false + for _, r := range []string{"memory", "cpu"} { + _, isLimitDefined := limits[r] + _, isRequestDefined := requests[r] + if !isLimitDefined && isRequestDefined { + limits[r] = requests[r] + anyChanges = true + } + } + + if anyChanges { + patchItems = append(patchItems, + map[string]interface{}{ + "op": "add", + "path": "/spec/template/spec/domain/resources/limits", + "value": limits, + }) + vmLogger.Info("setting resource limits on the virtual machine", "vm-name", unstructuredObj.GetName(), "namespace", unstructuredObj.GetNamespace(), "limits", limits) + } + return patchItems +} diff --git a/pkg/webhook/mutatingwebhook/vm_mutate_test.go b/pkg/webhook/mutatingwebhook/vm_mutate_test.go new file mode 100644 index 00000000..95b0a02e --- /dev/null +++ b/pkg/webhook/mutatingwebhook/vm_mutate_test.go @@ -0,0 +1,367 @@ +package mutatingwebhook + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestMutateVMSuccess(t *testing.T) { + + t.Run("no requests set", func(t *testing.T) { + // given + vmAdmReview := vmAdmissionReview(t, nil, nil) + expectedResp := vmSuccessResponse() + + // when + response := mutate(podLogger, vmAdmReview, vmMutator) + + // then + verifySuccessfulResponse(t, response, expectedResp) + }) + + t.Run("only memory request is set", func(t *testing.T) { + // given + req := resourceList("1Gi", "") + vmAdmReview := vmAdmissionReview(t, req, nil) + expectedLimits := resourceList("1Gi", "") + expectedResp := vmSuccessResponse(withPatch(t, expectedLimits)) + + // when + response := mutate(podLogger, vmAdmReview, vmMutator) + + // then + verifySuccessfulResponse(t, response, expectedResp) + }) + + t.Run("memory and cpu requests are set", func(t *testing.T) { + // given + req := resourceList("1Gi", "1") + vmAdmReview := vmAdmissionReview(t, req, nil) + expectedLimits := resourceList("1Gi", "1") + expectedResp := vmSuccessResponse(withPatch(t, expectedLimits)) + + // when + response := mutate(podLogger, vmAdmReview, vmMutator) + + // then + verifySuccessfulResponse(t, response, expectedResp) + }) + + t.Run("memory and cpu requests are set but both limits are already set", func(t *testing.T) { + // given + req := resourceList("1Gi", "1") + lim := resourceList("2Gi", "2") + vmAdmReview := vmAdmissionReview(t, req, lim) + expectedResp := vmSuccessResponse() // no patch expected because limits are already set + + // when + response := mutate(podLogger, vmAdmReview, vmMutator) + + // then + verifySuccessfulResponse(t, response, expectedResp) + }) + + t.Run("memory and cpu requests are set but memory limit is already set", func(t *testing.T) { + // given + req := resourceList("1Gi", "1") + lim := resourceList("2Gi", "") + vmAdmReview := vmAdmissionReview(t, req, lim) + expectedLimits := resourceList("2Gi", "1") // expect cpu limit to be set to the value of the cpu request + expectedResp := vmSuccessResponse(withPatch(t, expectedLimits)) + + // when + response := mutate(podLogger, vmAdmReview, vmMutator) + + // then + verifySuccessfulResponse(t, response, expectedResp) + }) + + t.Run("memory and cpu requests are set but cpu limit is already set", func(t *testing.T) { + // given + req := resourceList("1Gi", "1") + lim := resourceList("", "2") + vmAdmReview := vmAdmissionReview(t, req, lim) + expectedLimits := resourceList("1Gi", "2") // expect memory limit to be set to the value of the memory request + expectedResp := vmSuccessResponse(withPatch(t, expectedLimits)) + + // when + response := mutate(podLogger, vmAdmReview, vmMutator) + + // then + verifySuccessfulResponse(t, response, expectedResp) + }) +} + +func TestMutateVMsFailsOnInvalidJson(t *testing.T) { + // given + rawJSON := []byte(`something wrong !`) + expectedResp := expectedFailedResponse{ + auditAnnotationKey: "virtual_machines_mutating_webhook", + errMsg: "cannot unmarshal string into Go value of type struct", + } + + // when + response := mutate(vmLogger, rawJSON, vmMutator) + + // then + verifyFailedResponse(t, response, expectedResp) +} + +func TestMutateVmmFailsOnInvalidVM(t *testing.T) { + // when + rawJSON := []byte(`{ + "request": { + "object": 111 + } + }`) + expectedResp := expectedFailedResponse{ + auditAnnotationKey: "virtual_machines_mutating_webhook", + errMsg: "cannot unmarshal number into Go value of type map[string]interface {}", + } + + // when + response := mutate(vmLogger, rawJSON, vmMutator) + + // then + verifyFailedResponse(t, response, expectedResp) +} + +type vmSuccessResponseOption func(*expectedSuccessResponse) + +func withPatch(t *testing.T, expectedLimits map[string]interface{}) vmSuccessResponseOption { + return func(resp *expectedSuccessResponse) { + expectedLimitsJSONBytes, err := json.Marshal(expectedLimits) + require.NoError(t, err) + expectedLimitsJSON := string(expectedLimitsJSONBytes) + resp.patch = fmt.Sprintf(`[{"op":"add","path":"/spec/template/spec/domain/resources/limits","value":%s}]`, expectedLimitsJSON) + } +} + +func vmSuccessResponse(options ...vmSuccessResponseOption) expectedSuccessResponse { + resp := &expectedSuccessResponse{ + patch: "[]", + auditAnnotationKey: "virtual_machines_mutating_webhook", + auditAnnotationVal: "the resource limits were set", + uid: "d68b4f8c-c62d-4e83-bd73-de991ab8a56a", + } + + for _, opt := range options { + opt(resp) + } + + return *resp +} + +func vmAdmissionReview(t *testing.T, requests, limits map[string]interface{}) []byte { + unstructuredAdmReview := &unstructured.Unstructured{} + err := unstructuredAdmReview.UnmarshalJSON([]byte(vmRawAdmissionReviewJSONTemplate)) + require.NoError(t, err) + + // set requests + if requests != nil { + err := unstructured.SetNestedMap(unstructuredAdmReview.Object, requests, "request", "object", "spec", "template", "spec", "domain", "resources", "requests") + require.NoError(t, err) + } + + // set limits + if limits != nil { + err := unstructured.SetNestedMap(unstructuredAdmReview.Object, limits, "request", "object", "spec", "template", "spec", "domain", "resources", "limits") + require.NoError(t, err) + } + + admReviewJSON, err := unstructuredAdmReview.MarshalJSON() + require.NoError(t, err) + + return admReviewJSON +} + +func resourceList(mem, cpu string) map[string]interface{} { + req := map[string]interface{}{} + if mem != "" { + req["memory"] = mem + } + if cpu != "" { + req["cpu"] = cpu + } + return req +} + +var vmRawAdmissionReviewJSONTemplate = `{ + "kind": "AdmissionReview", + "apiVersion": "admission.k8s.io/v1", + "request": { + "uid": "d68b4f8c-c62d-4e83-bd73-de991ab8a56a", + "kind": { + "group": "kubevirt.io", + "version": "v1", + "kind": "VirtualMachine" + }, + "resource": { + "group": "kubevirt.io", + "version": "v1", + "resource": "virtualmachines" + }, + "requestKind": { + "group": "kubevirt.io", + "version": "v1", + "kind": "VirtualMachine" + }, + "requestResource": { + "group": "kubevirt.io", + "version": "v1", + "resource": "virtualmachines" + }, + "name": "rhel9-test", + "namespace": "userabc-dev", + "operation": "CREATE", + "userInfo": { + "username": "system:admin", + "groups": [ + "system:masters", + "system:authenticated" + ] + }, + "object": { + "apiVersion": "kubevirt.io/v1", + "kind": "VirtualMachine", + "metadata": { + "labels": { + "app": "rhel9-test", + "vm.kubevirt.io/template": "rhel9-server-small", + "vm.kubevirt.io/template.namespace": "openshift", + "vm.kubevirt.io/template.revision": "1", + "vm.kubevirt.io/template.version": "v0.25.0" + }, + "name": "rhel9-test", + "namespace": "userabc-dev" + }, + "spec": { + "dataVolumeTemplates": [ + { + "apiVersion": "cdi.kubevirt.io/v1beta1", + "kind": "DataVolume", + "metadata": { + "creationTimestamp": null, + "name": "rhel9-test" + }, + "spec": { + "sourceRef": { + "kind": "DataSource", + "name": "rhel9", + "namespace": "openshift-virtualization-os-images" + }, + "storage": { + "resources": { + "requests": { + "storage": "30Gi" + } + } + } + } + } + ], + "running": true, + "template": { + "metadata": { + "annotations": { + "vm.kubevirt.io/flavor": "small", + "vm.kubevirt.io/os": "rhel9", + "vm.kubevirt.io/workload": "server" + }, + "creationTimestamp": null, + "labels": { + "kubevirt.io/domain": "rhel9-test", + "kubevirt.io/size": "small" + } + }, + "spec": { + "domain": { + "cpu": { + "cores": 1, + "sockets": 1, + "threads": 1 + }, + "devices": { + "disks": [ + { + "disk": { + "bus": "virtio" + }, + "name": "rootdisk" + }, + { + "disk": { + "bus": "virtio" + }, + "name": "cloudinitdisk" + } + ], + "interfaces": [ + { + "macAddress": "02:24:d5:00:00:00", + "masquerade": {}, + "model": "virtio", + "name": "default" + } + ], + "networkInterfaceMultiqueue": true, + "rng": {} + }, + "features": { + "acpi": {}, + "smm": { + "enabled": true + } + }, + "firmware": { + "bootloader": { + "efi": {} + } + }, + "machine": { + "type": "pc-q35-rhel9.2.0" + }, + "resources": { + } + }, + "evictionStrategy": "LiveMigrate", + "networks": [ + { + "name": "default", + "pod": {} + } + ], + "terminationGracePeriodSeconds": 180, + "volumes": [ + { + "dataVolume": { + "name": "rhel9-test" + }, + "name": "rootdisk" + }, + { + "cloudInitNoCloud": { + "userData": "#cloud-config\nuser: cloud-user\npassword: 5as2-8nbk-7a4c\nchpasswd: { expire: False }" + }, + "name": "cloudinitdisk" + } + ] + } + } + } + }, + "oldObject": null, + "dryRun": false, + "options": { + "kind": "CreateOptions", + "apiVersion": "meta.k8s.io/v1", + "fieldManager": "kubectl-client-side-apply", + "fieldValidation": "Ignore" + } + } +}`