From 852b61b89d65e41cad5f3d3faf53306f46c9c716 Mon Sep 17 00:00:00 2001 From: Max McAdam Date: Fri, 7 Jun 2024 11:24:18 -0700 Subject: [PATCH 1/3] Agent cancels agreement after built-in property change Signed-off-by: Max McAdam --- agreement/agreement.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/agreement/agreement.go b/agreement/agreement.go index 986f932f0..c329ef409 100644 --- a/agreement/agreement.go +++ b/agreement/agreement.go @@ -648,9 +648,11 @@ func (w *AgreementWorker) syncOnInit() error { } else if proposal, err := w.producerPH[ag.AgreementProtocol].AgreementProtocolHandler("", "", "").DemarshalProposal(ag.Proposal); err != nil { glog.Errorf(logString(fmt.Sprintf("unable to demarshal proposal for agreement %v, error %v", ag.CurrentAgreementId, err))) - } else if pol, err := policy.DemarshalPolicy(proposal.ProducerPolicy()); err != nil { - glog.Errorf(logString(fmt.Sprintf("unable to demarshal policy for agreement %v, error %v", ag.CurrentAgreementId, err))) + } else if nodePol, err := persistence.FindNodePolicy(w.db); err != nil { + glog.Errorf(logString(fmt.Sprintf("unable to read node policy from the local database. %v", err))) + } else if pol, err := policy.GenPolicyFromExternalPolicy(nodePol.GetDeploymentPolicy(), policy.MakeExternalPolicyHeaderName(w.GetExchangeId())); err != nil { + glog.Errorf(logString(fmt.Sprintf("Failed to convert node policy to policy file format: %v", err))) } else if policies, err := w.pm.GetPolicyList(exchange.GetOrg(w.GetExchangeId()), pol); err != nil { glog.Errorf(logString(fmt.Sprintf("unable to get policy list for producer policy in agreement %v, error: %v", ag.CurrentAgreementId, err))) w.Messages() <- events.NewInitAgreementCancelationMessage(events.AGREEMENT_ENDED, w.producerPH[ag.AgreementProtocol].GetTerminationCode(producer.TERM_REASON_POLICY_CHANGED), ag.AgreementProtocol, ag.CurrentAgreementId, ag.GetDeploymentConfig()) From 143b28bd709babf7b91ebd4ef5a5220539f6a507 Mon Sep 17 00:00:00 2001 From: Max McAdam Date: Mon, 17 Jun 2024 13:06:09 -0700 Subject: [PATCH 2/3] Issue 4084 - Bug: failed to create k8s image tar file Signed-off-by: Max McAdam --- clusterupgrade/cluster_install_files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clusterupgrade/cluster_install_files.go b/clusterupgrade/cluster_install_files.go index 0b25a2258..3bc910c94 100644 --- a/clusterupgrade/cluster_install_files.go +++ b/clusterupgrade/cluster_install_files.go @@ -515,7 +515,7 @@ func addFiles(w *tar.Writer, srcPath, base string) error { } } else if fileInfo.IsDir() { // Recurse - newBase := fileInfo.Name() + "/" + newBase := base + fileInfo.Name() + "/" err = addFiles(w, srcPath, newBase) if err != nil { return err From 6e8c4ae7c1b397366a490703096aa3b64b60bfa9 Mon Sep 17 00:00:00 2001 From: Spencer Whaley Date: Fri, 7 Jun 2024 11:07:47 -0700 Subject: [PATCH 3/3] Issue 4081 - Make kube worker aware of Secret kube resource type Before, secrets were considered and UNSTRUCTURED type, and created last. This is problematic for Deployment resources that may depend on an image pull secret being created first. Signed-off-by: Spencer Whaley --- kube_operator/api_objects.go | 91 +++++++++++++++++++++++++++++++++++- kube_operator/client.go | 27 ++++++----- 2 files changed, 105 insertions(+), 13 deletions(-) diff --git a/kube_operator/api_objects.go b/kube_operator/api_objects.go index 616bb2787..90a728ee6 100644 --- a/kube_operator/api_objects.go +++ b/kube_operator/api_objects.go @@ -4,6 +4,9 @@ import ( "context" "encoding/base64" "fmt" + "strings" + "time" + "github.com/golang/glog" "github.com/open-horizon/anax/config" "github.com/open-horizon/anax/cutil" @@ -20,8 +23,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" dynamic "k8s.io/client-go/dynamic" - "strings" - "time" ) type APIObjectInterface interface { @@ -114,6 +115,25 @@ func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstr } else { return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: rolebinding object has unrecognized type %T: %v", obj.Object, obj.Object))) } + case K8S_SECRET_TYPE: + if typedSecret, ok := obj.Object.(*corev1.Secret); ok { + if typedSecret.ObjectMeta.Namespace != "" { + if namespace == "" { + namespace = typedSecret.ObjectMeta.Namespace + } else if namespace != typedSecret.ObjectMeta.Namespace { + return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: multiple namespaces specified in operator: %s and %s", namespace, typedSecret.ObjectMeta.Namespace))) + } + } + newSecret := SecretCoreV1{SecretObject: typedSecret} + if newSecret.Name() != "" { + glog.V(4).Infof(kwlog(fmt.Sprintf("Found kubernetes secret object %s.", newSecret.Name()))) + objMap[K8S_SECRET_TYPE] = append(objMap[K8S_SECRET_TYPE], newSecret) + } else { + return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: secret object must have a name in its metadata section."))) + } + } else { + return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: secret object has unrecognized type %T: %v", obj.Object, obj.Object))) + } case K8S_DEPLOYMENT_TYPE: if typedDeployment, ok := obj.Object.(*appsv1.Deployment); ok { if typedDeployment.ObjectMeta.Namespace != "" { @@ -623,6 +643,73 @@ func (sa ServiceAccountCoreV1) Namespace() string { return sa.ServiceAccountObject.ObjectMeta.Namespace } +// ----------------Secret---------------- +type SecretCoreV1 struct { + SecretObject *corev1.Secret +} + +func (s SecretCoreV1) Install(c KubeClient, namespace string) error { + glog.V(3).Infof(kwlog(fmt.Sprintf("attempting to create secret %v", s.Name()))) // Don't display the whole object to avoid logging secret data + if namespace != s.Namespace() { + glog.Warningf(kwlog(fmt.Sprintf("Embedded namespace '%v' is ignored. Service will be deployed to '%v'.", s.Namespace(), namespace))) + s.SecretObject.ObjectMeta.Namespace = namespace + } + + _, err := c.Client.CoreV1().Secrets(namespace).Create(context.Background(), s.SecretObject, metav1.CreateOptions{}) + if err != nil && errors.IsAlreadyExists(err) { + glog.Warningf(kwlog(fmt.Sprintf("Secret %s already exists, deleting and re-creating", s.Name()))) + s.Uninstall(c, s.Name()) + _, err = c.Client.CoreV1().Secrets(namespace).Create(context.Background(), s.SecretObject, metav1.CreateOptions{}) + } + if err != nil { + return fmt.Errorf(kwlog(fmt.Sprintf("Error creating the secret: %v", err))) + } + return nil +} + +func (s SecretCoreV1) Uninstall(c KubeClient, secretName string) { + glog.V(3).Infof(kwlog(fmt.Sprintf("deleting secret %v", secretName))) + err := c.Client.CoreV1().Secrets(s.Namespace()).Delete(context.Background(), secretName, metav1.DeleteOptions{}) + if err != nil { + glog.Errorf(kwlog(fmt.Sprintf("unable to delete secret %s. Error: %v", secretName, err))) + } +} + +func (s SecretCoreV1) Status(c KubeClient, namespace string) (interface{}, error) { + secretFromKube, err := c.Client.CoreV1().Secrets(namespace).Get(context.Background(), s.Name(), metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf(kwlog(fmt.Sprintf("Error getting secret status: %v", err))) + } + return secretFromKube, nil +} + +func (s SecretCoreV1) Update(c KubeClient, namespace string) error { + if secretFromKube, err := c.Client.CoreV1().Secrets(namespace).Get(context.Background(), s.SecretObject.Name, metav1.GetOptions{}); err != nil { + return fmt.Errorf(kwlog(fmt.Sprintf("Error getting secret from kube: %v", err))) + } else if secretFromKube == nil { + // invalid, return err + return fmt.Errorf(kwlog(fmt.Sprintf("Error updating secret %v in namespace %v: secret doesn't exist", s.SecretObject.Name, namespace))) + } else { + //Update the secret retrieved from kube with the new data + secretFromKube.Data = s.SecretObject.Data + updatedSecret, err := c.Client.CoreV1().Secrets(namespace).Update(context.Background(), secretFromKube, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf(kwlog(fmt.Sprintf("Error updating secret %v in namespace %v: %v", s.SecretObject.Name, namespace, err))) + } + glog.V(3).Infof(kwlog(fmt.Sprintf("Secret %v in namespace %v updated successfully", updatedSecret.Name, namespace))) + } + + return nil +} + +func (s SecretCoreV1) Name() string { + return s.SecretObject.ObjectMeta.Name +} + +func (s SecretCoreV1) Namespace() string { + return s.SecretObject.ObjectMeta.Namespace +} + //----------------Deployment---------------- // The deployment object includes the environment variable config map and vault secret as k8s secret diff --git a/kube_operator/client.go b/kube_operator/client.go index d66ee880d..8fde611e3 100644 --- a/kube_operator/client.go +++ b/kube_operator/client.go @@ -6,6 +6,13 @@ import ( "context" "encoding/base64" "fmt" + "io" + "io/ioutil" + "os" + "reflect" + "strconv" + "strings" + "github.com/golang/glog" "github.com/open-horizon/anax/config" "github.com/open-horizon/anax/cutil" @@ -15,8 +22,6 @@ import ( olmv1client "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1" olmv1alpha1client "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1alpha1" yaml "gopkg.in/yaml.v2" - "io" - "io/ioutil" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -34,10 +39,6 @@ import ( dynamic "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" - "os" - "reflect" - "strconv" - "strings" ) const ( @@ -61,6 +62,7 @@ const ( K8S_SERVICEACCOUNT_TYPE = "ServiceAccount" K8S_CRD_TYPE = "CustomResourceDefinition" K8S_NAMESPACE_TYPE = "Namespace" + K8S_SECRET_TYPE = "Secret" K8S_UNSTRUCTURED_TYPE = "Unstructured" K8S_OLM_OPERATOR_GROUP_TYPE = "OperatorGroup" K8S_MMS_SHARED_PVC_NAME = "mms-shared-storage-pvc" @@ -77,8 +79,11 @@ var ( } ) +// Order is important here since it will be used to determine install order. +// For example, secrets should be before deployments, +// because it may be an image pull secret used by the deployment func getBaseK8sKinds() []string { - return []string{K8S_NAMESPACE_TYPE, K8S_CLUSTER_ROLE_TYPE, K8S_CLUSTER_ROLEBINDING_TYPE, K8S_ROLE_TYPE, K8S_ROLEBINDING_TYPE, K8S_SERVICEACCOUNT_TYPE, K8S_CRD_TYPE, K8S_DEPLOYMENT_TYPE} + return []string{K8S_NAMESPACE_TYPE, K8S_CLUSTER_ROLE_TYPE, K8S_CLUSTER_ROLEBINDING_TYPE, K8S_ROLE_TYPE, K8S_ROLEBINDING_TYPE, K8S_SERVICEACCOUNT_TYPE, K8S_SECRET_TYPE, K8S_CRD_TYPE, K8S_DEPLOYMENT_TYPE} } func getDangerKinds() []string { @@ -163,10 +168,10 @@ func (c KubeClient) Install(tar string, metadata map[string]interface{}, mmsPVCC if namespace != nodeNamespace && nodeIsNamespaceScope { return fmt.Errorf("Service failed to start for agreement %v. Could not deploy service into namespace %v because the agent's namespace is namespace scoped, and it restricts all services to the agent namespace %v", agId, namespace, nodeNamespace) } else if namespace != nodeNamespace { - // create network policies to allow traffic between the node and service + // create network policies to allow traffic between the node and service ingress := networkingv1.NetworkPolicyIngressRule{From: []networkingv1.NetworkPolicyPeer{networkingv1.NetworkPolicyPeer{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace}}}}} egress := networkingv1.NetworkPolicyEgressRule{To: []networkingv1.NetworkPolicyPeer{networkingv1.NetworkPolicyPeer{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace}}}}} - spec := networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{}, Ingress: []networkingv1.NetworkPolicyIngressRule{ingress}, Egress: []networkingv1.NetworkPolicyEgressRule{egress}, PolicyTypes: []networkingv1.PolicyType{"Ingress","Egress"}} + spec := networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{}, Ingress: []networkingv1.NetworkPolicyIngressRule{ingress}, Egress: []networkingv1.NetworkPolicyEgressRule{egress}, PolicyTypes: []networkingv1.PolicyType{"Ingress", "Egress"}} netPol := networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-networkPolicy", agId), Namespace: nodeNamespace}, Spec: spec} _, err := c.Client.NetworkingV1().NetworkPolicies(nodeNamespace).Create(context.Background(), &netPol, metav1.CreateOptions{}) if err != nil { @@ -233,8 +238,8 @@ func (c KubeClient) Uninstall(tar string, metadata map[string]interface{}, agId nsObj := corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace"}, ObjectMeta: metav1.ObjectMeta{Name: namespace}} apiObjMap[K8S_NAMESPACE_TYPE] = []APIObjectInterface{NamespaceCoreV1{NamespaceObject: &nsObj}} } else if namespace != nodeNamespace { - // delete the network policy that allows traffic between the node and service - err := c.Client.NetworkingV1().NetworkPolicies(nodeNamespace).Delete(context.Background(), fmt.Sprintf("%s-networkPolicy", agId) , metav1.DeleteOptions{}) + // delete the network policy that allows traffic between the node and service + err := c.Client.NetworkingV1().NetworkPolicies(nodeNamespace).Delete(context.Background(), fmt.Sprintf("%s-networkPolicy", agId), metav1.DeleteOptions{}) if err != nil { glog.Errorf(kwlog(fmt.Sprintf("Error deleting network policy: %v", err))) }