Skip to content

Commit

Permalink
Issue 4081 - Make kube worker aware of Secret kube resource type
Browse files Browse the repository at this point in the history 
Before, secrets were considered and UNSTRUCTURED type, and created last. This is problematic for Deployment resources that may depend on an image pull secret being created first.

Signed-off-by: Spencer Whaley <swhaley@ibm.com>
  • Loading branch information
@scwhaley @MaxMcAdam
scwhaley authored and MaxMcAdam committed Jun 19, 2024
1 parent 143b28b commit 6e8c4ae
Show file tree
Hide file tree
Showing 2 changed files with 105 additions and 13 deletions.
91 changes: 89 additions & 2 deletions kube_operator/api_objects.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import (
"context"
"encoding/base64"
"fmt"
"strings"
"time"

"github.com/golang/glog"
"github.com/open-horizon/anax/config"
"github.com/open-horizon/anax/cutil"
Expand All @@ -20,8 +23,6 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
dynamic "k8s.io/client-go/dynamic"
"strings"
"time"
)

type APIObjectInterface interface {
Expand Down Expand Up @@ -114,6 +115,25 @@ func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstr
} else {
return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: rolebinding object has unrecognized type %T: %v", obj.Object, obj.Object)))
}
case K8S_SECRET_TYPE:
if typedSecret, ok := obj.Object.(*corev1.Secret); ok {
if typedSecret.ObjectMeta.Namespace != "" {
if namespace == "" {
namespace = typedSecret.ObjectMeta.Namespace
} else if namespace != typedSecret.ObjectMeta.Namespace {
return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: multiple namespaces specified in operator: %s and %s", namespace, typedSecret.ObjectMeta.Namespace)))
}
}
newSecret := SecretCoreV1{SecretObject: typedSecret}
if newSecret.Name() != "" {
glog.V(4).Infof(kwlog(fmt.Sprintf("Found kubernetes secret object %s.", newSecret.Name())))
objMap[K8S_SECRET_TYPE] = append(objMap[K8S_SECRET_TYPE], newSecret)
} else {
return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: secret object must have a name in its metadata section.")))
}
} else {
return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: secret object has unrecognized type %T: %v", obj.Object, obj.Object)))
}
case K8S_DEPLOYMENT_TYPE:
if typedDeployment, ok := obj.Object.(*appsv1.Deployment); ok {
if typedDeployment.ObjectMeta.Namespace != "" {
Expand Down Expand Up @@ -623,6 +643,73 @@ func (sa ServiceAccountCoreV1) Namespace() string {
return sa.ServiceAccountObject.ObjectMeta.Namespace
}

// ----------------Secret----------------
type SecretCoreV1 struct {
SecretObject *corev1.Secret
}

func (s SecretCoreV1) Install(c KubeClient, namespace string) error {
glog.V(3).Infof(kwlog(fmt.Sprintf("attempting to create secret %v", s.Name()))) // Don't display the whole object to avoid logging secret data
if namespace != s.Namespace() {
glog.Warningf(kwlog(fmt.Sprintf("Embedded namespace '%v' is ignored. Service will be deployed to '%v'.", s.Namespace(), namespace)))
s.SecretObject.ObjectMeta.Namespace = namespace
}

_, err := c.Client.CoreV1().Secrets(namespace).Create(context.Background(), s.SecretObject, metav1.CreateOptions{})
if err != nil && errors.IsAlreadyExists(err) {
glog.Warningf(kwlog(fmt.Sprintf("Secret %s already exists, deleting and re-creating", s.Name())))
s.Uninstall(c, s.Name())
_, err = c.Client.CoreV1().Secrets(namespace).Create(context.Background(), s.SecretObject, metav1.CreateOptions{})
}
if err != nil {
return fmt.Errorf(kwlog(fmt.Sprintf("Error creating the secret: %v", err)))
}
return nil
}

func (s SecretCoreV1) Uninstall(c KubeClient, secretName string) {
glog.V(3).Infof(kwlog(fmt.Sprintf("deleting secret %v", secretName)))
err := c.Client.CoreV1().Secrets(s.Namespace()).Delete(context.Background(), secretName, metav1.DeleteOptions{})
if err != nil {
glog.Errorf(kwlog(fmt.Sprintf("unable to delete secret %s. Error: %v", secretName, err)))
}
}

func (s SecretCoreV1) Status(c KubeClient, namespace string) (interface{}, error) {
secretFromKube, err := c.Client.CoreV1().Secrets(namespace).Get(context.Background(), s.Name(), metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf(kwlog(fmt.Sprintf("Error getting secret status: %v", err)))
}
return secretFromKube, nil
}

func (s SecretCoreV1) Update(c KubeClient, namespace string) error {
if secretFromKube, err := c.Client.CoreV1().Secrets(namespace).Get(context.Background(), s.SecretObject.Name, metav1.GetOptions{}); err != nil {
return fmt.Errorf(kwlog(fmt.Sprintf("Error getting secret from kube: %v", err)))
} else if secretFromKube == nil {
// invalid, return err
return fmt.Errorf(kwlog(fmt.Sprintf("Error updating secret %v in namespace %v: secret doesn't exist", s.SecretObject.Name, namespace)))
} else {
//Update the secret retrieved from kube with the new data
secretFromKube.Data = s.SecretObject.Data
updatedSecret, err := c.Client.CoreV1().Secrets(namespace).Update(context.Background(), secretFromKube, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf(kwlog(fmt.Sprintf("Error updating secret %v in namespace %v: %v", s.SecretObject.Name, namespace, err)))
}
glog.V(3).Infof(kwlog(fmt.Sprintf("Secret %v in namespace %v updated successfully", updatedSecret.Name, namespace)))
}

return nil
}

func (s SecretCoreV1) Name() string {
return s.SecretObject.ObjectMeta.Name
}

func (s SecretCoreV1) Namespace() string {
return s.SecretObject.ObjectMeta.Namespace
}

//----------------Deployment----------------
// The deployment object includes the environment variable config map and vault secret as k8s secret

Expand Down
27 changes: 16 additions & 11 deletions kube_operator/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ import (
"context"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"os"
"reflect"
"strconv"
"strings"

"github.com/golang/glog"
"github.com/open-horizon/anax/config"
"github.com/open-horizon/anax/cutil"
Expand All @@ -15,8 +22,6 @@ import (
olmv1client "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1"
olmv1alpha1client "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1alpha1"
yaml "gopkg.in/yaml.v2"
"io"
"io/ioutil"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
Expand All @@ -34,10 +39,6 @@ import (
dynamic "k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"os"
"reflect"
"strconv"
"strings"
)

const (
Expand All @@ -61,6 +62,7 @@ const (
K8S_SERVICEACCOUNT_TYPE = "ServiceAccount"
K8S_CRD_TYPE = "CustomResourceDefinition"
K8S_NAMESPACE_TYPE = "Namespace"
K8S_SECRET_TYPE = "Secret"
K8S_UNSTRUCTURED_TYPE = "Unstructured"
K8S_OLM_OPERATOR_GROUP_TYPE = "OperatorGroup"
K8S_MMS_SHARED_PVC_NAME = "mms-shared-storage-pvc"
Expand All @@ -77,8 +79,11 @@ var (
}
)

// Order is important here since it will be used to determine install order.
// For example, secrets should be before deployments,
// because it may be an image pull secret used by the deployment
func getBaseK8sKinds() []string {
return []string{K8S_NAMESPACE_TYPE, K8S_CLUSTER_ROLE_TYPE, K8S_CLUSTER_ROLEBINDING_TYPE, K8S_ROLE_TYPE, K8S_ROLEBINDING_TYPE, K8S_SERVICEACCOUNT_TYPE, K8S_CRD_TYPE, K8S_DEPLOYMENT_TYPE}
return []string{K8S_NAMESPACE_TYPE, K8S_CLUSTER_ROLE_TYPE, K8S_CLUSTER_ROLEBINDING_TYPE, K8S_ROLE_TYPE, K8S_ROLEBINDING_TYPE, K8S_SERVICEACCOUNT_TYPE, K8S_SECRET_TYPE, K8S_CRD_TYPE, K8S_DEPLOYMENT_TYPE}
}

func getDangerKinds() []string {
Expand Down Expand Up @@ -163,10 +168,10 @@ func (c KubeClient) Install(tar string, metadata map[string]interface{}, mmsPVCC
if namespace != nodeNamespace && nodeIsNamespaceScope {
return fmt.Errorf("Service failed to start for agreement %v. Could not deploy service into namespace %v because the agent's namespace is namespace scoped, and it restricts all services to the agent namespace %v", agId, namespace, nodeNamespace)
} else if namespace != nodeNamespace {
// create network policies to allow traffic between the node and service
// create network policies to allow traffic between the node and service
ingress := networkingv1.NetworkPolicyIngressRule{From: []networkingv1.NetworkPolicyPeer{networkingv1.NetworkPolicyPeer{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace}}}}}
egress := networkingv1.NetworkPolicyEgressRule{To: []networkingv1.NetworkPolicyPeer{networkingv1.NetworkPolicyPeer{NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace}}}}}
spec := networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{}, Ingress: []networkingv1.NetworkPolicyIngressRule{ingress}, Egress: []networkingv1.NetworkPolicyEgressRule{egress}, PolicyTypes: []networkingv1.PolicyType{"Ingress","Egress"}}
spec := networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{}, Ingress: []networkingv1.NetworkPolicyIngressRule{ingress}, Egress: []networkingv1.NetworkPolicyEgressRule{egress}, PolicyTypes: []networkingv1.PolicyType{"Ingress", "Egress"}}
netPol := networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-networkPolicy", agId), Namespace: nodeNamespace}, Spec: spec}
_, err := c.Client.NetworkingV1().NetworkPolicies(nodeNamespace).Create(context.Background(), &netPol, metav1.CreateOptions{})
if err != nil {
Expand Down Expand Up @@ -233,8 +238,8 @@ func (c KubeClient) Uninstall(tar string, metadata map[string]interface{}, agId
nsObj := corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace"}, ObjectMeta: metav1.ObjectMeta{Name: namespace}}
apiObjMap[K8S_NAMESPACE_TYPE] = []APIObjectInterface{NamespaceCoreV1{NamespaceObject: &nsObj}}
} else if namespace != nodeNamespace {
// delete the network policy that allows traffic between the node and service
err := c.Client.NetworkingV1().NetworkPolicies(nodeNamespace).Delete(context.Background(), fmt.Sprintf("%s-networkPolicy", agId) , metav1.DeleteOptions{})
// delete the network policy that allows traffic between the node and service
err := c.Client.NetworkingV1().NetworkPolicies(nodeNamespace).Delete(context.Background(), fmt.Sprintf("%s-networkPolicy", agId), metav1.DeleteOptions{})
if err != nil {
glog.Errorf(kwlog(fmt.Sprintf("Error deleting network policy: %v", err)))
}
Expand Down

0 comments on commit 6e8c4ae

Please sign in to comment.