From 5f781cd8f474fcadecc284f6b121e26ea65174d8 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Fri, 8 Nov 2024 10:51:22 +0800 Subject: [PATCH] minimize the RBAC permissions for the pull mode cluster Signed-off-by: zhzhuang-zju --- .../deploy/bootstrap-token-configuration.yaml | 260 +---------- pkg/karmadactl/cmdinit/karmada/deploy.go | 4 +- pkg/karmadactl/cmdinit/karmada/rbac.go | 79 +--- pkg/karmadactl/cmdinit/karmada/rbac_test.go | 4 +- pkg/karmadactl/register/register.go | 406 ++++++++++++++++-- pkg/karmadactl/unregister/unregister.go | 51 ++- pkg/karmadactl/unregister/unregister_test.go | 1 + pkg/karmadactl/util/work.go | 54 +++ pkg/util/rbac.go | 46 ++ 9 files changed, 544 insertions(+), 361 deletions(-) create mode 100644 pkg/karmadactl/util/work.go diff --git a/artifacts/deploy/bootstrap-token-configuration.yaml b/artifacts/deploy/bootstrap-token-configuration.yaml index 097718bf6338..2fb306ec740a 100644 --- a/artifacts/deploy/bootstrap-token-configuration.yaml +++ b/artifacts/deploy/bootstrap-token-configuration.yaml @@ -86,269 +86,27 @@ subjects: name: system:nodes --- +# ClusterRole is not used for the connection between the karmada-agent and the control plane, +# but is used by karmadactl register to generate the RBAC resources required by the karmada-agent. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: system:karmada:agent rules: -- apiGroups: - - cluster.karmada.io - resources: - - clusters - verbs: - - create - - get - - list - - watch - - delete -- apiGroups: - - cluster.karmada.io - resources: - - clusters/status - verbs: - - update -- apiGroups: - - work.karmada.io - resources: - - works - verbs: - - create - - get - - list - - watch - - update - - delete -- apiGroups: - - work.karmada.io - resources: - - works/status - verbs: - - patch - - update -- apiGroups: - - config.karmada.io - resources: - - resourceinterpreterwebhookconfigurations - - resourceinterpretercustomizations - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create - - patch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - create - - get -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update + - apiGroups: ['*'] + resources: ['*'] + verbs: ['*'] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: system:karmada:agent + name: system:karmada:agent-rbac-generator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:karmada:agent subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes - -# To ensure the agent has the minimal RBAC permissions, the ideal approach is to -# use different RBAC configurations for different agents of member clusters with pull mode. -# Below is the minimal set of RBAC permissions required for a single pull mode member cluster. -# Here are the definitions of the variables used: -# -# - clustername: the name of the member cluster. -# - cluster_namespace: the namespace where the member cluster secrets are stored, default to karmada-cluster. -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRole -# metadata: -# name: system:karmada:agent -# rules: -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters -# resourceNames: -# - {{clustername}} -# verbs: -# - create -# - get -# - delete -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters -# verbs: -# - list -# - watch -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters/status -# resourceNames: -# - {{clustername}} -# verbs: -# - update -# - apiGroups: -# - config.karmada.io -# resources: -# - resourceinterpreterwebhookconfigurations -# - resourceinterpretercustomizations -# verbs: -# - get -# - list -# - watch -# - apiGroups: -# - "" -# resources: -# - namespaces -# verbs: -# - get -# - apiGroups: -# - coordination.k8s.io -# resources: -# - leases -# verbs: -# - create -# - get -# - update -# - apiGroups: -# - certificates.k8s.io -# resources: -# - certificatesigningrequests -# verbs: -# - create -# - get -# - apiGroups: -# - "" -# resources: -# - events -# verbs: -# - create -# - patch -# - update -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: system:karmada:agent -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: system:karmada:agent -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: Role -# metadata: -# name: system:karmada:agent-secret -# namespace: "{{cluster_namespace}}" -# rules: -# - apiGroups: -# - "" -# resources: -# - secrets -# resourceNames: -# - {{clustername}}-impersonator -# - {{clustername}} -# verbs: -# - get -# - create -# - patch -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: RoleBinding -# metadata: -# name: system:karmada:agent-secret -# namespace: "{{cluster_namespace}}" -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: Role -# name: system:karmada:agent-secret -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: Role -# metadata: -# name: system:karmada:agent-work -# namespace: "karmada-es-{{clustername}}" -# rules: -# - apiGroups: -# - work.karmada.io -# resources: -# - works -# verbs: -# - create -# - get -# - list -# - watch -# - update -# - delete -# - apiGroups: -# - work.karmada.io -# resources: -# - works/status -# verbs: -# - patch -# - update -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: RoleBinding -# metadata: -# name: system:karmada:agent-work -# namespace: "karmada-es-{{clustername}}" -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: Role -# name: system:karmada:agent-work -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:node:agent-rbac-generator diff --git a/pkg/karmadactl/cmdinit/karmada/deploy.go b/pkg/karmadactl/cmdinit/karmada/deploy.go index 7af7791d3601..899adfead9cf 100644 --- a/pkg/karmadactl/cmdinit/karmada/deploy.go +++ b/pkg/karmadactl/cmdinit/karmada/deploy.go @@ -185,8 +185,8 @@ func createExtraResources(clientSet *kubernetes.Clientset, dir string) error { return fmt.Errorf("error creating clusterinfo RBAC rules: %v", err) } - // grant limited access permission to 'karmada-agent' - if err := grantAccessPermissionToAgent(clientSet); err != nil { + // grant access permission to 'karmada-agent-rbac-generator' + if err := grantAccessPermissionToAgentRBACGenerator(clientSet); err != nil { return err } diff --git a/pkg/karmadactl/cmdinit/karmada/rbac.go b/pkg/karmadactl/cmdinit/karmada/rbac.go index 7ee4281fee83..7a1e4622e037 100644 --- a/pkg/karmadactl/cmdinit/karmada/rbac.go +++ b/pkg/karmadactl/cmdinit/karmada/rbac.go @@ -26,10 +26,11 @@ import ( ) const ( - karmadaViewClusterRole = "karmada-view" - karmadaEditClusterRole = "karmada-edit" - karmadaAgentAccessClusterRole = "system:karmada:agent" - karmadaAgentGroup = "system:nodes" + karmadaViewClusterRole = "karmada-view" + karmadaEditClusterRole = "karmada-edit" + karmadaAgentRBACGeneratorClusterRole = "system:karmada:agent" + karmadaAgentRBACGeneratorClusterRoleBinding = "system:karmada:agent-rbac-generator" + agentRBACGenerator = "system:node:agent-rbac-generator" ) // grantProxyPermissionToAdmin grants the proxy permission to "system:admin" @@ -62,63 +63,13 @@ func grantProxyPermissionToAdmin(clientSet kubernetes.Interface) error { return nil } -// grantAccessPermissionToAgent grants the limited access permission to 'karmada-agent' -func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error { - clusterRole := utils.ClusterRoleFromRules(karmadaAgentAccessClusterRole, []rbacv1.PolicyRule{ +// grantAccessPermissionToAgentRBACGenerator grants the access permission to 'karmada-agent-rbac-generator' +func grantAccessPermissionToAgentRBACGenerator(clientSet kubernetes.Interface) error { + clusterRole := utils.ClusterRoleFromRules(karmadaAgentRBACGeneratorClusterRole, []rbacv1.PolicyRule{ { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"cluster.karmada.io"}, - Resources: []string{"clusters"}, - Verbs: []string{"create", "get", "list", "watch", "patch", "update", "delete"}, - }, - { - APIGroups: []string{"cluster.karmada.io"}, - Resources: []string{"clusters/status"}, - Verbs: []string{"patch", "update"}, - }, - { - APIGroups: []string{"work.karmada.io"}, - Resources: []string{"works"}, - Verbs: []string{"create", "get", "list", "watch", "update", "delete"}, - }, - { - APIGroups: []string{"work.karmada.io"}, - Resources: []string{"works/status"}, - Verbs: []string{"patch", "update"}, - }, - { - APIGroups: []string{"config.karmada.io"}, - Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch", "create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch", "create", "patch"}, - }, - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"create", "delete", "get", "patch", "update"}, - }, - { - APIGroups: []string{"certificates.k8s.io"}, - Resources: []string{"certificatesigningrequests"}, - Verbs: []string{"create", "get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch", "update"}, + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, }, }, nil, nil) err := cmdutil.CreateOrUpdateClusterRole(clientSet, clusterRole) @@ -126,14 +77,14 @@ func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error { return err } - clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentAccessClusterRole, karmadaAgentAccessClusterRole, + clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentRBACGeneratorClusterRoleBinding, karmadaAgentRBACGeneratorClusterRole, []rbacv1.Subject{ { - Kind: rbacv1.GroupKind, - Name: karmadaAgentGroup, + Kind: rbacv1.UserKind, + Name: agentRBACGenerator, }}, nil) - klog.V(1).Info("Grant the limited access permission to 'karmada-agent'") + klog.V(1).Info("Grant the access permission to 'karmada-agent-rbac-generator'") err = cmdutil.CreateOrUpdateClusterRoleBinding(clientSet, clusterRoleBinding) if err != nil { return err diff --git a/pkg/karmadactl/cmdinit/karmada/rbac_test.go b/pkg/karmadactl/cmdinit/karmada/rbac_test.go index 501adae9325d..7767cb6aa384 100644 --- a/pkg/karmadactl/cmdinit/karmada/rbac_test.go +++ b/pkg/karmadactl/cmdinit/karmada/rbac_test.go @@ -31,8 +31,8 @@ func Test_grantProxyPermissionToAdmin(t *testing.T) { func Test_grantAccessPermissionToAgent(t *testing.T) { client := fake.NewSimpleClientset() - if err := grantAccessPermissionToAgent(client); err != nil { - t.Errorf("grantAccessPermissionToAgent() expected no error, but got err: %v", err) + if err := grantAccessPermissionToAgentRBACGenerator(client); err != nil { + t.Errorf("grantAccessPermissionToAgentRBACGenerator() expected no error, but got err: %v", err) } } diff --git a/pkg/karmadactl/register/register.go b/pkg/karmadactl/register/register.go index c07db7cc78f7..701a490871f5 100644 --- a/pkg/karmadactl/register/register.go +++ b/pkg/karmadactl/register/register.go @@ -51,7 +51,6 @@ import ( "github.com/karmada-io/karmada/pkg/apis/cluster/validation" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" - addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" @@ -85,6 +84,8 @@ const ( ClusterPermissionPrefix = "system:node:" // ClusterPermissionGroups defines the organization of karmada agent certificate ClusterPermissionGroups = "system:nodes" + // AgentRBACGenerator defines the common name of karmada agent rbac generator certificate + AgentRBACGenerator = "system:node:agent-rbac-generator" // KarmadaAgentBootstrapKubeConfigFileName defines the file name for the kubeconfig that the karmada-agent will use to do // the TLS bootstrap to get itself an unique credential KarmadaAgentBootstrapKubeConfigFileName = "bootstrap-karmada-agent.conf" @@ -260,6 +261,8 @@ type CommandRegisterOption struct { memberClusterEndpoint string memberClusterClient *kubeclient.Clientset + + rbacResources *RBACResources } // Complete ensures that options are valid and marshals them if necessary. @@ -288,6 +291,8 @@ func (o *CommandRegisterOption) Complete(args []string) error { o.ClusterName = config.Contexts[config.CurrentContext].Cluster } + o.rbacResources = GenerateRBACResources(o.ClusterName, o.ClusterNamespace) + o.memberClusterEndpoint = restConfig.Host o.memberClusterClient, err = apiclient.NewClientSet(restConfig) @@ -320,6 +325,8 @@ func (o *CommandRegisterOption) Validate() error { } // Run is the implementation of the 'register' command. +// +//nolint:gocyclo func (o *CommandRegisterOption) Run(parentCommand string) error { klog.V(1).Infof("Registering cluster. cluster name: %s", o.ClusterName) klog.V(1).Infof("Registering cluster. cluster namespace: %s", o.ClusterNamespace) @@ -358,15 +365,25 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { return err } - // construct the final kubeconfig file used by karmada agent to connect to karmada apiserver - fmt.Println("[karmada-agent-start] Waiting to construct karmada-agent kubeconfig") - karmadaAgentCfg, err := o.constructKarmadaAgentConfig(bootstrapClient, karmadaClusterInfo) + csrName := "agent-rbac-generator-" + o.ClusterName + k8srand.String(5) + rbacCfg, err := o.constructAgentRBACGeneratorConfig(bootstrapClient, karmadaClusterInfo, csrName) + if err != nil { + return err + } + + RBACClient, err := ToClientSet(rbacCfg) if err != nil { return err } + defer func() { + err = RBACClient.CertificatesV1().CertificateSigningRequests().Delete(context.Background(), csrName, metav1.DeleteOptions{}) + if err != nil { + klog.Warningf("Failed to delete CertificateSigningRequests %s: %v", csrName, err) + } + }() fmt.Println("[karmada-agent-start] Waiting to check cluster exists") - karmadaClient, err := ToKarmadaClient(karmadaAgentCfg) + karmadaClient, err := ToKarmadaClient(rbacCfg) if err != nil { return err } @@ -374,7 +391,30 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { if err != nil { return err } else if exist { - return fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName) + err = fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName) + return err + } + + fmt.Println("[karmada-agent-start] Assign the necessary RBAC permissions to the agent") + defer func() { + if err != nil { + fmt.Println("karmadactl register failed and started deleting the created resources") + err = o.rbacResources.Delete(RBACClient) + if err != nil { + klog.Warningf("Failed to delete rbac resources: %v", err) + } + } + }() + err = o.CreateAgentRBACResourcesInControlPlane(RBACClient) + if err != nil { + return err + } + + // construct the final kubeconfig file used by karmada agent to connect to karmada apiserver + fmt.Println("[karmada-agent-start] Waiting to construct karmada-agent kubeconfig") + karmadaAgentCfg, err := o.constructKarmadaAgentConfig(bootstrapClient, karmadaClusterInfo) + if err != nil { + return err } // It's necessary to set the label of namespace to make sure that the namespace is created by Karmada. @@ -382,24 +422,24 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue, } // ensure namespace where the karmada-agent resources be deployed exists in the member cluster - if _, err := karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil { + if _, err = karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil { return err } // create the necessary secret and RBAC in the member cluster fmt.Println("[karmada-agent-start] Waiting the necessary secret and RBAC") - if err := o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil { + if err = o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil { return err } // create karmada-agent Deployment in the member cluster fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment") KarmadaAgentDeployment := o.makeKarmadaAgentDeployment() - if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { + if _, err = o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { return err } - if err := addonutils.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { + if err = cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { return err } @@ -505,11 +545,311 @@ func (o *CommandRegisterOption) discoveryBootstrapConfigAndClusterInfo(bootstrap return bootstrapClient, clusterinfo, nil } -// constructKarmadaAgentConfig construct the final kubeconfig used by karmada-agent -func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) { +// CreateAgentRBACResourcesInControlPlane create rbac resources for karmada-agent in control plane. +func (o *CommandRegisterOption) CreateAgentRBACResourcesInControlPlane(client kubeclient.Interface) error { + for i := range o.rbacResources.ClusterRoles { + _, err := karmadautil.CreateClusterRole(client, o.rbacResources.ClusterRoles[i]) + if err != nil { + return err + } + } + for i := range o.rbacResources.ClusterRoleBindings { + _, err := karmadautil.CreateClusterRoleBinding(client, o.rbacResources.ClusterRoleBindings[i]) + if err != nil { + return err + } + } + + for i := range o.rbacResources.Roles { + roleNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: o.rbacResources.Roles[i].GetNamespace(), + Labels: map[string]string{ + karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue, + }, + }, + } + _, err := karmadautil.CreateNamespace(client, roleNamespace) + if err != nil { + return err + } + _, err = karmadautil.CreateRole(client, o.rbacResources.Roles[i]) + if err != nil { + return err + } + } + + for i := range o.rbacResources.RoleBindings { + _, err := karmadautil.CreateRoleBinding(client, o.rbacResources.RoleBindings[i]) + if err != nil { + return err + } + } + + return nil +} + +// RBACResources defines the list of rbac resources. +type RBACResources struct { + ClusterRoles []*rbacv1.ClusterRole + ClusterRoleBindings []*rbacv1.ClusterRoleBinding + Roles []*rbacv1.Role + RoleBindings []*rbacv1.RoleBinding +} + +// GenerateRBACResources generates rbac resources. +func GenerateRBACResources(clusterName, clusterNamespace string) *RBACResources { + return &RBACResources{ + ClusterRoles: []*rbacv1.ClusterRole{GenerateClusterRole(clusterName)}, + ClusterRoleBindings: []*rbacv1.ClusterRoleBinding{GenerateClusterRoleBinding(clusterName)}, + Roles: []*rbacv1.Role{GenerateSecretAccessRole(clusterName, clusterNamespace), GenerateWorkAccessRole(clusterName)}, + RoleBindings: []*rbacv1.RoleBinding{GenerateSecretAccessRoleBinding(clusterName, clusterNamespace), GenerateWorkAccessRoleBinding(clusterName)}, + } +} + +// List return the list of rbac resources. +func (r *RBACResources) List() []Obj { + var obj []Obj + for i := range r.ClusterRoles { + obj = append(obj, Obj{Kind: "ClusterRole", Name: r.ClusterRoles[i].GetName()}) + } + for i := range r.ClusterRoleBindings { + obj = append(obj, Obj{Kind: "ClusterRoleBinding", Name: r.ClusterRoleBindings[i].GetName()}) + } + for i := range r.Roles { + obj = append(obj, Obj{Kind: "Role", Name: r.Roles[i].GetName(), Namespace: r.Roles[i].GetNamespace()}) + } + for i := range r.RoleBindings { + obj = append(obj, Obj{Kind: "RoleBinding", Name: r.RoleBindings[i].GetName(), Namespace: r.RoleBindings[i].GetNamespace()}) + } + return obj +} + +// ToString returns a list of RBAC resources in string format. +func (r *RBACResources) ToString() string { + var resources []string + for i := range r.List() { + resources = append(resources, r.List()[i].ToString()) + } + return strings.Join(resources, "\n") +} + +// Delete deletes RBAC resources. +func (r *RBACResources) Delete(client kubeclient.Interface) error { + var err error + for _, resource := range r.List() { + switch resource.Kind { + case "ClusterRole": + err = karmadautil.DeleteClusterRole(client, resource.Name) + case "ClusterRoleBinding": + err = karmadautil.DeleteClusterRoleBinding(client, resource.Name) + case "Role": + err = karmadautil.DeleteRole(client, resource.Namespace, resource.Name) + case "RoleBinding": + err = karmadautil.DeleteRoleBinding(client, resource.Namespace, resource.Name) + } + if err != nil { + return err + } + } + return nil +} + +// Obj defines the struct which contains the information of kind, name and namespace. +type Obj struct{ Kind, Name, Namespace string } + +// ToString returns a string that concatenates kind, name, and namespace using "/". +func (o *Obj) ToString() string { + if o.Namespace == "" { + return fmt.Sprintf("%s/%s", o.Kind, o.Name) + } + return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name) +} + +// GenerateClusterRole generates the clusterRole that karmada-agent needed. +func GenerateClusterRole(clusterName string) *rbacv1.ClusterRole { + clusterRoleName := fmt.Sprintf("system:karmada:%s:agent", clusterName) + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters"}, + ResourceNames: []string{clusterName}, + Verbs: []string{"get", "delete"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters"}, + Verbs: []string{"create", "list", "watch"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters/status"}, + ResourceNames: []string{clusterName}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"config.karmada.io"}, + Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "create", "update"}, + }, + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests"}, + Verbs: []string{"get", "create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"patch", "create", "update"}, + }, + }, + } +} + +// GenerateClusterRoleBinding generates the clusterRoleBinding that karmada-agent needed. +func GenerateClusterRoleBinding(clusterName string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("system:karmada:%s:agent", clusterName)}, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: fmt.Sprintf("system:karmada:%s:agent", clusterName), + }, + } +} + +// GenerateSecretAccessRole generates the secret-related Role that karmada-agent needed. +func GenerateSecretAccessRole(clusterName, clusterNamespace string) *rbacv1.Role { + secretAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-secret", clusterName) + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretAccessRoleName, + Namespace: clusterNamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "patch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + ResourceNames: []string{clusterName, clusterName + "-impersonator"}, + }, + { + Verbs: []string{"create"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } +} + +// GenerateSecretAccessRoleBinding generates the secret-related RoleBinding that karmada-agent needed. +func GenerateSecretAccessRoleBinding(clusterName, clusterNamespace string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName), + Namespace: clusterNamespace, + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName), + }, + } +} + +// GenerateWorkAccessRole generates the work-related Role that karmada-agent needed. +func GenerateWorkAccessRole(clusterName string) *rbacv1.Role { + workAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-work", clusterName) + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: workAccessRoleName, + Namespace: "karmada-es-" + clusterName, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "list", "watch", "update", "delete"}, + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works"}, + }, + { + Verbs: []string{"patch", "update"}, + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works/status"}, + }, + }, + } +} + +// GenerateWorkAccessRoleBinding generates the work-related RoleBinding that karmada-agent needed. +func GenerateWorkAccessRoleBinding(clusterName string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName), + Namespace: "karmada-es-" + clusterName, + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName), + }, + } +} + +func (o *CommandRegisterOption) constructKubeConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName, commonName string, organization []string) (*clientcmdapi.Config, error) { var cert []byte - pk, csr, err := generateKeyAndCSR(o.ClusterName) + pk, csr, err := generateKeyAndCSR(commonName, organization) if err != nil { return nil, err } @@ -519,8 +859,6 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub return nil, err } - csrName := o.ClusterName + "-" + k8srand.String(5) - certificateSigningRequest := &certificatesv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: csrName, @@ -547,35 +885,44 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub if err != nil { return nil, err } - - klog.V(1).Infof("Waiting for the client certificate to be issued") + klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate %s to be issued", csrName)) err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, o.Timeout, false, func(context.Context) (done bool, err error) { csrOK, err := bootstrapClient.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), csrName, metav1.GetOptions{}) if err != nil { - return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", o.ClusterName, err) + return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", csrName, err) } if csrOK.Status.Certificate != nil { - klog.V(1).Infof("Signing certificate successfully") + klog.V(1).Infof(fmt.Sprintf("Signing certificate of csr %s successfully", csrName)) cert = csrOK.Status.Certificate return true, nil } - klog.V(1).Infof("Waiting for the client certificate to be issued") + klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate of csr %s to be issued", csrName)) return false, nil }) if err != nil { return nil, err } - karmadaAgentCfg := CreateWithCert( + return CreateWithCert( karmadaClusterInfo.Server, DefaultClusterName, o.ClusterName, karmadaClusterInfo.CertificateAuthorityData, cert, pkData, - ) + ), nil +} + +// constructKarmadaAgentConfig constructs the final kubeconfig used by karmada-agent +func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) { + csrName := o.ClusterName + "-" + k8srand.String(5) + + karmadaAgentCfg, err := o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, generateAgentUserName(o.ClusterName), []string{ClusterPermissionGroups}) + if err != nil { + return nil, err + } kubeConfigFile := filepath.Join(KarmadaDir, KarmadaAgentKubeConfigFileName) @@ -588,6 +935,11 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub return karmadaAgentCfg, nil } +// constructKarmadaAgentConfig constructs the kubeconfig to generate rbac config for karmada-agent. +func (o *CommandRegisterOption) constructAgentRBACGeneratorConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName string) (*clientcmdapi.Config, error) { + return o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, AgentRBACGenerator, []string{ClusterPermissionGroups}) +} + // createSecretAndRBACInMemberCluster create required secrets and rbac in member cluster func (o *CommandRegisterOption) createSecretAndRBACInMemberCluster(karmadaAgentCfg *clientcmdapi.Config) error { configBytes, err := clientcmd.Write(*karmadaAgentCfg) @@ -781,7 +1133,7 @@ func (o *CommandRegisterOption) makeKarmadaAgentDeployment() *appsv1.Deployment } // generateKeyAndCSR generate private key and csr -func generateKeyAndCSR(clusterName string) (*rsa.PrivateKey, []byte, error) { +func generateKeyAndCSR(commonName string, organization []string) (*rsa.PrivateKey, []byte, error) { pk, err := rsa.GenerateKey(rand.Reader, 3072) if err != nil { return nil, nil, err @@ -789,8 +1141,8 @@ func generateKeyAndCSR(clusterName string) (*rsa.PrivateKey, []byte, error) { csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ Subject: pkix.Name{ - CommonName: ClusterPermissionPrefix + clusterName, - Organization: []string{ClusterPermissionGroups}, + CommonName: commonName, + Organization: organization, }, }, pk) if err != nil { @@ -1070,3 +1422,7 @@ func ToKarmadaClient(config *clientcmdapi.Config) (*karmadaclientset.Clientset, return karmadaClient, nil } + +func generateAgentUserName(clusterName string) string { + return ClusterPermissionPrefix + clusterName + ":agent" +} diff --git a/pkg/karmadactl/unregister/unregister.go b/pkg/karmadactl/unregister/unregister.go index 90b7ccfaa98d..15f70f3c42a2 100644 --- a/pkg/karmadactl/unregister/unregister.go +++ b/pkg/karmadactl/unregister/unregister.go @@ -135,6 +135,8 @@ type CommandUnregisterOption struct { // MemberClusterClient member cluster client set MemberClusterClient kubeclient.Interface + + rbacResources *register.RBACResources } // AddFlags adds flags to the specified FlagSet. @@ -157,6 +159,8 @@ func (j *CommandUnregisterOption) Complete(args []string) error { if len(args) > 0 { j.ClusterName = args[0] } + + j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace) return nil } @@ -303,31 +307,44 @@ func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment return clientcmd.Load(agentConfigSecret.Data[fileName]) } -type obj struct{ Kind, Name, Namespace string } - -func (o *obj) ToString() string { - if o.Namespace == "" { - return fmt.Sprintf("%s/%s", o.Kind, o.Name) - } - return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name) -} - // RunUnregisterCluster unregister the pull mode cluster from karmada. +// +//nolint:gocyclo func (j *CommandUnregisterOption) RunUnregisterCluster() error { if j.DryRun { return nil } - // 1. delete the cluster object from the Karmada control plane + start := time.Now() + // 1. delete the work object from the Karmada control plane + // When deleting a cluster, the deletion triggers the removal of executionSpace, which can lead to the deletion of RBAC roles related to work. + // Therefore, the deletion of work should be performed before deleting the cluster. + err := cmdutil.DeleteWorksObject(j.ControlPlaneClient, names.GenerateExecutionSpaceName(j.ClusterName), j.Wait) + if err != nil { + klog.Errorf("Failed to delete works object. cluster name: %s, error: %v", j.ClusterName, err) + return err + } + j.Wait = j.Wait - time.Since(start) + + // 2. delete the cluster object from the Karmada control plane //TODO: add flag --force to implement force deletion. - if err := cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { + if err = cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err } klog.Infof("Successfully delete cluster object (%s) from control plane.", j.ClusterName) - // 2. delete resource created by karmada in member cluster - var err error + if j.KarmadaConfig != "" { + if err = j.rbacResources.Delete(j.ControlPlaneKubeClient); err != nil { + klog.Errorf("Failed to delete karmada-agent rbac resources in control plane. cluster name: %s, error: %v", j.ClusterName, err) + return err + } + klog.Infof("Successfully delete karmada-agent rbac resources in control plane. cluster name: %s", j.ClusterName) + } else { + klog.Warningf("The RBAC resources on the control plane need to be manually cleaned up, including the following resources:\n%s", j.rbacResources.ToString()) + } + + // 3. delete resource created by karmada in member cluster for _, resource := range j.listMemberClusterResources() { switch resource.Kind { case "ClusterRole": @@ -351,8 +368,8 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { klog.Infof("Successfully delete resource (%v) from member cluster (%s).", resource, j.ClusterName) } - // 3. delete local obsolete files generated by karmadactl - localObsoleteFiles := []obj{ + // 4. delete local obsolete files generated by karmadactl + localObsoleteFiles := []register.Obj{ {Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)}, {Kind: "File", Name: register.CACertPath}, } @@ -371,8 +388,8 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { } // listMemberClusterResources lists resources to be deleted which created by karmada in member cluster -func (j *CommandUnregisterOption) listMemberClusterResources() []obj { - return []obj{ +func (j *CommandUnregisterOption) listMemberClusterResources() []register.Obj { + return []register.Obj{ // the rbac resource prepared for karmada-controller-manager to access member cluster's kube-apiserver {Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)}, {Kind: "ClusterRole", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))}, diff --git a/pkg/karmadactl/unregister/unregister_test.go b/pkg/karmadactl/unregister/unregister_test.go index ab3c97e9df63..875ddc83ba88 100644 --- a/pkg/karmadactl/unregister/unregister_test.go +++ b/pkg/karmadactl/unregister/unregister_test.go @@ -216,6 +216,7 @@ func TestCommandUnregisterOption_RunUnregisterCluster(t *testing.T) { } j.ControlPlaneClient = fakekarmadaclient.NewSimpleClientset(tt.clusterObject...) j.MemberClusterClient = fake.NewSimpleClientset(tt.clusterResources...) + j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace) err := j.RunUnregisterCluster() if (err == nil && tt.wantErr) || (err != nil && !tt.wantErr) { t.Errorf("RunUnregisterCluster() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/karmadactl/util/work.go b/pkg/karmadactl/util/work.go new file mode 100644 index 000000000000..921518d3efaf --- /dev/null +++ b/pkg/karmadactl/util/work.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// DeleteWorksObject deletes the works object from the Karmada control plane. +func DeleteWorksObject(controlPlaneKarmadaClient karmadaclientset.Interface, namespace string, + timeout time.Duration) error { + // make sure the works object under the given namespace has been deleted. + err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) { + list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("failed to list work in namespace %s", namespace) + } + + if len(list.Items) == 0 { + return true, nil + } + for i := range list.Items { + work := &list.Items[i] + err = controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).Delete(context.TODO(), work.GetName(), metav1.DeleteOptions{}) + if err != nil { + return false, fmt.Errorf("failed to delete the work(%s/%s)", namespace, work.GetName()) + } + } + return false, nil + }) + + return err +} diff --git a/pkg/util/rbac.go b/pkg/util/rbac.go index 81496680ed49..93bf5379b7ea 100644 --- a/pkg/util/rbac.go +++ b/pkg/util/rbac.go @@ -191,3 +191,49 @@ func GenerateImpersonationRules(subjects []rbacv1.Subject) []rbacv1.PolicyRule { return rules } + +// CreateRole just try to create the Role. +func CreateRole(client kubeclient.Interface, roleObj *rbacv1.Role) (*rbacv1.Role, error) { + createdObj, err := client.RbacV1().Roles(roleObj.GetNamespace()).Create(context.TODO(), roleObj, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return roleObj, nil + } + + return nil, err + } + + return createdObj, nil +} + +// CreateRoleBinding just try to create the RoleBinding. +func CreateRoleBinding(client kubeclient.Interface, roleBindingObj *rbacv1.RoleBinding) (*rbacv1.RoleBinding, error) { + createdObj, err := client.RbacV1().RoleBindings(roleBindingObj.GetNamespace()).Create(context.TODO(), roleBindingObj, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return roleBindingObj, nil + } + + return nil, err + } + + return createdObj, nil +} + +// DeleteRole just try to delete the Role. +func DeleteRole(client kubeclient.Interface, namespace, name string) error { + err := client.RbacV1().Roles(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +// DeleteRoleBinding just try to delete the RoleBinding. +func DeleteRoleBinding(client kubeclient.Interface, namespace, name string) error { + err := client.RbacV1().RoleBindings(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +}