Skip to content

Commit

Permalink
Provision a Per-Cluster User
Browse files Browse the repository at this point in the history 
Why you may ask, well, if the password of the main user changes, all
application credentials will be invalidated, and that essentially brings
down the entire region.  So to limit the blash radius we need to
provision a per project user that isn't affected by global nuclear
annihilation.
  • Loading branch information
@spjmurray
spjmurray committed Apr 17, 2024
1 parent 77e8d87 commit aa1f923
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 17 deletions.
26 changes: 26 additions & 0 deletions pkg/providers/openstack/identity.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,22 @@ func (c *IdentityClient) DeleteApplicationCredential(ctx context.Context, userID
return applicationcredentials.Delete(c.client, userID, id).ExtractErr()
}

// CreateUser creates a new user.
func (c *IdentityClient) CreateUser(ctx context.Context, domainID, name, password string) (*users.User, error) {
tracer := otel.GetTracerProvider().Tracer(constants.Application)

_, span := tracer.Start(ctx, "/identity/v3/users", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()

opts := &users.CreateOpts{
Name: name,
DomainID: domainID,
Password: password,
}

return users.Create(c.client, opts).Extract()
}

// GetUser returns user details.
func (c *IdentityClient) GetUser(ctx context.Context, userID string) (*users.User, error) {
tracer := otel.GetTracerProvider().Tracer(constants.Application)
Expand All @@ -320,3 +336,13 @@ func (c *IdentityClient) GetUser(ctx context.Context, userID string) (*users.Use

return users.Get(c.client, userID).Extract()
}

// DeleteUser removes an existing user.
func (c *IdentityClient) DeleteUser(ctx context.Context, userID string) error {
tracer := otel.GetTracerProvider().Tracer(constants.Application)

_, span := tracer.Start(ctx, "/identity/v3/users/"+userID, trace.WithSpanKind(trace.SpanKindClient))
defer span.End()

return users.Delete(c.client, userID).Err
}
83 changes: 66 additions & 17 deletions pkg/providers/openstack/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ import (
"reflect"
"sync"

"github.com/google/uuid"
"github.com/gophercloud/gophercloud/openstack/identity/v3/applicationcredentials"
"github.com/gophercloud/gophercloud/openstack/identity/v3/projects"
"github.com/gophercloud/gophercloud/openstack/identity/v3/roles"
"github.com/gophercloud/gophercloud/openstack/identity/v3/users"
"github.com/gophercloud/utils/openstack/clientconfig"

"github.com/unikorn-cloud/core/pkg/constants"
Expand Down Expand Up @@ -307,6 +309,8 @@ func (p *Provider) Images(ctx context.Context) (providers.ImageList, error) {
const (
// ProjectIDAnnotation records the project ID created for a cluster.
ProjectIDAnnotation = "openstack." + providers.MetdataDomain + "/project-id"
// UserIDAnnotation records the user ID create for a cluster.
UserIDAnnotation = "openstack." + providers.MetdataDomain + "/user-id"

// Projects are randomly named to avoid clashes, so we need to add some tags
// in order to be able to reason about who they really belong to. It is also
Expand All @@ -328,6 +332,20 @@ func projectTags(cluster *unikornv1.KubernetesCluster) []string {
return tags
}

// provisionUser creates a new user in the managed domain with a random password.
// There is a 1:1 mapping of user to project, and the project name is unique in the
// domain, so just reuse this, we can clean them up at the same time.
func (p *Provider) provisionUser(ctx context.Context, identityService *IdentityClient, project *projects.Project) (*users.User, string, error) {
password := uuid.New().String()

user, err := identityService.CreateUser(ctx, p.domainID, project.Name, password)
if err != nil {
return nil, "", err
}

return user, password, nil
}

// provisionProject creates a project per-cluster. Cluster API provider Openstack is
// somewhat broken in that networks can alias and cause all kinds of disasters, so it's
// safest to have one cluster in one project so it has its own namespace.
Expand All @@ -339,14 +357,6 @@ func (p *Provider) provisionProject(ctx context.Context, identityService *Identi
return nil, err
}

if cluster.Annotations == nil {
cluster.Annotations = map[string]string{}
}

// Annotate the cluster with the project ID so we know a) we have created it
// and b) we can find it to make modifications e.g. add tags for garbage collection.
cluster.Annotations[ProjectIDAnnotation] = project.ID

return project, nil
}

Expand Down Expand Up @@ -376,7 +386,7 @@ func (p *Provider) getRequiredRoles() []string {
// provisionProjectRoles creates a binding between our service account and the project
// with the required roles to provision an application credential that will allow cluster
// creation, deletion and life-cycle management.
func (p *Provider) provisionProjectRoles(ctx context.Context, identityService *IdentityClient, project *projects.Project) error {
func (p *Provider) provisionProjectRoles(ctx context.Context, identityService *IdentityClient, userID string, project *projects.Project) error {
allRoles, err := identityService.ListRoles(ctx)
if err != nil {
return err
Expand All @@ -388,17 +398,17 @@ func (p *Provider) provisionProjectRoles(ctx context.Context, identityService *I
return err
}

if err := identityService.CreateRoleAssignment(ctx, p.userID, project.ID, roleID); err != nil {
if err := identityService.CreateRoleAssignment(ctx, userID, project.ID, roleID); err != nil {
return err
}
}

return nil
}

func (p *Provider) provisionApplicationCredential(ctx context.Context, project *projects.Project) (*applicationcredentials.ApplicationCredential, error) {
func (p *Provider) provisionApplicationCredential(ctx context.Context, userID, password string, project *projects.Project) (*applicationcredentials.ApplicationCredential, error) {
// Rescope to the project...
providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, p.userID, p.password, project.ID)
providerClient := NewPasswordProvider(p.region.Spec.Openstack.Endpoint, userID, password, project.ID)

projectScopedIdentity, err := NewIdentityClient(ctx, providerClient)
if err != nil {
Expand Down Expand Up @@ -442,22 +452,38 @@ func (p *Provider) createClientConfig(cluster *unikornv1.KubernetesCluster, appl
}

// ConfigureCluster does any provider specific configuration for a cluster.
//
//nolint:cyclop
func (p *Provider) ConfigureCluster(ctx context.Context, cluster *unikornv1.KubernetesCluster) error {
identityService, err := p.identity(ctx)
if err != nil {
return err
}

// Every cluster has its own project to mitigate "nuances" in CAPO i.e. it's
// totally broken when it comes to network aliasing.
project, err := p.provisionProject(ctx, identityService, cluster)
if err != nil {
return err
}

if err := p.provisionProjectRoles(ctx, identityService, project); err != nil {
// You MUST provision a new user, if we rotate a password, any application credentials
// hanging off it will stop working, i.e. doing that to the unikorn management user
// will be pretty catastrophic for all clusters in the region.
user, password, err := p.provisionUser(ctx, identityService, project)
if err != nil {
return err
}

applicationCredential, err := p.provisionApplicationCredential(ctx, project)
// Give the user only what permissions they need to provision a cluster and
// manage it during its lifetime.
if err := p.provisionProjectRoles(ctx, identityService, user.ID, project); err != nil {
return err
}

// Always use application credentials, they are scoped to a single project and
// cannot be used to break from that jail.
applicationCredential, err := p.provisionApplicationCredential(ctx, user.ID, password, project)
if err != nil {
return err
}
Expand All @@ -471,6 +497,16 @@ func (p *Provider) ConfigureCluster(ctx context.Context, cluster *unikornv1.Kube
return err
}

if cluster.Annotations == nil {
cluster.Annotations = map[string]string{}
}

// Annotate the cluster with the project ID so we know a) we have created it
// and b) we can find it to make modifications e.g. add tags for garbage collection.
// User ID allows us to clean that up too.
cluster.Annotations[ProjectIDAnnotation] = project.ID
cluster.Annotations[UserIDAnnotation] = user.ID

if cluster.Spec.Openstack == nil {
cluster.Spec.Openstack = &unikornv1.KubernetesClusterOpenstackSpec{}
}
Expand All @@ -489,15 +525,28 @@ func (p *Provider) ConfigureCluster(ctx context.Context, cluster *unikornv1.Kube

// DeconfigureCluster does any provider specific cluster cleanup.
func (p *Provider) DeconfigureCluster(ctx context.Context, annotations map[string]string) error {
identityService, err := p.identity(ctx)
if err != nil {
return err
}

userID, ok := annotations[UserIDAnnotation]
if !ok {
return fmt.Errorf("%w: missing user ID annotation", ErrKeyUndefined)
}

projectID, ok := annotations[ProjectIDAnnotation]
if !ok {
return fmt.Errorf("%w: missing project ID annotation", ErrKeyUndefined)
}

identityService, err := p.identity(ctx)
if err != nil {
if err := identityService.DeleteUser(ctx, userID); err != nil {
return err
}

return identityService.DeleteProject(ctx, projectID)
if err := identityService.DeleteProject(ctx, projectID); err != nil {
return err
}

return nil
}

0 comments on commit aa1f923

Please sign in to comment.