From 3aadf293542f8cd0146b535568eee74c75c17b1d Mon Sep 17 00:00:00 2001 From: michaeljguarino Date: Sat, 30 Dec 2023 15:23:13 -0500 Subject: [PATCH] `plural up` command This will use a new bootstrapping method more in line with CD-exclusive usecases. It works basically as follows: * set up a git submodule of our bootstrap repo w/ some base charts and terraform * template some of the variable settings from there into the installation repo * run terraform off those templates This will set up a management cluster and fully install console onto it. We can further extend this to do the full OSS app installation process on top of CD. The main benefits here are separating cluster provisioning concerns out of our purview (users will manually manage their own terraform after running up), and reduce our scope to just maintaining the helm chart updates for the console and our runtime via CD. It also makes reconfiguration more natural since the users can delink the submodule and take full ownership whenever needed. --- cmd/plural/cd.go | 21 ++++++++++++ cmd/plural/crypto.go | 1 + cmd/plural/plural.go | 57 ++++++++++++++++++++++----------- cmd/plural/up.go | 34 ++++++++++++++++++++ pkg/cd/control_plane_install.go | 52 ++++++++++++++++++++++++++++++ pkg/up/context.go | 35 ++++++++++++++++++++ pkg/up/deploy.go | 23 +++++++++++++ pkg/up/generate.go | 35 ++++++++++++++++++++ pkg/up/template.go | 57 +++++++++++++++++++++++++++++++++ pkg/utils/git/repo.go | 5 +++ 10 files changed, 301 insertions(+), 19 deletions(-) create mode 100644 cmd/plural/up.go create mode 100644 pkg/up/context.go create mode 100644 pkg/up/deploy.go create mode 100644 pkg/up/generate.go create mode 100644 pkg/up/template.go diff --git a/cmd/plural/cd.go b/cmd/plural/cd.go index 9de1acbc5..199829508 100644 --- a/cmd/plural/cd.go +++ b/cmd/plural/cd.go @@ -44,6 +44,17 @@ func (p *Plural) cdCommands() []cli.Command { Action: p.handleInstallControlPlane, Usage: "sets up the plural console in an existing k8s cluster", }, + { + Name: "control-plane-values", + Action: p.handlePrintControlPlaneValues, + Usage: "dumps a values file for installing the plural console", + Flags: []cli.Flag{ + cli.StringFlag{Name: "domain", Usage: "The plural domain to use for this console", Required: true}, + cli.StringFlag{Name: "dsn", Usage: "The Postgres DSN to use for database connections", Required: true}, + cli.StringFlag{Name: "name", Usage: "The name given to the cluster", Required: true}, + cli.StringFlag{Name: "file", Usage: "The file to dump values to", Required: true}, + }, + }, { Name: "uninstall", Action: p.handleUninstallOperator, @@ -166,6 +177,16 @@ func (p *Plural) handleInstallControlPlane(_ *cli.Context) error { return nil } +func (p *Plural) handlePrintControlPlaneValues(c *cli.Context) error { + conf := config.Read() + vals, err := cd.ControlPlaneValues(conf, c.String("domain"), c.String("dsn"), c.String("name")) + if err != nil { + return err + } + + return os.WriteFile(c.String("file"), []byte(vals), 0644) +} + func (p *Plural) handleEject(c *cli.Context) (err error) { if !c.Args().Present() { return fmt.Errorf("clusterid cannot be empty") diff --git a/cmd/plural/crypto.go b/cmd/plural/crypto.go index 3193b18c6..feb0046fd 100644 --- a/cmd/plural/crypto.go +++ b/cmd/plural/crypto.go @@ -41,6 +41,7 @@ context.yaml filter=plural-crypt diff=plural-crypt workspace.yaml filter=plural-crypt diff=plural-crypt context.yaml* filter=plural-crypt diff=plural-crypt workspace.yaml* filter=plural-crypt diff=plural-crypt +helm-values/*.yaml filter=plural-crypt diff=plural-crypt .gitattributes !filter !diff ` diff --git a/cmd/plural/plural.go b/cmd/plural/plural.go index d832e1235..33f83662b 100644 --- a/cmd/plural/plural.go +++ b/cmd/plural/plural.go @@ -105,6 +105,44 @@ func (p *Plural) getCommands() []cli.Command { Usage: "Gets cli version info", Action: versionInfo, }, + { + Name: "up", + Usage: "sets up your repository and an initial management cluster", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "endpoint", + Usage: "the endpoint for the plural installation you're working with", + }, + cli.StringFlag{ + Name: "service-account", + Usage: "email for the service account you'd like to use for this workspace", + }, + cli.BoolFlag{ + Name: "ignore-preflights", + Usage: "whether to ignore preflight check failures prior to init", + }, + }, + Action: rooted(latestVersion(owned(p.handleUp))), + }, + { + Name: "init", + Usage: "initializes plural within a git repo", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "endpoint", + Usage: "the endpoint for the plural installation you're working with", + }, + cli.StringFlag{ + Name: "service-account", + Usage: "email for the service account you'd like to use for this workspace", + }, + cli.BoolFlag{ + Name: "ignore-preflights", + Usage: "whether to ignore preflight check failures prior to init", + }, + }, + Action: tracked(latestVersion(p.handleInit), "cli.init"), + }, { Name: "build", Aliases: []string{"bld"}, @@ -272,25 +310,6 @@ func (p *Plural) getCommands() []cli.Command { Usage: "Handles authentication to the plural api", Subcommands: p.authCommands(), }, - { - Name: "init", - Usage: "initializes plural within a git repo", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "endpoint", - Usage: "the endpoint for the plural installation you're working with", - }, - cli.StringFlag{ - Name: "service-account", - Usage: "email for the service account you'd like to use for this workspace", - }, - cli.BoolFlag{ - Name: "ignore-preflights", - Usage: "whether to ignore preflight check failures prior to init", - }, - }, - Action: tracked(latestVersion(p.handleInit), "cli.init"), - }, { Name: "preflights", Usage: "runs provider preflight checks", diff --git a/cmd/plural/up.go b/cmd/plural/up.go new file mode 100644 index 000000000..b0e761b08 --- /dev/null +++ b/cmd/plural/up.go @@ -0,0 +1,34 @@ +package plural + +import ( + "fmt" + + "github.com/urfave/cli" + + "github.com/pluralsh/plural-cli/pkg/up" +) + +const ( + affirmUp = "Are you ready to set up your initial management cluster? You can check the generated terraform/helm to confirm everything looks good first" +) + +func (p *Plural) handleUp(c *cli.Context) error { + if err := p.handleInit(c); err != nil { + return err + } + + ctx, err := up.Build() + if err != nil { + return err + } + + if err := ctx.Generate(); err != nil { + return err + } + + if !affirm(affirmUp, "PLURAL_UP_AFFIRM_DEPLOY") { + return fmt.Errorf("cancelled deploy") + } + + return ctx.Deploy() +} diff --git a/pkg/cd/control_plane_install.go b/pkg/cd/control_plane_install.go index bbe9c7dbf..2fbf09af1 100644 --- a/pkg/cd/control_plane_install.go +++ b/pkg/cd/control_plane_install.go @@ -23,6 +23,58 @@ const ( templateUrl = "https://raw.githubusercontent.com/pluralsh/console/cd-scaffolding/charts/console/values.yaml.liquid" ) +func ControlPlaneValues(conf config.Config, domain, dsn, name string) (string, error) { + consoleDns := fmt.Sprintf("console.%s", domain) + kasDns := fmt.Sprintf("kas.%s", domain) + randoms := map[string]string{} + for _, key := range []string{"jwt", "erlang", "adminPassword", "kasApi", "kasPrivateApi", "kasRedis"} { + rand, err := crypto.RandStr(32) + if err != nil { + return "", err + } + randoms[key] = rand + } + + client := api.FromConfig(&conf) + me, err := client.Me() + if err != nil { + return "", fmt.Errorf("you must run `plural login` before installing") + } + + configuration := map[string]string{ + "consoleDns": consoleDns, + "kasDns": kasDns, + "aesKey": utils.GenAESKey(), + "adminName": me.Email, + "adminEmail": me.Email, + "clusterName": name, + "pluralToken": conf.Token, + "postgresUrl": dsn, + } + for k, v := range randoms { + configuration[k] = v + } + + clientId, clientSecret, err := ensureInstalledAndOidc(client, consoleDns) + if err != nil { + return "", err + } + configuration["pluralClientId"] = clientId + configuration["pluralClientSecret"] = clientSecret + + tpl, err := fetchTemplate() + if err != nil { + return "", err + } + + bindings := map[string]interface{}{ + "configuration": configuration, + } + + res, err := liquidEngine.ParseAndRender(tpl, bindings) + return string(res), err +} + func CreateControlPlane(conf config.Config) (string, error) { client := api.FromConfig(&conf) me, err := client.Me() diff --git a/pkg/up/context.go b/pkg/up/context.go new file mode 100644 index 000000000..484b74cd7 --- /dev/null +++ b/pkg/up/context.go @@ -0,0 +1,35 @@ +package up + +import ( + "path/filepath" + + "github.com/pluralsh/plural-cli/pkg/config" + "github.com/pluralsh/plural-cli/pkg/manifest" + "github.com/pluralsh/plural-cli/pkg/provider" +) + +type Context struct { + Provider provider.Provider + Manifest *manifest.ProjectManifest + Config *config.Config +} + +func Build() (*Context, error) { + projPath, _ := filepath.Abs("workspace.yaml") + project, err := manifest.ReadProject(projPath) + if err != nil { + return nil, err + } + + prov, err := provider.FromManifest(project) + if err != nil { + return nil, err + } + + conf := config.Read() + return &Context{ + Provider: prov, + Config: &conf, + Manifest: project, + }, nil +} diff --git a/pkg/up/deploy.go b/pkg/up/deploy.go new file mode 100644 index 000000000..b70c13f36 --- /dev/null +++ b/pkg/up/deploy.go @@ -0,0 +1,23 @@ +package up + +import ( + "os" + "os/exec" +) + +func (ctx *Context) Deploy() error { + if err := terraform("./cluster", "init", "-upgrade"); err != nil { + return err + } + + return terraform("./cluster", "apply") +} + +func terraform(dir, cmd string, args ...string) error { + cmdArgs := append([]string{cmd}, args...) + osCmd := exec.Command("terraform", cmdArgs...) + osCmd.Dir = dir + osCmd.Stdout = os.Stdout + osCmd.Stderr = os.Stderr + return osCmd.Run() +} diff --git a/pkg/up/generate.go b/pkg/up/generate.go new file mode 100644 index 000000000..89f392014 --- /dev/null +++ b/pkg/up/generate.go @@ -0,0 +1,35 @@ +package up + +import ( + "fmt" + + "github.com/pluralsh/plural-cli/pkg/utils/git" +) + +type templatePair struct { + from string + to string +} + +func (ctx *Context) Generate() error { + if err := git.Submodule("https://github.com/pluralsh/bootstrap.git"); err != nil { + return err + } + + prov := ctx.Provider.Name() + tpls := []templatePair{ + {from: "./bootstrap/charts/runtime/values.yaml.tpl", to: "./helm-values/runtime.yaml"}, + {from: fmt.Sprintf("./bootstrap/templates/providers/bootstrap/%s.tf", prov), to: "clusters/provider.tf"}, + {from: fmt.Sprintf("./bootstrap/templates/setup/providers/%s.tf", prov), to: "clusters/mgmt.tf"}, + {from: "./bootstrap/templates/console.tf", to: "clusters/console.tf"}, + {from: fmt.Sprintf("./bootstrap/templates/providers/apps/%s.tf", prov), to: "apps/provider.tf"}, + } + + for _, tpl := range tpls { + if err := ctx.templateFrom(tpl.from, tpl.to); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/up/template.go b/pkg/up/template.go new file mode 100644 index 000000000..2d41398f8 --- /dev/null +++ b/pkg/up/template.go @@ -0,0 +1,57 @@ +package up + +import ( + "bytes" + "time" + + "text/template" + + "github.com/pluralsh/plural-cli/pkg/api" + "github.com/pluralsh/plural-cli/pkg/utils" + "github.com/pluralsh/polly/retry" +) + +func (ctx *Context) templateFrom(file, to string) error { + buf, err := utils.ReadFile(file) + if err != nil { + return err + } + + res, err := ctx.template(buf) + if err != nil { + return err + } + + return utils.WriteFile(to, []byte(res)) +} + +func (ctx *Context) template(tmplate string) (string, error) { + cluster, provider := ctx.Provider.Cluster(), ctx.Provider.Name() + client := api.NewClient() + retrier := retry.NewConstant(15*time.Millisecond, 3) + eabCredential, err := retry.Retry(retrier, func() (*api.EabCredential, error) { + return client.GetEabCredential(cluster, provider) + }) + if err != nil { + return "", err + } + + values := map[string]interface{}{ + "Cluster": cluster, + "Provider": provider, + "Project": ctx.Provider.Project(), + "Region": ctx.Provider.Region(), + "Context": ctx.Provider.Context(), + "Config": ctx.Config, + "Acme": eabCredential, + } + + tpl, err := template.New("gotpl").Parse(tmplate) + if err != nil { + return "", err + } + + var buf bytes.Buffer + err = tpl.Execute(&buf, values) + return buf.String(), err +} diff --git a/pkg/utils/git/repo.go b/pkg/utils/git/repo.go index 33939748e..dfc11119c 100644 --- a/pkg/utils/git/repo.go +++ b/pkg/utils/git/repo.go @@ -24,6 +24,11 @@ func CurrentBranch() (string, error) { return GitRaw("rev-parse", "--abbrev-ref", "HEAD") } +func Submodule(url string) error { + _, err := GitRaw("submodule", "add", url) + return err +} + func HasUpstreamChanges() (bool, string, error) { headRef, err := GitRaw("rev-parse", "--symbolic-full-name", "HEAD") if err != nil {