plural up command

This will use a new bootstrapping method more in line with CD-exclusive usecases.  It works basically as follows:

* set up a git submodule of our bootstrap repo w/ some base charts and terraform
* template some of the variable settings from there into the installation repo
* run terraform off those templates

This will set up a management cluster and fully install console onto it.  We can further extend this to do the full OSS app installation process on top of CD.

The main benefits here are separating cluster provisioning concerns out of our purview (users will manually manage their own terraform after running up), and reduce our scope to just maintaining the helm chart updates for the console and our runtime via CD.  It also makes reconfiguration more natural since the users can delink the submodule and take full ownership whenever needed.

cmd/plural/cd.go

@@ -44,6 +44,17 @@ func (p *Plural) cdCommands() []cli.Command {
 			Action: p.handleInstallControlPlane,
 			Usage:  "sets up the plural console in an existing k8s cluster",
 		},
+		{
+			Name:   "control-plane-values",
+			Action: p.handlePrintControlPlaneValues,
+			Usage:  "dumps a values file for installing the plural console",
+			Flags: []cli.Flag{
+				cli.StringFlag{Name: "domain", Usage: "The plural domain to use for this console", Required: true},
+				cli.StringFlag{Name: "dsn", Usage: "The Postgres DSN to use for database connections", Required: true},
+				cli.StringFlag{Name: "name", Usage: "The name given to the cluster", Required: true},
+				cli.StringFlag{Name: "file", Usage: "The file to dump values to", Required: true},
+			},
+		},
 		{
 			Name:   "uninstall",
 			Action: p.handleUninstallOperator,
@@ -166,6 +177,16 @@ func (p *Plural) handleInstallControlPlane(_ *cli.Context) error {
 	return nil
 }
 
+func (p *Plural) handlePrintControlPlaneValues(c *cli.Context) error {
+	conf := config.Read()
+	vals, err := cd.ControlPlaneValues(conf, c.String("domain"), c.String("dsn"), c.String("name"))
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(c.String("file"), []byte(vals), 0644)
+}
+
 func (p *Plural) handleEject(c *cli.Context) (err error) {
 	if !c.Args().Present() {
 		return fmt.Errorf("clusterid cannot be empty")

cmd/plural/crypto.go

@@ -41,6 +41,7 @@ context.yaml filter=plural-crypt diff=plural-crypt
 workspace.yaml filter=plural-crypt diff=plural-crypt
 context.yaml* filter=plural-crypt diff=plural-crypt
 workspace.yaml* filter=plural-crypt diff=plural-crypt
+helm-values/*.yaml filter=plural-crypt diff=plural-crypt
 .gitattributes !filter !diff
 `
 

cmd/plural/init.go

@@ -29,6 +29,11 @@ func (p *Plural) handleInit(c *cli.Context) error {
 	gitCreated := false
 	repo := ""
 
+	if utils.Exists("./workspace.yaml") {
+		utils.Highlight("Found workspace.yaml, skipping init as this repo has already been initialized...\n")
+		return nil
+	}
+
 	git, err := wkspace.Preflight()
 	if err != nil && git {
 		return err

cmd/plural/plural.go

@@ -105,6 +105,44 @@ func (p *Plural) getCommands() []cli.Command {
 			Usage:   "Gets cli version info",
 			Action:  versionInfo,
 		},
+		{
+			Name:  "up",
+			Usage: "sets up your repository and an initial management cluster",
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "endpoint",
+					Usage: "the endpoint for the plural installation you're working with",
+				},
+				cli.StringFlag{
+					Name:  "service-account",
+					Usage: "email for the service account you'd like to use for this workspace",
+				},
+				cli.BoolFlag{
+					Name:  "ignore-preflights",
+					Usage: "whether to ignore preflight check failures prior to init",
+				},
+			},
+			Action: latestVersion(p.handleUp),
+		},
+		{
+			Name:  "init",
+			Usage: "initializes plural within a git repo",
+			Flags: []cli.Flag{
+				cli.StringFlag{
+					Name:  "endpoint",
+					Usage: "the endpoint for the plural installation you're working with",
+				},
+				cli.StringFlag{
+					Name:  "service-account",
+					Usage: "email for the service account you'd like to use for this workspace",
+				},
+				cli.BoolFlag{
+					Name:  "ignore-preflights",
+					Usage: "whether to ignore preflight check failures prior to init",
+				},
+			},
+			Action: tracked(latestVersion(p.handleInit), "cli.init"),
+		},
 		{
 			Name:    "build",
 			Aliases: []string{"bld"},
@@ -272,25 +310,6 @@ func (p *Plural) getCommands() []cli.Command {
 			Usage:       "Handles authentication to the plural api",
 			Subcommands: p.authCommands(),
 		},
-		{
-			Name:  "init",
-			Usage: "initializes plural within a git repo",
-			Flags: []cli.Flag{
-				cli.StringFlag{
-					Name:  "endpoint",
-					Usage: "the endpoint for the plural installation you're working with",
-				},
-				cli.StringFlag{
-					Name:  "service-account",
-					Usage: "email for the service account you'd like to use for this workspace",
-				},
-				cli.BoolFlag{
-					Name:  "ignore-preflights",
-					Usage: "whether to ignore preflight check failures prior to init",
-				},
-			},
-			Action: tracked(latestVersion(p.handleInit), "cli.init"),
-		},
 		{
 			Name:     "preflights",
 			Usage:    "runs provider preflight checks",

cmd/plural/up.go

@@ -0,0 +1,36 @@
+package plural
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+
+	"github.com/pluralsh/plural-cli/pkg/up"
+)
+
+const (
+	affirmUp = "Are you ready to set up your initial management cluster?  You can check the generated terraform/helm to confirm everything looks good first"
+)
+
+func (p *Plural) handleUp(c *cli.Context) error {
+	if err := p.handleInit(c); err != nil {
+		return err
+	}
+
+	p.InitPluralClient()
+
+	ctx, err := up.Build()
+	if err != nil {
+		return err
+	}
+
+	if err := ctx.Generate(); err != nil {
+		return err
+	}
+
+	if !affirm(affirmUp, "PLURAL_UP_AFFIRM_DEPLOY") {
+		return fmt.Errorf("cancelled deploy")
+	}
+
+	return ctx.Deploy()
+}

pkg/cd/control_plane_install.go

@@ -20,9 +20,61 @@ var (
 )
 
 const (
-	templateUrl = "https://raw.githubusercontent.com/pluralsh/console/cd-scaffolding/charts/console/values.yaml.liquid"
+	templateUrl = "https://raw.githubusercontent.com/pluralsh/console/master/charts/console/values.yaml.liquid"
 )
 
+func ControlPlaneValues(conf config.Config, domain, dsn, name string) (string, error) {
+	consoleDns := fmt.Sprintf("console.%s", domain)
+	kasDns := fmt.Sprintf("kas.%s", domain)
+	randoms := map[string]string{}
+	for _, key := range []string{"jwt", "erlang", "adminPassword", "kasApi", "kasPrivateApi", "kasRedis"} {
+		rand, err := crypto.RandStr(32)
+		if err != nil {
+			return "", err
+		}
+		randoms[key] = rand
+	}
+
+	client := api.FromConfig(&conf)
+	me, err := client.Me()
+	if err != nil {
+		return "", fmt.Errorf("you must run `plural login` before installing")
+	}
+
+	configuration := map[string]string{
+		"consoleDns":  consoleDns,
+		"kasDns":      kasDns,
+		"aesKey":      utils.GenAESKey(),
+		"adminName":   me.Email,
+		"adminEmail":  me.Email,
+		"clusterName": name,
+		"pluralToken": conf.Token,
+		"postgresUrl": dsn,
+	}
+	for k, v := range randoms {
+		configuration[k] = v
+	}
+
+	clientId, clientSecret, err := ensureInstalledAndOidc(client, consoleDns)
+	if err != nil {
+		return "", err
+	}
+	configuration["pluralClientId"] = clientId
+	configuration["pluralClientSecret"] = clientSecret
+
+	tpl, err := fetchTemplate()
+	if err != nil {
+		return "", err
+	}
+
+	bindings := map[string]interface{}{
+		"configuration": configuration,
+	}
+
+	res, err := liquidEngine.ParseAndRender(tpl, bindings)
+	return string(res), err
+}
+
 func CreateControlPlane(conf config.Config) (string, error) {
 	client := api.FromConfig(&conf)
 	me, err := client.Me()

pkg/provider/aws.go

@@ -237,6 +237,10 @@ func getAwsConfig(ctx context.Context) (aws.Config, error) {
 	return cfg, plrlErrors.ErrorWrap(err, "Failed to initialize aws client: ")
 }
 
+func (aws *AWSProvider) CreateBucket() error {
+	return aws.mkBucket(aws.bucket)
+}
+
 func (aws *AWSProvider) CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error) {
 	if err := aws.mkBucket(aws.bucket); err != nil {
 		return "", plrlErrors.ErrorWrap(err, fmt.Sprintf("Failed to create terraform state bucket %s", aws.bucket))

pkg/provider/azure.go

@@ -201,13 +201,21 @@ func AzureFromManifest(man *manifest.ProjectManifest, clientSet *ClientSet) (*Az
 	return &AzureProvider{man.Cluster, man.Project, man.Bucket, man.Region, man.Context, nil, clients}, nil
 }
 
-func (az *AzureProvider) CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error) {
+func (az *AzureProvider) CreateBucket() error {
 	if err := az.CreateResourceGroup(az.Project()); err != nil {
-		return "", pluralerr.ErrorWrap(err, fmt.Sprintf("Failed to create terraform state resource group %s", az.Project()))
+		return pluralerr.ErrorWrap(err, fmt.Sprintf("Failed to create terraform state resource group %s", az.Project()))
+	}
+
+	if err := az.createContainer(az.bucket); err != nil {
+		return pluralerr.ErrorWrap(err, fmt.Sprintf("Failed to create terraform state bucket %s", az.bucket))
 	}
 
-	if err := az.CreateBucket(az.bucket); err != nil {
-		return "", pluralerr.ErrorWrap(err, fmt.Sprintf("Failed to create terraform state bucket %s", az.bucket))
+	return nil
+}
+
+func (az *AzureProvider) CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error) {
+	if err := az.CreateBucket(); err != nil {
+		return "", err
 	}
 
 	ctx["Region"] = az.Region()
@@ -230,7 +238,7 @@ func (az *AzureProvider) CreateBackend(prefix string, version string, ctx map[st
 	return template.RenderString(scaffold, ctx)
 }
 
-func (az *AzureProvider) CreateBucket(bucket string) (err error) {
+func (az *AzureProvider) createContainer(bucket string) (err error) {
 	acc, err := az.upsertStorageAccount(utils.ToString(az.Context()["StorageAccount"]))
 	if err != nil {
 		return

pkg/provider/equinix.go

@@ -117,6 +117,10 @@ func equinixFromManifest(man *manifest.ProjectManifest) (*EQUINIXProvider, error
 	return &EQUINIXProvider{man.Cluster, man.Project, man.Bucket, man.Region, man.Context, nil}, nil
 }
 
+func (equinix *EQUINIXProvider) CreateBucket() error {
+	return nil
+}
+
 func (equinix *EQUINIXProvider) CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error) {
 
 	ctx["Region"] = equinix.Region()

pkg/provider/gcp.go

@@ -248,9 +248,14 @@ func (gcp *GCPProvider) Flush() error {
 	return gcp.writer()
 }
 
+func (gcp *GCPProvider) CreateBucket() error {
+	err := gcp.mkBucket(gcp.bucket)
+	return utilerr.ErrorWrap(err, fmt.Sprintf("Failed to create terraform state bucket %s", gcp.Bucket()))
+}
+
 func (gcp *GCPProvider) CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error) {
-	if err := gcp.mkBucket(gcp.bucket); err != nil {
-		return "", utilerr.ErrorWrap(err, fmt.Sprintf("Failed to create terraform state bucket %s", gcp.Bucket()))
+	if err := gcp.CreateBucket(); err != nil {
+		return "", err
 	}
 
 	ctx["Project"] = gcp.Project()

pkg/provider/kind.go

@@ -71,6 +71,8 @@ func kindFromManifest(man *manifest.ProjectManifest) (*KINDProvider, error) {
 	return &KINDProvider{man.Cluster, man.Project, man.Bucket, man.Region, man.Context, nil}, nil
 }
 
+func (kind *KINDProvider) CreateBucket() error { return nil }
+
 func (kind *KINDProvider) CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error) {
 
 	ctx["Region"] = kind.Region()

pkg/provider/provider.go

@@ -25,6 +25,7 @@ type Provider interface {
 	KubeConfig() error
 	KubeContext() string
 	CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error)
+	CreateBucket() error
 	Context() map[string]interface{}
 	Decommision(node *v1.Node) error
 	Preflights() []*Preflight

pkg/provider/test.go

@@ -44,6 +44,8 @@ func (t TestProvider) KubeContext() string {
 	return t.Clust
 }
 
+func (t TestProvider) CreateBucket() error { return nil }
+
 func (t TestProvider) CreateBackend(prefix string, version string, ctx map[string]interface{}) (string, error) {
 	return "test", nil
 }

pkg/up/context.go

@@ -0,0 +1,35 @@
+package up
+
+import (
+	"path/filepath"
+
+	"github.com/pluralsh/plural-cli/pkg/config"
+	"github.com/pluralsh/plural-cli/pkg/manifest"
+	"github.com/pluralsh/plural-cli/pkg/provider"
+)
+
+type Context struct {
+	Provider provider.Provider
+	Manifest *manifest.ProjectManifest
+	Config   *config.Config
+}
+
+func Build() (*Context, error) {
+	projPath, _ := filepath.Abs("workspace.yaml")
+	project, err := manifest.ReadProject(projPath)
+	if err != nil {
+		return nil, err
+	}
+
+	prov, err := provider.FromManifest(project)
+	if err != nil {
+		return nil, err
+	}
+
+	conf := config.Read()
+	return &Context{
+		Provider: prov,
+		Config:   &conf,
+		Manifest: project,
+	}, nil
+}

pkg/up/deploy.go

@@ -0,0 +1,27 @@
+package up
+
+import (
+	"os"
+	"os/exec"
+)
+
+func (ctx *Context) Deploy() error {
+	if err := ctx.Provider.CreateBucket(); err != nil {
+		return err
+	}
+
+	if err := terraform("./clusters", "init", "-upgrade"); err != nil {
+		return err
+	}
+
+	return terraform("./clusters", "apply")
+}
+
+func terraform(dir, cmd string, args ...string) error {
+	cmdArgs := append([]string{cmd}, args...)
+	osCmd := exec.Command("terraform", cmdArgs...)
+	osCmd.Dir = dir
+	osCmd.Stdout = os.Stdout
+	osCmd.Stderr = os.Stderr
+	return osCmd.Run()
+}