feat: support for credentials in AWS Secrets Manager (#174)

* feat: support for credentials in AWS Secrets Manager

* new field for secretsmanager instead of hacking existing password field

* improve docs

* fix tests

* unit test fetching credentials

* add flag for specifying secrets manager arn and improve error message

* more documentation

README.md

@@ -338,6 +338,26 @@ using [`os.ExpandEnv`](https://pkg.go.dev/os#ExpandEnv) at load time. The latter
 references of the form `$ENV_VAR_NAME` or `${ENV_VAR_NAME}` with the associated values from the
 environment.
 
+Additionally, the Amazon Resource Name (ARN) of a secret in AWS Secrets Manager can be provided
+instead of the username and password. Topicctl will then retrieve the secret value from Secrets 
+Manager and use it as the credentials. The secret in Secrets Manager must have a value in the format 
+shown below, identical to what [AWS MSK requires](https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html#msk-password-tutorial).
+```json
+{
+  "username": "alice",
+  "password": "alice-secret"
+}
+```
+
+An example of secrets manager being used can be seen below. Be sure to include the [6Random-Characters
+AWS Secrets Manager tacks on to the end of a secrets ARN](https://docs.aws.amazon.com/secretsmanager/latest/userguide/getting-started.html).
+```yaml
+sasl:
+    enabled: true
+    mechanism: SCRAM-SHA-512
+    secretsManagerArn: arn:aws:secretsmanager:<Region>:<AccountId>:secret:SecretName-6RandomCharacters
+```
+
 ### Topics
 
 Each topic is configured in a YAML file. The following is an

cmd/topicctl/subcmd/apply.go

@@ -197,9 +197,12 @@ func applyTopic(
 		adminClient, err = clusterConfig.NewAdminClient(
 			ctx,
 			nil,
-			applyConfig.dryRun,
-			applyConfig.shared.saslUsername,
-			applyConfig.shared.saslPassword,
+			config.AdminClientOpts{
+				ReadOnly:                  applyConfig.dryRun,
+				UsernameOverride:          applyConfig.shared.saslUsername,
+				PasswordOverride:          applyConfig.shared.saslPassword,
+				SecretsManagerArnOverride: applyConfig.shared.saslSecretsManagerArn,
+			},
 		)
 		if err != nil {
 			return err

cmd/topicctl/subcmd/bootstrap.go

@@ -72,9 +72,12 @@ func bootstrapRun(cmd *cobra.Command, args []string) error {
 	adminClient, err := clusterConfig.NewAdminClient(
 		ctx,
 		nil,
-		true,
-		bootstrapConfig.shared.saslUsername,
-		bootstrapConfig.shared.saslPassword,
+		config.AdminClientOpts{
+			ReadOnly:                  true,
+			UsernameOverride:          bootstrapConfig.shared.saslUsername,
+			PasswordOverride:          bootstrapConfig.shared.saslPassword,
+			SecretsManagerArnOverride: bootstrapConfig.shared.saslSecretsManagerArn,
+		},
 	)
 	if err != nil {
 		return err

cmd/topicctl/subcmd/check.go

@@ -137,9 +137,12 @@ func checkTopicFile(
 			adminClient, err = clusterConfig.NewAdminClient(
 				ctx,
 				nil,
-				true,
-				checkConfig.shared.saslUsername,
-				checkConfig.shared.saslPassword,
+				config.AdminClientOpts{
+					ReadOnly:                  true,
+					UsernameOverride:          checkConfig.shared.saslUsername,
+					PasswordOverride:          checkConfig.shared.saslPassword,
+					SecretsManagerArnOverride: checkConfig.shared.saslSecretsManagerArn,
+				},
 			)
 			if err != nil {
 				return false, err

cmd/topicctl/subcmd/create.go

@@ -150,9 +150,12 @@ func createACL(
 		adminClient, err = clusterConfig.NewAdminClient(
 			ctx,
 			nil,
-			createConfig.dryRun,
-			createConfig.shared.saslUsername,
-			createConfig.shared.saslPassword,
+			config.AdminClientOpts{
+				ReadOnly:                  createConfig.dryRun,
+				UsernameOverride:          createConfig.shared.saslUsername,
+				PasswordOverride:          createConfig.shared.saslPassword,
+				SecretsManagerArnOverride: createConfig.shared.saslSecretsManagerArn,
+			},
 		)
 		if err != nil {
 			return err

cmd/topicctl/subcmd/rebalance.go

@@ -125,9 +125,12 @@ func rebalanceRun(cmd *cobra.Command, args []string) error {
 
 	adminClient, err := clusterConfig.NewAdminClient(ctx,
 		nil,
-		rebalanceConfig.dryRun,
-		rebalanceConfig.shared.saslUsername,
-		rebalanceConfig.shared.saslPassword,
+		config.AdminClientOpts{
+			ReadOnly:                  rebalanceConfig.dryRun,
+			UsernameOverride:          rebalanceConfig.shared.saslUsername,
+			PasswordOverride:          rebalanceConfig.shared.saslPassword,
+			SecretsManagerArnOverride: rebalanceConfig.shared.saslSecretsManagerArn,
+		},
 	)
 	if err != nil {
 		log.Fatal(err)

cmd/topicctl/subcmd/shared.go

@@ -14,20 +14,21 @@ import (
 )
 
 type sharedOptions struct {
-	brokerAddr    string
-	clusterConfig string
-	expandEnv     bool
-	saslMechanism string
-	saslPassword  string
-	saslUsername  string
-	tlsCACert     string
-	tlsCert       string
-	tlsEnabled    bool
-	tlsKey        string
-	tlsSkipVerify bool
-	tlsServerName string
-	zkAddr        string
-	zkPrefix      string
+	brokerAddr            string
+	clusterConfig         string
+	expandEnv             bool
+	saslMechanism         string
+	saslPassword          string
+	saslUsername          string
+	saslSecretsManagerArn string
+	tlsCACert             string
+	tlsCert               string
+	tlsEnabled            bool
+	tlsKey                string
+	tlsSkipVerify         bool
+	tlsServerName         string
+	zkAddr                string
+	zkPrefix              string
 }
 
 func (s sharedOptions) validate() error {
@@ -76,7 +77,7 @@ func (s sharedOptions) validate() error {
 	}
 
 	useTLS := s.tlsEnabled || s.tlsCACert != "" || s.tlsCert != "" || s.tlsKey != ""
-	useSASL := s.saslMechanism != "" || s.saslPassword != "" || s.saslUsername != ""
+	useSASL := s.saslMechanism != "" || s.saslPassword != "" || s.saslUsername != "" || s.saslSecretsManagerArn != ""
 
 	if useTLS && s.zkAddr != "" {
 		log.Warn("TLS flags are ignored accessing cluster via zookeeper")
@@ -95,6 +96,10 @@ func (s sharedOptions) validate() error {
 			(s.saslUsername != "" || s.saslPassword != "") {
 			log.Warn("Username and password are ignored if using SASL AWS-MSK-IAM")
 		}
+
+		if s.saslUsername != "" || s.saslPassword != "" && s.saslSecretsManagerArn != "" {
+			err = multierror.Append(err, errors.New("Cannot set both sasl-username or sasl-password and sasl-secrets-manager-arn"))
+		}
 	}
 
 	return err
@@ -113,9 +118,12 @@ func (s sharedOptions) getAdminClient(
 		return clusterConfig.NewAdminClient(
 			ctx,
 			sess,
-			readOnly,
-			s.saslUsername,
-			s.saslPassword,
+			config.AdminClientOpts{
+				ReadOnly:                  readOnly,
+				UsernameOverride:          s.saslUsername,
+				PasswordOverride:          s.saslPassword,
+				SecretsManagerArnOverride: s.saslSecretsManagerArn,
+			},
 		)
 	} else if s.brokerAddr != "" {
 		tlsEnabled := (s.tlsEnabled ||
@@ -150,10 +158,11 @@ func (s sharedOptions) getAdminClient(
 						SkipVerify: s.tlsSkipVerify,
 					},
 					SASL: admin.SASLConfig{
-						Enabled:   saslEnabled,
-						Mechanism: saslMechanism,
-						Password:  s.saslPassword,
-						Username:  s.saslUsername,
+						Enabled:           saslEnabled,
+						Mechanism:         saslMechanism,
+						Password:          s.saslPassword,
+						Username:          s.saslUsername,
+						SecretsManagerArn: s.saslSecretsManagerArn,
 					},
 				},
 				ReadOnly: readOnly,
@@ -211,6 +220,12 @@ func addSharedFlags(cmd *cobra.Command, options *sharedOptions) {
 		os.Getenv("TOPICCTL_SASL_USERNAME"),
 		"SASL username if using SASL; will override value set in cluster config",
 	)
+	cmd.PersistentFlags().StringVar(
+		&options.saslSecretsManagerArn,
+		"sasl-secrets-manager-arn",
+		os.Getenv("TOPICCTL_SASL_SECRETS_MANAGER_ARN"),
+		"Secrets Manager ARN to use for credentials if using SASL; will override value set in cluster config",
+	)
 	cmd.PersistentFlags().StringVar(
 		&options.tlsCACert,
 		"tls-ca-cert",

examples/auth/cluster.yaml

@@ -31,3 +31,14 @@ spec:
     #
     username: adminscram
     password: admin-secret-512
+    #
+    # Another alternative is to omit these values and use secretsManagerArn to reference
+    # an AWS Secrets Manager secret containing the username and password. More information
+    # can be found in the README. An example can be seen below.
+    # secretsManagerArn: arn:aws:secretsmanager:us-west-2:1000000000:secret:AmazonMSK_kafka-admin-wEiwjV
+    #
+    # This can also be set via:
+    #
+    #   1. The --sasl-secrets-manager-arn command-line flag,
+    #   2. The TOPICCTL_SASL_SECRETS_MANAGER_ARN environment variable, or
+    #   3. Putting a placeholder string in the config and running with the --expand-env flag as

go.mod

@@ -3,7 +3,7 @@ module github.com/segmentio/topicctl
 go 1.18
 
 require (
-	github.com/aws/aws-sdk-go v1.44.208
+	github.com/aws/aws-sdk-go v1.49.12
 	github.com/briandowns/spinner v1.19.0
 	github.com/c-bata/go-prompt v0.2.3
 	github.com/fatih/color v1.13.0

go.sum

@@ -1,6 +1,6 @@
 github.com/aws/aws-sdk-go v1.41.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
-github.com/aws/aws-sdk-go v1.44.208 h1:xk1E2zWAIskrOP+huXuCYFR9ZdQWfTVid8Cjiwj2H1o=
-github.com/aws/aws-sdk-go v1.44.208/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.49.12 h1:SbGHDdMjtuTL8zpRXKjvIvQHLt9cCqcxcHoJps23WxI=
+github.com/aws/aws-sdk-go v1.49.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/briandowns/spinner v1.19.0 h1:s8aq38H+Qju89yhp89b4iIiMzMm8YN3p6vGpwyh/a8E=
 github.com/briandowns/spinner v1.19.0/go.mod h1:mQak9GHqbspjC/5iUx3qMlIho8xBS/ppAL/hX5SmPJU=
 github.com/c-bata/go-prompt v0.2.3 h1:jjCS+QhG/sULBhAaBdjb2PlMRVaKXQgn+4yzaauvs2s=
@@ -116,7 +116,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
@@ -141,15 +140,13 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
@@ -160,7 +157,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

pkg/acl/acl_test.go

@@ -673,7 +673,7 @@ func testACLAdmin(
 		},
 	}
 
-	adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "")
+	adminClient, err := clusterConfig.NewAdminClient(ctx, nil, config.AdminClientOpts{})
 	require.NoError(t, err)
 
 	aclAdmin, err := NewACLAdmin(