From b86d182268555bfb4685fe0135fa788de25e695a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 11 Aug 2023 11:10:02 -0400 Subject: [PATCH 01/57] bump kafka-go to include acl apis --- go.mod | 10 +++++----- go.sum | 12 ++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 4293590e..294f18c6 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.35 + github.com/segmentio/kafka-go v0.4.42 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 @@ -25,7 +25,7 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.15.7 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect @@ -39,9 +39,9 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/xdg/scram v1.0.5 // indirect github.com/xdg/stringprep v1.0.3 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 4e4f327e..43f1bcdd 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -76,6 +78,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= github.com/segmentio/kafka-go v0.4.35 h1:TAsQ7q1SjS39PcFvU0zDJhCuVAxHomy7xOAfbdSuhzs= github.com/segmentio/kafka-go v0.4.35/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= +github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= +github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= @@ -93,6 +97,9 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= @@ -116,6 +123,7 @@ golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -137,16 +145,20 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From f1ec537ab4165dac41eaedac65ae97e4dae5cfab Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 11 Aug 2023 11:12:07 -0400 Subject: [PATCH 02/57] add acl interfaces and aclinfo type stub --- pkg/admin/client.go | 20 ++++++++++++++++++++ pkg/admin/types.go | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 733a9ab0..46f3fbdf 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -94,4 +94,24 @@ type Client interface { // Close closes the client. Close() error + + // GetACLs gets full information about each ACL in the cluster. + GetACLs( + ctx context.Context, + names []string, + detailed bool, + ) ([]ACLInfo, error) + + // GetACL gets the details of a single ACL in the cluster. + GetACL( + ctx context.Context, + name string, + detailed bool, + ) (ACLInfo, error) + + // CreateACL creates an ACL in the cluster. + CreateACL( + ctx context.Context, + config kafka.ACLEntry, + ) error } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 0221a39b..10a2c3d3 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -792,3 +792,7 @@ func NewLeaderPartitions( return newLeaderPartitions } + +// ACLInfo represents the information stored about an ACL in zookeeper. +type ACLInfo struct { +} From 07a63c7815873d1fb60e109893924b4370185c3b Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 11 Aug 2023 11:52:07 -0400 Subject: [PATCH 03/57] pull latest kafka-go and use kafka-go aclresource type --- go.mod | 2 +- go.sum | 2 ++ pkg/admin/client.go | 4 ++-- pkg/admin/types.go | 4 ---- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 294f18c6..f3e7e4ff 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.42 + github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 43f1bcdd..7bbae149 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,8 @@ github.com/segmentio/kafka-go v0.4.35 h1:TAsQ7q1SjS39PcFvU0zDJhCuVAxHomy7xOAfbdS github.com/segmentio/kafka-go v0.4.35/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= +github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= +github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 46f3fbdf..45c467c7 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -100,14 +100,14 @@ type Client interface { ctx context.Context, names []string, detailed bool, - ) ([]ACLInfo, error) + ) ([]kafka.ACLResource, error) // GetACL gets the details of a single ACL in the cluster. GetACL( ctx context.Context, name string, detailed bool, - ) (ACLInfo, error) + ) (kafka.ACLResource, error) // CreateACL creates an ACL in the cluster. CreateACL( diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 10a2c3d3..0221a39b 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -792,7 +792,3 @@ func NewLeaderPartitions( return newLeaderPartitions } - -// ACLInfo represents the information stored about an ACL in zookeeper. -type ACLInfo struct { -} From 474c260efbc47cb8b622ca3da223def3daab714d Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 8 Sep 2023 09:25:34 -0400 Subject: [PATCH 04/57] wip --- docker-compose-auth.yml | 2 ++ go.mod | 5 +-- go.sum | 23 ++++---------- pkg/admin/brokerclient.go | 49 ++++++++++++++++++++++++++++ pkg/admin/brokerclient_test.go | 58 ++++++++++++++++++++++++++++++++++ pkg/admin/client.go | 20 ------------ pkg/util/testing.go | 10 ++++++ 7 files changed, 129 insertions(+), 38 deletions(-) diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml index 16b70690..7897cb8d 100644 --- a/docker-compose-auth.yml +++ b/docker-compose-auth.yml @@ -34,6 +34,8 @@ services: KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095" KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512" + KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer' + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' KAFKA_SSL_KEYSTORE_LOCATION: /certs/kafka.keystore.jks KAFKA_SSL_KEYSTORE_PASSWORD: test123 KAFKA_SSL_KEY_PASSWORD: test123 diff --git a/go.mod b/go.mod index f3e7e4ff..7f969696 100644 --- a/go.mod +++ b/go.mod @@ -37,8 +37,9 @@ require ( github.com/pkg/term v0.0.0-20200520122047-c3ffed290a03 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/xdg/scram v1.0.5 // indirect - github.com/xdg/stringprep v1.0.3 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect diff --git a/go.sum b/go.sum index 7bbae149..4d6c3b80 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,6 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= -github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -76,10 +74,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= -github.com/segmentio/kafka-go v0.4.35 h1:TAsQ7q1SjS39PcFvU0zDJhCuVAxHomy7xOAfbdSuhzs= -github.com/segmentio/kafka-go v0.4.35/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= -github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= -github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= @@ -99,15 +93,14 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= -github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= -github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -120,11 +113,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -145,21 +136,21 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 4bc4063b..4246ea7c 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -690,3 +690,52 @@ func configEntriesToAPIConfigs( return apiConfigs } + +// TODO: what fields should we let people filter on / how best to support that? +// It could be really useful to be able to use this to answer questions like what services have access topics x,y,z or +// who has write access to topic b? +// Is GetACL (single ACL) even applicable? +// GetACLs gets full information about each ACL in the cluster. +func (c *BrokerAdminClient) GetACLs( + ctx context.Context, + filter kafka.ACLFilter, +) ([]kafka.ACLResource, error) { + req := kafka.DescribeACLsRequest{ + Filter: filter, + } + log.Debugf("DescribeACLs request: %+v", req) + + resp, err := c.client.DescribeACLs(ctx, &req) + log.Debugf("DescribeACLs response: %+v (%+v)", resp, err) + if err != nil { + return nil, err + } + return resp.Resources, nil +} + +// CreateACL creates an ACL in the cluster. +func (c *BrokerAdminClient) CreateACL( + ctx context.Context, + entry kafka.ACLEntry, +) error { + if c.config.ReadOnly { + return errors.New("Cannot create ACL in read-only mode") + } + + req := kafka.CreateACLsRequest{ + ACLs: []kafka.ACLEntry{ + entry, + }, + } + log.Debugf("CreateACLs request: %+v", req) + + resp, err := c.client.CreateACLs(ctx, &req) + log.Debugf("CreateACLs response: %+v (%+v)", resp, err) + if err != nil { + return err + } + if len(resp.Errors) > 0 { + fmt.Errorf("%+v", resp.Errors) + } + return nil +} diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index d998df84..0c494ca3 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -574,3 +574,61 @@ func TestBrokerClientCreateTopicError(t *testing.T) { ) require.Error(t, err) } + +func TestBrokerClientCreateGetACL(t *testing.T) { + if !util.CanTestBrokerAdminSecurity() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") + } + + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, + ) + require.NoError(t, err) + + principal := util.RandomString("User:user-create-", 6) + topicName := util.RandomString("topic-create-", 6) + + err = client.CreateACL( + ctx, + kafka.ACLEntry{ + Principal: principal, + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + ResourceType: kafka.ResourceTypeTopic, + ResourcePatternType: kafka.PatternTypeLiteral, + ResourceName: topicName, + Host: "*", + }, + ) + require.NoError(t, err) + + filter := kafka.ACLFilter{ + ResourceNameFilter: topicName, + } + + aclsInfo, err := client.GetACLs(ctx, filter) + require.NoError(t, err) + expected := []kafka.ACLResource{ + { + ResourceType: kafka.ResourceTypeTopic, + ResourceName: topicName, + PatternType: kafka.PatternTypeLiteral, + ACLs: []kafka.ACLDescription{ + { + Principal: principal, + Host: "*", + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, + }, + }, + }, + } + assert.Equal(t, expected, aclsInfo) + +} diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 45c467c7..733a9ab0 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -94,24 +94,4 @@ type Client interface { // Close closes the client. Close() error - - // GetACLs gets full information about each ACL in the cluster. - GetACLs( - ctx context.Context, - names []string, - detailed bool, - ) ([]kafka.ACLResource, error) - - // GetACL gets the details of a single ACL in the cluster. - GetACL( - ctx context.Context, - name string, - detailed bool, - ) (kafka.ACLResource, error) - - // CreateACL creates an ACL in the cluster. - CreateACL( - ctx context.Context, - config kafka.ACLEntry, - ) error } diff --git a/pkg/util/testing.go b/pkg/util/testing.go index 2734086b..e81e412c 100644 --- a/pkg/util/testing.go +++ b/pkg/util/testing.go @@ -48,6 +48,16 @@ func CanTestBrokerAdmin() bool { return false } +// CanTestBrokerAdminSecurity returns whether we can test the broker-only admin client security features. +func CanTestBrokerAdminSecurity() bool { + value, ok := os.LookupEnv("KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY") + if ok && value != "" { + return true + } + + return false +} + var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") // RandomString returns a random string with the argument length. From 6e0ec36e1298e0e0aa72a3ea6103c7c5ad0986f0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 8 Sep 2023 11:21:29 -0400 Subject: [PATCH 05/57] fix test --- pkg/admin/brokerclient_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 0c494ca3..918c422e 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -577,7 +577,7 @@ func TestBrokerClientCreateTopicError(t *testing.T) { func TestBrokerClientCreateGetACL(t *testing.T) { if !util.CanTestBrokerAdminSecurity() { - t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") } ctx := context.Background() @@ -586,6 +586,12 @@ func TestBrokerClientCreateGetACL(t *testing.T) { BrokerAdminClientConfig{ ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), + SASL: SASLConfig{ + Enabled: true, + Mechanism: SASLMechanismScramSHA512, + Username: "adminscram", + Password: "admin-secret-512", + }, }, }, ) @@ -609,7 +615,11 @@ func TestBrokerClientCreateGetACL(t *testing.T) { require.NoError(t, err) filter := kafka.ACLFilter{ - ResourceNameFilter: topicName, + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, } aclsInfo, err := client.GetACLs(ctx, filter) From 7b9454d4f0d930e4c3c53fe5646e7daa486e5f53 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 11 Sep 2023 15:08:52 -0400 Subject: [PATCH 06/57] fix typos --- cmd/topicctl/subcmd/get.go | 6 +++- pkg/admin/brokerclient.go | 2 +- pkg/admin/brokerclient_test.go | 5 ++-- pkg/admin/client.go | 6 ++++ pkg/admin/format.go | 54 ++++++++++++++++++++++++++++++++++ pkg/admin/types.go | 12 ++++++++ pkg/admin/zkclient.go | 7 +++++ pkg/cli/cli.go | 17 ++++++++++- 8 files changed, 104 insertions(+), 5 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 804c851d..5b028058 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -17,7 +17,7 @@ var getCmd = &cobra.Command{ Long: strings.Join( []string{ "Get instances of a particular type.", - "Supported types currently include: balance, brokers, config, groups, lags, members, partitions, offsets, and topics.", + "Supported types currently include: balance, brokers, config, groups, lags, members, partitions, offsets, topics, and acls.", "", "See the tool README for a detailed description of each one.", }, @@ -140,6 +140,10 @@ func getRun(cmd *cobra.Command, args []string) error { } return cliRunner.GetTopics(ctx, getConfig.full) + case "acls": + // TODO: add arg validation once we figure out filtering args + + return cliRunner.GetAcls(ctx, nil) default: return fmt.Errorf("Unrecognized resource type: %s", resource) } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 4246ea7c..79c36b35 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -699,7 +699,7 @@ func configEntriesToAPIConfigs( func (c *BrokerAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, -) ([]kafka.ACLResource, error) { +) ([]ACLInfo, error) { req := kafka.DescribeACLsRequest{ Filter: filter, } diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 918c422e..ed8e053d 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -589,8 +589,9 @@ func TestBrokerClientCreateGetACL(t *testing.T) { SASL: SASLConfig{ Enabled: true, Mechanism: SASLMechanismScramSHA512, - Username: "adminscram", - Password: "admin-secret-512", + // TODO: don't hardcode these in tests, pull from env vars + Username: "adminscram", + Password: "admin-secret-512", }, }, }, diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 733a9ab0..c3b4bf8c 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -38,6 +38,12 @@ type Client interface { detailed bool, ) (TopicInfo, error) + // GetACLs gets full information about each ACL in the cluster. + GetACLs( + ctx context.Context, + filter kafka.ACLFilter, + ) ([]ACLInfo, error) + // UpdateTopicConfig updates the configuration for the argument topic. It returns the config // keys that were updated. UpdateTopicConfig( diff --git a/pkg/admin/format.go b/pkg/admin/format.go index c6b9c64f..ec7e4645 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -745,6 +745,60 @@ func FormatBrokerMaxPartitions( return string(bytes.TrimRight(buf.Bytes(), "\n")) } +// FormatAcls creates a pretty table that lists the details of the +// argument acls. +func FormatAcls(acls []ACLInfo) string { + buf := &bytes.Buffer{} + + headers := []string{ + "Resource Type", + "Resource Name", + "Principal", + "Host", + "Operation", + "Permission Type", + } + + table := tablewriter.NewWriter(buf) + table.SetHeader(headers) + table.SetAutoWrapText(false) + table.SetColumnAlignment( + []int{ + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, + }, + ) + table.SetBorders( + tablewriter.Border{ + Left: false, + Top: true, + Right: false, + Bottom: true, + }, + ) + + for _, acl := range acls { + row := []string{ + // TODO: convert ints to something human readable + fmt.Sprintf("%d", acl.ResourceType), + acl.ResourceName, + acl.Principal, + acl.Host, + fmt.Sprintf("%d", acl.Operation), + fmt.Sprintf("%d", acl.PermissionType), + } + + table.Append(row) + } + + table.Render() + return string(bytes.TrimRight(buf.Bytes(), "\n")) +} + func prettyConfig(config map[string]string) string { rows := []string{} diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 0221a39b..6ab33ba8 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -8,6 +8,7 @@ import ( "strconv" "time" + "github.com/segmentio/kafka-go" "github.com/segmentio/topicctl/pkg/util" ) @@ -73,6 +74,17 @@ type PartitionAssignment struct { Replicas []int `json:"replicas"` } +// PartitionInfo represents the information stored about an ACL +// in zookeeper. +type ACLInfo struct { + ResourceType kafka.ResourceType + ResourceName string + Principal string + Host string + Operation kafka.ACLOperationType + PermissionType kafka.ACLPermissionType +} + type zkClusterID struct { Version string `json:"version"` ID string `json:"id"` diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 575dee8e..07168774 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -422,6 +422,13 @@ func (c *ZKAdminClient) GetTopic( return topics[0], nil } +func (c *ZKAdminClient) GetACLs( + ctx context.Context, + filter kafka.ACLFilter, +) ([]ACLInfo, error) { + return nil, nil +} + // UpdateTopicConfig updates the config JSON for a topic and sets a change // notification so that the brokers are notified. If overwrite is true, then // it will overwrite existing config entries. diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 7b0542c7..c0caa45c 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -476,7 +476,6 @@ func (c *CLIRunner) GetTopics(ctx context.Context, full bool) error { c.stopSpinner() return err } - brokers, err := c.adminClient.GetBrokers(ctx, nil) c.stopSpinner() if err != nil { return err @@ -552,6 +551,22 @@ func (c *CLIRunner) Tail( return err } +// TODO add options for filtering +// GetAcls fetches the details of each acl in the cluster and prints out a summary. +func (c *CLIRunner) GetAcls(ctx context.Context) error { + c.startSpinner() + + acls, err := c.adminClient.GetAcls(ctx, nil) + c.stopSpinner() + if err != nil { + return err + } + + c.printer("Acls:\n%s", admin.FormatAcls(acls)) + + return nil +} + func (c *CLIRunner) startSpinner() { if c.spinnerObj != nil { c.spinnerObj.Start() From 49f7e1936c9b91859b939b3e54281ea7faf86d8a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 11 Sep 2023 15:22:10 -0400 Subject: [PATCH 07/57] get acls working --- pkg/admin/brokerclient.go | 22 +++++++++++++++++++++- pkg/admin/brokerclient_test.go | 19 +++++++------------ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 79c36b35..6f0cf20a 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -710,7 +710,27 @@ func (c *BrokerAdminClient) GetACLs( if err != nil { return nil, err } - return resp.Resources, nil + + if resp.Error != nil { + return nil, resp.Error + } + + aclinfos := []ACLInfo{} + + for _, resource := range resp.Resources { + for _, acl := range resource.ACLs { + aclinfos = append(aclinfos, ACLInfo{ + ResourceType: resource.ResourceType, + ResourceName: resource.ResourceName, + Principal: acl.Principal, + Host: acl.Host, + Operation: acl.Operation, + PermissionType: acl.PermissionType, + }) + } + } + + return aclinfos, nil } // CreateACL creates an ACL in the cluster. diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index ed8e053d..d4b896e4 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -625,19 +625,14 @@ func TestBrokerClientCreateGetACL(t *testing.T) { aclsInfo, err := client.GetACLs(ctx, filter) require.NoError(t, err) - expected := []kafka.ACLResource{ + expected := []ACLInfo{ { - ResourceType: kafka.ResourceTypeTopic, - ResourceName: topicName, - PatternType: kafka.PatternTypeLiteral, - ACLs: []kafka.ACLDescription{ - { - Principal: principal, - Host: "*", - Operation: kafka.ACLOperationTypeRead, - PermissionType: kafka.ACLPermissionTypeAllow, - }, - }, + ResourceType: kafka.ResourceTypeTopic, + ResourceName: topicName, + Principal: principal, + Host: "*", + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, }, } assert.Equal(t, expected, aclsInfo) From 8382e98624ecc4efbb9813fe0c752b55c39afa5a Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 11:21:36 -0400 Subject: [PATCH 08/57] getacls working --- cmd/topicctl/subcmd/get.go | 4 ++-- pkg/admin/brokerclient.go | 7 +++++++ pkg/admin/brokerclient_test.go | 1 + pkg/admin/support.go | 3 +++ pkg/admin/types.go | 1 + pkg/admin/zkclient.go | 4 +++- pkg/cli/cli.go | 13 ++++++++++--- 7 files changed, 27 insertions(+), 6 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 5b028058..7acc36b1 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -59,6 +59,7 @@ func getPreRun(cmd *cobra.Command, args []string) error { return getConfig.shared.validate() } +// TODO: make each of these "gets" a separate subcommand func getRun(cmd *cobra.Command, args []string) error { ctx := context.Background() sess := session.Must(session.NewSession()) @@ -142,8 +143,7 @@ func getRun(cmd *cobra.Command, args []string) error { return cliRunner.GetTopics(ctx, getConfig.full) case "acls": // TODO: add arg validation once we figure out filtering args - - return cliRunner.GetAcls(ctx, nil) + return cliRunner.GetACLs(ctx) default: return fmt.Errorf("Unrecognized resource type: %s", resource) } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 6f0cf20a..fc34e000 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -98,6 +98,12 @@ func NewBrokerAdminClient( if _, ok := maxVersions["AlterClientQuotas"]; ok { supportedFeatures.DynamicBrokerConfigs = true } + + // If we have DescribeAcls, than we're running a version of Kafka > 2.0.1, + // that will have support for all ACLs APIs. + if _, ok := maxVersions["DescribeAcls"]; ok { + supportedFeatures.ACLs = true + } log.Debugf("Supported features: %+v", supportedFeatures) adminClient := &BrokerAdminClient{ @@ -722,6 +728,7 @@ func (c *BrokerAdminClient) GetACLs( aclinfos = append(aclinfos, ACLInfo{ ResourceType: resource.ResourceType, ResourceName: resource.ResourceName, + PatternType: resource.PatternType, Principal: acl.Principal, Host: acl.Host, Operation: acl.Operation, diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index d4b896e4..2d8c21b6 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -629,6 +629,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { { ResourceType: kafka.ResourceTypeTopic, ResourceName: topicName, + PatternType: kafka.PatternTypeLiteral, Principal: principal, Host: "*", Operation: kafka.ACLOperationTypeRead, diff --git a/pkg/admin/support.go b/pkg/admin/support.go index 61c91118..242032b4 100644 --- a/pkg/admin/support.go +++ b/pkg/admin/support.go @@ -16,4 +16,7 @@ type SupportedFeatures struct { // DynamicBrokerConfigs indicates whether the client can return dynamic broker configs // like leader.replication.throttled.rate. DynamicBrokerConfigs bool + + // ACLs indicates whether the client supports access control levels. + ACLs bool } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 6ab33ba8..125f2160 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -79,6 +79,7 @@ type PartitionAssignment struct { type ACLInfo struct { ResourceType kafka.ResourceType ResourceName string + PatternType kafka.PatternType Principal string Host string Operation kafka.ACLOperationType diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 07168774..fa032764 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -763,12 +763,14 @@ func (c *ZKAdminClient) LockHeld( // GetSupportedFeatures returns the features that are supported by this client. func (c *ZKAdminClient) GetSupportedFeatures() SupportedFeatures { - // The zk-based client supports everything. + // The zk-based client supports everything except for ACLs. + // Zookeeper can support ACLs, topicctl just hasn't added support for it yet. return SupportedFeatures{ Reads: true, Applies: true, Locks: true, DynamicBrokerConfigs: true, + ACLs: false, } } diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index c0caa45c..68223baa 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -14,6 +14,7 @@ import ( "github.com/briandowns/spinner" "github.com/fatih/color" + "github.com/segmentio/kafka-go" "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/check" @@ -476,6 +477,7 @@ func (c *CLIRunner) GetTopics(ctx context.Context, full bool) error { c.stopSpinner() return err } + brokers, err := c.adminClient.GetBrokers(ctx, nil) c.stopSpinner() if err != nil { return err @@ -553,16 +555,21 @@ func (c *CLIRunner) Tail( // TODO add options for filtering // GetAcls fetches the details of each acl in the cluster and prints out a summary. -func (c *CLIRunner) GetAcls(ctx context.Context) error { +func (c *CLIRunner) GetACLs(ctx context.Context) error { c.startSpinner() - acls, err := c.adminClient.GetAcls(ctx, nil) + acls, err := c.adminClient.GetACLs(ctx, kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceTypeAny, + ResourcePatternTypeFilter: kafka.PatternTypeAny, + Operation: kafka.ACLOperationTypeAny, + PermissionType: kafka.ACLPermissionTypeAny, + }) c.stopSpinner() if err != nil { return err } - c.printer("Acls:\n%s", admin.FormatAcls(acls)) + c.printer("ACLs:\n%s", admin.FormatAcls(acls)) return nil } From 7b8ee4282d309441a9586db13dde05c5ec30eff5 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 11:50:30 -0400 Subject: [PATCH 09/57] upgrade cobra to latest --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4293590e..0f232842 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/segmentio/kafka-go v0.4.35 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d @@ -23,7 +23,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/mattn/go-colorable v0.1.9 // indirect diff --git a/go.sum b/go.sum index 4e4f327e..b8fb8ecf 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -80,8 +80,8 @@ github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d07 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 2a7d2decaf30d7fd75b98852b8d0a76e724bb205 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 14:32:36 -0400 Subject: [PATCH 10/57] finish separating get into separate subcommands --- cmd/topicctl/subcmd/get.go | 316 ++++++++++++++++++++++++---------- cmd/topicctl/subcmd/shared.go | 28 +-- 2 files changed, 239 insertions(+), 105 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 804c851d..67f8332b 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -2,7 +2,6 @@ package subcmd import ( "context" - "fmt" "strings" "github.com/aws/aws-sdk-go/aws/session" @@ -17,15 +16,10 @@ var getCmd = &cobra.Command{ Long: strings.Join( []string{ "Get instances of a particular type.", - "Supported types currently include: balance, brokers, config, groups, lags, members, partitions, offsets, and topics.", - "", - "See the tool README for a detailed description of each one.", }, "\n", ), - Args: cobra.MinimumNArgs(1), - PreRunE: getPreRun, - RunE: getRun, + PersistentPreRunE: getPreRun, } type getCmdConfig struct { @@ -38,20 +32,30 @@ type getCmdConfig struct { var getConfig getCmdConfig func init() { - getCmd.Flags().BoolVar( + getCmd.PersistentFlags().BoolVar( &getConfig.full, "full", false, "Show more full information for resources", ) - getCmd.Flags().BoolVar( + getCmd.PersistentFlags().BoolVar( &getConfig.sortValues, "sort-values", false, "Sort by value instead of name; only applies for lags at the moment", ) - addSharedFlags(getCmd, &getConfig.shared) + getCmd.AddCommand( + balanceCmd(), + brokersCmd(), + configCmd(), + groupsCmd(), + lagsCmd(), + membersCmd(), + partitionsCmd(), + offsetsCmd(), + topicsCmd(), + ) RootCmd.AddCommand(getCmd) } @@ -59,88 +63,218 @@ func getPreRun(cmd *cobra.Command, args []string) error { return getConfig.shared.validate() } -func getRun(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) +func balanceCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "balance [optional topic]", + Short: "Number of replicas per broker position for topic or cluster as a whole", + Long: strings.Join([]string{ + "Displays the number of replicas per broker position.", + "Accepts an optional argument of a topic, which will just scope this to that topic. If topic is omitted, the balance displayed will be for the entire cluster.", + }, + "\n", + ), + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + + var topicName string + if len(args) == 1 { + topicName = args[0] + } + return cliRunner.GetBrokerBalance(ctx, topicName) + }, + PreRunE: getPreRun, + } + return cmd +} + +func brokersCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "brokers", + Short: "Displays descriptions of each broker in the cluster.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) - if err != nil { - return err + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetBrokers(ctx, getConfig.full) + }, } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) - - resource := args[0] - - switch resource { - case "balance": - var topicName string - - if len(args) == 2 { - topicName = args[1] - } else if len(args) > 2 { - return fmt.Errorf("Can provide at most one positional argument with brokers") - } - - return cliRunner.GetBrokerBalance(ctx, topicName) - case "brokers": - if len(args) > 1 { - return fmt.Errorf("Can only provide one positional argument with brokers") - } - - return cliRunner.GetBrokers(ctx, getConfig.full) - case "config": - if len(args) != 2 { - return fmt.Errorf("Must provide broker ID or topic name as second positional argument") - } - - return cliRunner.GetConfig(ctx, args[1]) - case "groups": - if len(args) > 1 { - return fmt.Errorf("Can only provide one positional argument with groups") - } - - return cliRunner.GetGroups(ctx) - case "lags": - if len(args) != 3 { - return fmt.Errorf("Must provide topic and groupID as additional positional arguments") - } - - return cliRunner.GetMemberLags( - ctx, - args[1], - args[2], - getConfig.full, - getConfig.sortValues, - ) - case "members": - if len(args) != 2 { - return fmt.Errorf("Must provide group ID as second positional argument") - } - - return cliRunner.GetGroupMembers(ctx, args[1], getConfig.full) - case "partitions": - if len(args) != 2 { - return fmt.Errorf("Must provide topic as second positional argument") - } - topicName := args[1] - - return cliRunner.GetPartitions(ctx, topicName) - case "offsets": - if len(args) != 2 { - return fmt.Errorf("Must provide topic as second positional argument") - } - topicName := args[1] - - return cliRunner.GetOffsets(ctx, topicName) - case "topics": - if len(args) > 1 { - return fmt.Errorf("Can only provide one positional argument with args") - } - - return cliRunner.GetTopics(ctx, getConfig.full) - default: - return fmt.Errorf("Unrecognized resource type: %s", resource) + return cmd +} + +func configCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config [broker or topic]", + Short: "Displays configuration for the provider broker or topic.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetConfig(ctx, args[0]) + }, + } + return cmd +} + +func groupsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "groups", + Short: "Displays consumer group informatin for the cluster.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetGroups(ctx) + }, + } + return cmd +} + +func lagsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "lags [topic] [group]", + Short: "Displays consumer group lag for the specified topic and consumer group.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetMemberLags( + ctx, + args[0], + args[1], + getConfig.full, + getConfig.sortValues, + ) + }, + } + return cmd +} + +func membersCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "members [group]", + Short: "Details of each member in the specified consumer group.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetGroupMembers(ctx, args[0], getConfig.full) + }, + } + return cmd +} + +func partitionsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "partitions [topic]", + Short: "Displays partition information for the specified topic.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetPartitions(ctx, args[0]) + }, + } + return cmd +} + +func offsetsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "offsets [topic]", + Short: "Displays offset information for the specified topic along with start and end times.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetOffsets(ctx, args[0]) + }, + } + return cmd +} + +func topicsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "topics", + Short: "Displays information for all topics in the cluster.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetTopics(ctx, getConfig.full) + }, } + return cmd } diff --git a/cmd/topicctl/subcmd/shared.go b/cmd/topicctl/subcmd/shared.go index a73f1e44..7f5d925c 100644 --- a/cmd/topicctl/subcmd/shared.go +++ b/cmd/topicctl/subcmd/shared.go @@ -173,88 +173,88 @@ func (s sharedOptions) getAdminClient( } func addSharedFlags(cmd *cobra.Command, options *sharedOptions) { - cmd.Flags().StringVarP( + cmd.PersistentFlags().StringVarP( &options.brokerAddr, "broker-addr", "b", "", "Broker address", ) - cmd.Flags().BoolVarP( + cmd.PersistentFlags().BoolVarP( &options.expandEnv, "expand-env", "", false, "Expand environment in cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.clusterConfig, "cluster-config", os.Getenv("TOPICCTL_CLUSTER_CONFIG"), "Cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.saslMechanism, "sasl-mechanism", "", "SASL mechanism if using SASL (choices: AWS-MSK-IAM, PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512)", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.saslPassword, "sasl-password", os.Getenv("TOPICCTL_SASL_PASSWORD"), "SASL password if using SASL; will override value set in cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.saslUsername, "sasl-username", os.Getenv("TOPICCTL_SASL_USERNAME"), "SASL username if using SASL; will override value set in cluster config", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsCACert, "tls-ca-cert", "", "Path to client CA cert PEM file if using TLS", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsCert, "tls-cert", "", "Path to client cert PEM file if using TLS", ) - cmd.Flags().BoolVar( + cmd.PersistentFlags().BoolVar( &options.tlsEnabled, "tls-enabled", false, "Use TLS for communication with brokers", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsKey, "tls-key", "", "Path to client private key PEM file if using TLS", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.tlsServerName, "tls-server-name", "", "Server name to use for TLS cert verification", ) - cmd.Flags().BoolVar( + cmd.PersistentFlags().BoolVar( &options.tlsSkipVerify, "tls-skip-verify", false, "Skip hostname verification when using TLS", ) - cmd.Flags().StringVarP( + cmd.PersistentFlags().StringVarP( &options.zkAddr, "zk-addr", "z", "", "ZooKeeper address", ) - cmd.Flags().StringVar( + cmd.PersistentFlags().StringVar( &options.zkPrefix, "zk-prefix", "", From 1b84ef384c12aa83494141102a4a47e135054ffb Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 15:17:15 -0400 Subject: [PATCH 11/57] remove unneeded variables --- cmd/topicctl/subcmd/get.go | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 67f8332b..33f7abfd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -64,7 +64,7 @@ func getPreRun(cmd *cobra.Command, args []string) error { } func balanceCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "balance [optional topic]", Short: "Number of replicas per broker position for topic or cluster as a whole", Long: strings.Join([]string{ @@ -94,11 +94,10 @@ func balanceCmd() *cobra.Command { }, PreRunE: getPreRun, } - return cmd } func brokersCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "brokers", Short: "Displays descriptions of each broker in the cluster.", Args: cobra.NoArgs, @@ -116,11 +115,10 @@ func brokersCmd() *cobra.Command { return cliRunner.GetBrokers(ctx, getConfig.full) }, } - return cmd } func configCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "config [broker or topic]", Short: "Displays configuration for the provider broker or topic.", Args: cobra.ExactArgs(1), @@ -138,11 +136,10 @@ func configCmd() *cobra.Command { return cliRunner.GetConfig(ctx, args[0]) }, } - return cmd } func groupsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "groups", Short: "Displays consumer group informatin for the cluster.", Args: cobra.NoArgs, @@ -160,11 +157,10 @@ func groupsCmd() *cobra.Command { return cliRunner.GetGroups(ctx) }, } - return cmd } func lagsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "lags [topic] [group]", Short: "Displays consumer group lag for the specified topic and consumer group.", Args: cobra.ExactArgs(2), @@ -188,11 +184,10 @@ func lagsCmd() *cobra.Command { ) }, } - return cmd } func membersCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "members [group]", Short: "Details of each member in the specified consumer group.", Args: cobra.ExactArgs(1), @@ -210,11 +205,10 @@ func membersCmd() *cobra.Command { return cliRunner.GetGroupMembers(ctx, args[0], getConfig.full) }, } - return cmd } func partitionsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "partitions [topic]", Short: "Displays partition information for the specified topic.", Args: cobra.ExactArgs(1), @@ -232,11 +226,10 @@ func partitionsCmd() *cobra.Command { return cliRunner.GetPartitions(ctx, args[0]) }, } - return cmd } func offsetsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "offsets [topic]", Short: "Displays offset information for the specified topic along with start and end times.", Args: cobra.ExactArgs(1), @@ -254,11 +247,10 @@ func offsetsCmd() *cobra.Command { return cliRunner.GetOffsets(ctx, args[0]) }, } - return cmd } func topicsCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "topics", Short: "Displays information for all topics in the cluster.", Args: cobra.NoArgs, @@ -276,5 +268,4 @@ func topicsCmd() *cobra.Command { return cliRunner.GetTopics(ctx, getConfig.full) }, } - return cmd } From ea28ea99ac2dd1e55a69df0676000b1b907cffe7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 16:42:11 -0400 Subject: [PATCH 12/57] wip --- cmd/topicctl/subcmd/get.go | 23 +++++++++- pkg/admin/brokerclient.go | 2 +- pkg/admin/format.go | 2 +- pkg/admin/types.go | 90 +++++++++++++++++++++++++++++++++++++- 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index a33069ec..fb72c525 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -55,6 +55,7 @@ func init() { partitionsCmd(), offsetsCmd(), topicsCmd(), + aclsCmd(), ) RootCmd.AddCommand(getCmd) } @@ -267,6 +268,26 @@ func topicsCmd() *cobra.Command { cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetTopics(ctx, getConfig.full) }, ->>>>>>> chore/separate-subcmd-for-get + } +} + +func aclsCmd() *cobra.Command { + return &cobra.Command{ + Use: "acls", + Short: "Displays information for acls in the cluster. Supports filtering with flags.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return cliRunner.GetACLs(ctx) + }, } } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index fc34e000..9131e031 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -726,7 +726,7 @@ func (c *BrokerAdminClient) GetACLs( for _, resource := range resp.Resources { for _, acl := range resource.ACLs { aclinfos = append(aclinfos, ACLInfo{ - ResourceType: resource.ResourceType, + ResourceType: kafkaGoResourceTypeToTopicctl(resource.ResourceType), ResourceName: resource.ResourceName, PatternType: resource.PatternType, Principal: acl.Principal, diff --git a/pkg/admin/format.go b/pkg/admin/format.go index ec7e4645..87b1e963 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -784,7 +784,7 @@ func FormatAcls(acls []ACLInfo) string { for _, acl := range acls { row := []string{ // TODO: convert ints to something human readable - fmt.Sprintf("%d", acl.ResourceType), + string(acl.ResourceType), acl.ResourceName, acl.Principal, acl.Host, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 125f2160..13be6a9b 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -77,7 +77,7 @@ type PartitionAssignment struct { // PartitionInfo represents the information stored about an ACL // in zookeeper. type ACLInfo struct { - ResourceType kafka.ResourceType + ResourceType ACLResourceType ResourceName string PatternType kafka.PatternType Principal string @@ -86,6 +86,94 @@ type ACLInfo struct { PermissionType kafka.ACLPermissionType } +type ACLResourceType string + +func kafkaGoResourceTypeToTopicctl(r kafka.ResourceType) ACLResourceType { + switch r { + case kafka.ResourceTypeUnknown: + return "Unknown" + case kafka.ResourceTypeAny: + return "Any" + case kafka.ResourceTypeTopic: + return "Topic" + case kafka.ResourceTypeGroup: + return "Group" + case kafka.ResourceTypeCluster: + return "Cluster" + case kafka.ResourceTypeTransactionalID: + return "TransactionalID" + case kafka.ResourceTypeDelegationToken: + return "DelegationToken" + default: + return "Invalid ResourceType" + } +} + +// func (p kafka.PatternType) String() string { +// switch p { +// case kafka.PatternTypeUnknown: +// return "Unknown" +// case kafka.PatternTypeAny: +// return "Any" +// case kafka.PatternTypeMatch: +// return "Match" +// case kafka.PatternTypeLiteral: +// return "Literal" +// case kafka.PatternTypePrefixed: +// return "Prefixed" +// default: +// return "Invalid PatternType" +// } +// } + +// func (o kafka.ACLOperationType) String() string { +// switch o { +// case kafka.ACLOperationTypeUnknown: +// return "Unknown" +// case kafka.ACLOperationTypeAny: +// return "Any" +// case kafka.ACLOperationTypeAll: +// return "All" +// case kafka.ACLOperationTypeRead: +// return "Read" +// case kafka.ACLOperationTypeWrite: +// return "Write" +// case kafka.ACLOperationTypeCreate: +// return "Create" +// case kafka.ACLOperationTypeDelete: +// return "Delete" +// case kafka.ACLOperationTypeAlter: +// return "Alter" +// case kafka.ACLOperationTypeDescribe: +// return "Describe" +// case kafka.ACLOperationTypeClusterAction: +// return "ClusterAction" +// case kafka.ACLOperationTypeDescribeConfigs: +// return "DescribeConfigs" +// case kafka.ACLOperationTypeAlterConfigs: +// return "AlterConfigs" +// case kafka.ACLOperationTypeIdempotentWrite: +// return "IdempotentWrite" +// default: +// return "Invalid OperationType" +// } +// } + +// func (p kafka.ACLPermissionType) String() string { +// switch p { +// case kafka.ACLPermissionTypeUnknown: +// return "Unknown" +// case kafka.ACLPermissionTypeAny: +// return "Any" +// case kafka.ACLPermissionTypeDeny: +// return "Deny" +// case kafka.ACLPermissionTypeAllow: +// return "Allow" +// default: +// return "Invalid PermissionType" +// } +// } + type zkClusterID struct { Version string `json:"version"` ID string `json:"id"` From 07667ddf64bfbe9383cb2ae9f2f6b548f169f157 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 12 Sep 2023 16:49:39 -0400 Subject: [PATCH 13/57] pr feedback --- cmd/topicctl/subcmd/get.go | 89 ++++++++++++-------------------------- 1 file changed, 27 insertions(+), 62 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 33f7abfd..dd1a4c28 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -63,6 +63,24 @@ func getPreRun(cmd *cobra.Command, args []string) error { return getConfig.shared.validate() } +func getCliRunnerAndCtx() ( + context.Context, + *cli.CLIRunner, + error, +) { + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return nil, nil, err + } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) + return ctx, cliRunner, nil +} + func balanceCmd() *cobra.Command { return &cobra.Command{ Use: "balance [optional topic]", @@ -75,16 +93,10 @@ func balanceCmd() *cobra.Command { ), Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) var topicName string if len(args) == 1 { @@ -102,16 +114,10 @@ func brokersCmd() *cobra.Command { Short: "Displays descriptions of each broker in the cluster.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetBrokers(ctx, getConfig.full) }, } @@ -123,16 +129,11 @@ func configCmd() *cobra.Command { Short: "Displays configuration for the provider broker or topic.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetConfig(ctx, args[0]) }, } @@ -144,16 +145,10 @@ func groupsCmd() *cobra.Command { Short: "Displays consumer group informatin for the cluster.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetGroups(ctx) }, } @@ -165,16 +160,10 @@ func lagsCmd() *cobra.Command { Short: "Displays consumer group lag for the specified topic and consumer group.", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetMemberLags( ctx, args[0], @@ -192,16 +181,10 @@ func membersCmd() *cobra.Command { Short: "Details of each member in the specified consumer group.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetGroupMembers(ctx, args[0], getConfig.full) }, } @@ -213,16 +196,10 @@ func partitionsCmd() *cobra.Command { Short: "Displays partition information for the specified topic.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetPartitions(ctx, args[0]) }, } @@ -234,16 +211,10 @@ func offsetsCmd() *cobra.Command { Short: "Displays offset information for the specified topic along with start and end times.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetOffsets(ctx, args[0]) }, } @@ -255,16 +226,10 @@ func topicsCmd() *cobra.Command { Short: "Displays information for all topics in the cluster.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetTopics(ctx, getConfig.full) }, } From dcdd0e8d3abd4988ac8b7a8db39467f6c22e5215 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 08:44:15 -0400 Subject: [PATCH 14/57] Revert "upgrade cobra to latest" This reverts commit 7b8ee4282d309441a9586db13dde05c5ec30eff5. --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0f232842..4293590e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/segmentio/kafka-go v0.4.35 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d @@ -23,7 +23,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/mattn/go-colorable v0.1.9 // indirect diff --git a/go.sum b/go.sum index b8fb8ecf..4e4f327e 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -80,8 +80,8 @@ github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d07 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 9f8f5507a41b21547814366c33f36949e105d6ec Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 09:08:59 -0400 Subject: [PATCH 15/57] use getCliRunnerAndCtx in get acls --- cmd/topicctl/subcmd/get.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index a1c32579..f8b57b79 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -242,16 +242,10 @@ func aclsCmd() *cobra.Command { Short: "Displays information for acls in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - ctx := context.Background() - sess := session.Must(session.NewSession()) - - adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - defer adminClient.Close() - - cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) return cliRunner.GetACLs(ctx) }, } From 4a78af2c5b6b4632eb9154b0abff621bf2cbf5d0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 09:10:30 -0400 Subject: [PATCH 16/57] more consistent variable names --- pkg/admin/format.go | 4 ++-- pkg/cli/cli.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 87b1e963..5a3916ab 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -745,9 +745,9 @@ func FormatBrokerMaxPartitions( return string(bytes.TrimRight(buf.Bytes(), "\n")) } -// FormatAcls creates a pretty table that lists the details of the +// FormatACLs creates a pretty table that lists the details of the // argument acls. -func FormatAcls(acls []ACLInfo) string { +func FormatACLs(acls []ACLInfo) string { buf := &bytes.Buffer{} headers := []string{ diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 68223baa..c6de45ce 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -554,7 +554,7 @@ func (c *CLIRunner) Tail( } // TODO add options for filtering -// GetAcls fetches the details of each acl in the cluster and prints out a summary. +// GetACLs fetches the details of each acl in the cluster and prints out a summary. func (c *CLIRunner) GetACLs(ctx context.Context) error { c.startSpinner() @@ -569,7 +569,7 @@ func (c *CLIRunner) GetACLs(ctx context.Context) error { return err } - c.printer("ACLs:\n%s", admin.FormatAcls(acls)) + c.printer("ACLs:\n%s", admin.FormatACLs(acls)) return nil } From 1dbf20027e98233171465055bc4b9ea022dd2ca7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 11:23:37 -0400 Subject: [PATCH 17/57] custom cobra type --- cmd/topicctl/subcmd/get.go | 29 +++++++++++++++++--- pkg/admin/brokerclient.go | 2 +- pkg/admin/format.go | 2 +- pkg/admin/types.go | 55 +++++++++++++++++++++++++++++--------- pkg/cli/cli.go | 12 ++++----- 5 files changed, 76 insertions(+), 24 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index f8b57b79..6276fe50 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -5,6 +5,8 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws/session" + "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/cli" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -236,17 +238,38 @@ func topicsCmd() *cobra.Command { } } +type aclsCmdConfig struct { + resourceType admin.ResourceType +} + +var aclsConfig = aclsCmdConfig{ + resourceType: admin.ResourceType(kafka.ResourceTypeAny), +} + func aclsCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "acls", - Short: "Displays information for acls in the cluster. Supports filtering with flags.", + Short: "Displays information for ACLs in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { return err } - return cliRunner.GetACLs(ctx) + + filter := kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), + ResourcePatternTypeFilter: kafka.PatternTypeAny, + Operation: kafka.ACLOperationTypeAny, + PermissionType: kafka.ACLPermissionTypeAny, + } + return cliRunner.GetACLs(ctx, filter) }, } + cmd.Flags().Var( + &aclsConfig.resourceType, + "resource-type", + `Resource type. allowed: "unknown", "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + ) + return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 9131e031..dbe723df 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -726,7 +726,7 @@ func (c *BrokerAdminClient) GetACLs( for _, resource := range resp.Resources { for _, acl := range resource.ACLs { aclinfos = append(aclinfos, ACLInfo{ - ResourceType: kafkaGoResourceTypeToTopicctl(resource.ResourceType), + ResourceType: ResourceType(resource.ResourceType), ResourceName: resource.ResourceName, PatternType: resource.PatternType, Principal: acl.Principal, diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 5a3916ab..9c9d49bc 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -784,7 +784,7 @@ func FormatACLs(acls []ACLInfo) string { for _, acl := range acls { row := []string{ // TODO: convert ints to something human readable - string(acl.ResourceType), + acl.ResourceType.String(), acl.ResourceName, acl.Principal, acl.Host, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 13be6a9b..0599eb55 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -6,6 +6,7 @@ import ( "reflect" "sort" "strconv" + "strings" "time" "github.com/segmentio/kafka-go" @@ -77,7 +78,7 @@ type PartitionAssignment struct { // PartitionInfo represents the information stored about an ACL // in zookeeper. type ACLInfo struct { - ResourceType ACLResourceType + ResourceType ResourceType ResourceName string PatternType kafka.PatternType Principal string @@ -86,29 +87,59 @@ type ACLInfo struct { PermissionType kafka.ACLPermissionType } -type ACLResourceType string +// ResourceType presents the Kafka resource type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type ResourceType kafka.ResourceType -func kafkaGoResourceTypeToTopicctl(r kafka.ResourceType) ACLResourceType { - switch r { +var resourceTypeMap = map[string]kafka.ResourceType{ + "unknown": kafka.ResourceTypeUnknown, + "any": kafka.ResourceTypeAny, + "topic": kafka.ResourceTypeTopic, + "group": kafka.ResourceTypeGroup, + "cluster": kafka.ResourceTypeCluster, + "transactionalid": kafka.ResourceTypeTransactionalID, + "delegationtoken": kafka.ResourceTypeDelegationToken, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (r *ResourceType) String() string { + switch kafka.ResourceType(*r) { case kafka.ResourceTypeUnknown: - return "Unknown" + return "unknown" case kafka.ResourceTypeAny: - return "Any" + return "any" case kafka.ResourceTypeTopic: - return "Topic" + return "topic" case kafka.ResourceTypeGroup: - return "Group" + return "group" case kafka.ResourceTypeCluster: - return "Cluster" + return "cluster" case kafka.ResourceTypeTransactionalID: - return "TransactionalID" + return "transactionalid" case kafka.ResourceTypeDelegationToken: - return "DelegationToken" + return "delegationtoken" default: - return "Invalid ResourceType" + return "invalid ResourceType" } } +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (r *ResourceType) Set(v string) error { + rt, ok := resourceTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "unknown", "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) + } + *r = ResourceType(rt) + return nil +} + +// Type is used by Cobra in help text. +func (r *ResourceType) Type() string { + return "ResourceType" +} + // func (p kafka.PatternType) String() string { // switch p { // case kafka.PatternTypeUnknown: diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index c6de45ce..40e4f901 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -555,15 +555,13 @@ func (c *CLIRunner) Tail( // TODO add options for filtering // GetACLs fetches the details of each acl in the cluster and prints out a summary. -func (c *CLIRunner) GetACLs(ctx context.Context) error { +func (c *CLIRunner) GetACLs( + ctx context.Context, + filter kafka.ACLFilter, +) error { c.startSpinner() - acls, err := c.adminClient.GetACLs(ctx, kafka.ACLFilter{ - ResourceTypeFilter: kafka.ResourceTypeAny, - ResourcePatternTypeFilter: kafka.PatternTypeAny, - Operation: kafka.ACLOperationTypeAny, - PermissionType: kafka.ACLPermissionTypeAny, - }) + acls, err := c.adminClient.GetACLs(ctx, filter) c.stopSpinner() if err != nil { return err From 226ae1c27dec4ade687d07159acd05dc830d4cc4 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 13:01:03 -0400 Subject: [PATCH 18/57] bring in new kafka-go --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7f969696..66a35960 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 + github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 4d6c3b80..6e87f76d 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,12 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= +github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= +github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= +github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 h1:Gok6q1P5yUVLYt+auyZgcRBP89thqCmU9MNT65Ms1SI= +github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= From acc011f144e50e58f4c0e952838e4569f32b54b0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 13 Sep 2023 15:36:44 -0400 Subject: [PATCH 19/57] support resource pattern type --- cmd/topicctl/subcmd/get.go | 13 +++++-- pkg/admin/brokerclient.go | 2 +- pkg/admin/brokerclient_test.go | 31 +++++++++++----- pkg/admin/format.go | 3 ++ pkg/admin/types.go | 64 +++++++++++++++++++++++++--------- 5 files changed, 83 insertions(+), 30 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 6276fe50..acda69d1 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -239,11 +239,13 @@ func topicsCmd() *cobra.Command { } type aclsCmdConfig struct { - resourceType admin.ResourceType + resourceType admin.ResourceType + resourcePatternType admin.PatternType } var aclsConfig = aclsCmdConfig{ - resourceType: admin.ResourceType(kafka.ResourceTypeAny), + resourceType: admin.ResourceType(kafka.ResourceTypeAny), + resourcePatternType: admin.PatternType(kafka.PatternTypeAny), } func aclsCmd() *cobra.Command { @@ -259,7 +261,7 @@ func aclsCmd() *cobra.Command { filter := kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), - ResourcePatternTypeFilter: kafka.PatternTypeAny, + ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), Operation: kafka.ACLOperationTypeAny, PermissionType: kafka.ACLPermissionTypeAny, } @@ -271,5 +273,10 @@ func aclsCmd() *cobra.Command { "resource-type", `Resource type. allowed: "unknown", "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, ) + cmd.Flags().Var( + &aclsConfig.resourcePatternType, + "resource-pattern-type", + `Resource pattern type. allowed: "unknown", "any", "match", "literal", "prefixed"`, + ) return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index dbe723df..1bb7e22f 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -728,7 +728,7 @@ func (c *BrokerAdminClient) GetACLs( aclinfos = append(aclinfos, ACLInfo{ ResourceType: ResourceType(resource.ResourceType), ResourceName: resource.ResourceName, - PatternType: resource.PatternType, + PatternType: PatternType(resource.PatternType), Principal: acl.Principal, Host: acl.Host, Operation: acl.Operation, diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 2d8c21b6..f849fc6b 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -586,13 +586,6 @@ func TestBrokerClientCreateGetACL(t *testing.T) { BrokerAdminClientConfig{ ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), - SASL: SASLConfig{ - Enabled: true, - Mechanism: SASLMechanismScramSHA512, - // TODO: don't hardcode these in tests, pull from env vars - Username: "adminscram", - Password: "admin-secret-512", - }, }, }, ) @@ -601,6 +594,27 @@ func TestBrokerClientCreateGetACL(t *testing.T) { principal := util.RandomString("User:user-create-", 6) topicName := util.RandomString("topic-create-", 6) + defer func() { + _, err := client.client.DeleteACLs( + ctx, + &kafka.DeleteACLsRequest{ + Filters: []kafka.DeleteACLsFilter{ + { + ResourceTypeFilter: kafka.ResourceTypeTopic, + ResourceNameFilter: topicName, + ResourcePatternTypeFilter: kafka.PatternTypeLiteral, + Operation: kafka.ACLOperationTypeRead, + PermissionType: kafka.ACLPermissionTypeAllow, + }, + }, + }, + ) + + if err != nil { + t.Fatal(fmt.Errorf("failed to clean up ACL, err: %v", err)) + } + }() + err = client.CreateACL( ctx, kafka.ACLEntry{ @@ -627,7 +641,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { require.NoError(t, err) expected := []ACLInfo{ { - ResourceType: kafka.ResourceTypeTopic, + ResourceType: ResourceType(kafka.ResourceTypeTopic), ResourceName: topicName, PatternType: kafka.PatternTypeLiteral, Principal: principal, @@ -637,5 +651,4 @@ func TestBrokerClientCreateGetACL(t *testing.T) { }, } assert.Equal(t, expected, aclsInfo) - } diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 9c9d49bc..2347d1c2 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -752,6 +752,7 @@ func FormatACLs(acls []ACLInfo) string { headers := []string{ "Resource Type", + "Pattern Type", "Resource Name", "Principal", "Host", @@ -770,6 +771,7 @@ func FormatACLs(acls []ACLInfo) string { tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, + tablewriter.ALIGN_LEFT, }, ) table.SetBorders( @@ -785,6 +787,7 @@ func FormatACLs(acls []ACLInfo) string { row := []string{ // TODO: convert ints to something human readable acl.ResourceType.String(), + acl.PatternType.String(), acl.ResourceName, acl.Principal, acl.Host, diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 0599eb55..3fd427b8 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -80,7 +80,7 @@ type PartitionAssignment struct { type ACLInfo struct { ResourceType ResourceType ResourceName string - PatternType kafka.PatternType + PatternType PatternType Principal string Host string Operation kafka.ACLOperationType @@ -140,22 +140,52 @@ func (r *ResourceType) Type() string { return "ResourceType" } -// func (p kafka.PatternType) String() string { -// switch p { -// case kafka.PatternTypeUnknown: -// return "Unknown" -// case kafka.PatternTypeAny: -// return "Any" -// case kafka.PatternTypeMatch: -// return "Match" -// case kafka.PatternTypeLiteral: -// return "Literal" -// case kafka.PatternTypePrefixed: -// return "Prefixed" -// default: -// return "Invalid PatternType" -// } -// } +// PatternType presents the Kafka resource type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type PatternType kafka.PatternType + +var patternTypeMap = map[string]kafka.PatternType{ + "unknown": kafka.PatternTypeUnknown, + "any": kafka.PatternTypeAny, + "match": kafka.PatternTypeMatch, + "literal": kafka.PatternTypeLiteral, + "prefixed": kafka.PatternTypePrefixed, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (p *PatternType) String() string { + switch kafka.PatternType(*p) { + case kafka.PatternTypeUnknown: + return "unknown" + case kafka.PatternTypeAny: + return "any" + case kafka.PatternTypeMatch: + return "match" + case kafka.PatternTypeLiteral: + return "literal" + case kafka.PatternTypePrefixed: + return "prefixed" + default: + return "invalid PatternType" + } +} + +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (r *PatternType) Set(v string) error { + rt, ok := patternTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "unknown", "any", "match", "literal", or "prefixed"`) + } + *r = PatternType(rt) + return nil +} + +// Type is used by Cobra in help text. +func (r *PatternType) Type() string { + return "PatternType" +} // func (o kafka.ACLOperationType) String() string { // switch o { From 2536e506fa11d3c79fade5dc3ba7b8b976ab78e1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 10:25:07 -0400 Subject: [PATCH 20/57] add support for acloperationtype and remove options for unknown --- cmd/topicctl/subcmd/get.go | 17 +++++-- pkg/admin/brokerclient.go | 2 +- pkg/admin/brokerclient_test.go | 4 +- pkg/admin/format.go | 2 +- pkg/admin/types.go | 84 +++++++++++++++++++++++++++++----- 5 files changed, 91 insertions(+), 18 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index acda69d1..9df654cd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -241,11 +241,13 @@ func topicsCmd() *cobra.Command { type aclsCmdConfig struct { resourceType admin.ResourceType resourcePatternType admin.PatternType + aclOperationType admin.ACLOperationType } var aclsConfig = aclsCmdConfig{ resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), + aclOperationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), } func aclsCmd() *cobra.Command { @@ -253,6 +255,8 @@ func aclsCmd() *cobra.Command { Use: "acls", Short: "Displays information for ACLs in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, + // TODO: make common examples here + // Example: RunE: func(cmd *cobra.Command, args []string) error { ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { @@ -260,9 +264,10 @@ func aclsCmd() *cobra.Command { } filter := kafka.ACLFilter{ + //ResourceNameFilter: "", ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), - Operation: kafka.ACLOperationTypeAny, + Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), PermissionType: kafka.ACLPermissionTypeAny, } return cliRunner.GetACLs(ctx, filter) @@ -271,12 +276,18 @@ func aclsCmd() *cobra.Command { cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", - `Resource type. allowed: "unknown", "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + `Resource type. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", - `Resource pattern type. allowed: "unknown", "any", "match", "literal", "prefixed"`, + // TODO: document the behavior of each of these + `Resource pattern type. allowed: "any", "match", "literal", "prefixed"`, + ) + cmd.Flags().Var( + &aclsConfig.aclOperationType, + "operations", + `ACL operation type. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, ) return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 1bb7e22f..c6910c41 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -731,7 +731,7 @@ func (c *BrokerAdminClient) GetACLs( PatternType: PatternType(resource.PatternType), Principal: acl.Principal, Host: acl.Host, - Operation: acl.Operation, + Operation: ACLOperationType(acl.Operation), PermissionType: acl.PermissionType, }) } diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index f849fc6b..9a9c77c3 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -643,10 +643,10 @@ func TestBrokerClientCreateGetACL(t *testing.T) { { ResourceType: ResourceType(kafka.ResourceTypeTopic), ResourceName: topicName, - PatternType: kafka.PatternTypeLiteral, + PatternType: PatternType(kafka.PatternTypeLiteral), Principal: principal, Host: "*", - Operation: kafka.ACLOperationTypeRead, + Operation: ACLOperationType(kafka.ACLOperationTypeRead), PermissionType: kafka.ACLPermissionTypeAllow, }, } diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 2347d1c2..6cfa33aa 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -791,7 +791,7 @@ func FormatACLs(acls []ACLInfo) string { acl.ResourceName, acl.Principal, acl.Host, - fmt.Sprintf("%d", acl.Operation), + acl.Operation.String(), fmt.Sprintf("%d", acl.PermissionType), } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 3fd427b8..a336bcfe 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -83,7 +83,7 @@ type ACLInfo struct { PatternType PatternType Principal string Host string - Operation kafka.ACLOperationType + Operation ACLOperationType PermissionType kafka.ACLPermissionType } @@ -94,7 +94,6 @@ type ACLInfo struct { type ResourceType kafka.ResourceType var resourceTypeMap = map[string]kafka.ResourceType{ - "unknown": kafka.ResourceTypeUnknown, "any": kafka.ResourceTypeAny, "topic": kafka.ResourceTypeTopic, "group": kafka.ResourceTypeGroup, @@ -106,8 +105,6 @@ var resourceTypeMap = map[string]kafka.ResourceType{ // String is used both by fmt.Print and by Cobra in help text. func (r *ResourceType) String() string { switch kafka.ResourceType(*r) { - case kafka.ResourceTypeUnknown: - return "unknown" case kafka.ResourceTypeAny: return "any" case kafka.ResourceTypeTopic: @@ -121,7 +118,7 @@ func (r *ResourceType) String() string { case kafka.ResourceTypeDelegationToken: return "delegationtoken" default: - return "invalid ResourceType" + return "unknown" } } @@ -129,7 +126,7 @@ func (r *ResourceType) String() string { func (r *ResourceType) Set(v string) error { rt, ok := resourceTypeMap[strings.ToLower(v)] if !ok { - return errors.New(`must be one of "unknown", "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) + return errors.New(`must be one of "any", "topic", "group", "cluster", "transactionalid", or "delegationtoken"`) } *r = ResourceType(rt) return nil @@ -147,7 +144,6 @@ func (r *ResourceType) Type() string { type PatternType kafka.PatternType var patternTypeMap = map[string]kafka.PatternType{ - "unknown": kafka.PatternTypeUnknown, "any": kafka.PatternTypeAny, "match": kafka.PatternTypeMatch, "literal": kafka.PatternTypeLiteral, @@ -157,8 +153,6 @@ var patternTypeMap = map[string]kafka.PatternType{ // String is used both by fmt.Print and by Cobra in help text. func (p *PatternType) String() string { switch kafka.PatternType(*p) { - case kafka.PatternTypeUnknown: - return "unknown" case kafka.PatternTypeAny: return "any" case kafka.PatternTypeMatch: @@ -168,7 +162,7 @@ func (p *PatternType) String() string { case kafka.PatternTypePrefixed: return "prefixed" default: - return "invalid PatternType" + return "unknown" } } @@ -176,7 +170,7 @@ func (p *PatternType) String() string { func (r *PatternType) Set(v string) error { rt, ok := patternTypeMap[strings.ToLower(v)] if !ok { - return errors.New(`must be one of "unknown", "any", "match", "literal", or "prefixed"`) + return errors.New(`must be one of "any", "match", "literal", or "prefixed"`) } *r = PatternType(rt) return nil @@ -187,6 +181,74 @@ func (r *PatternType) Type() string { return "PatternType" } +// ACLOperationType presents the Kafka resource type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type ACLOperationType kafka.ACLOperationType + +var aclOperationTypeMap = map[string]kafka.ACLOperationType{ + "any": kafka.ACLOperationTypeAny, + "all": kafka.ACLOperationTypeAll, + "read": kafka.ACLOperationTypeRead, + "write": kafka.ACLOperationTypeWrite, + "create": kafka.ACLOperationTypeCreate, + "delete": kafka.ACLOperationTypeDelete, + "alter": kafka.ACLOperationTypeAlter, + "describe": kafka.ACLOperationTypeDescribe, + "clusteraction": kafka.ACLOperationTypeClusterAction, + "describeconfigs": kafka.ACLOperationTypeDescribeConfigs, + "alterconfigs": kafka.ACLOperationTypeAlterConfigs, + "idempotentwrite": kafka.ACLOperationTypeIdempotentWrite, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (o *ACLOperationType) String() string { + switch kafka.ACLOperationType(*o) { + case kafka.ACLOperationTypeAny: + return "any" + case kafka.ACLOperationTypeAll: + return "all" + case kafka.ACLOperationTypeRead: + return "read" + case kafka.ACLOperationTypeWrite: + return "write" + case kafka.ACLOperationTypeCreate: + return "create" + case kafka.ACLOperationTypeDelete: + return "delete" + case kafka.ACLOperationTypeAlter: + return "alter" + case kafka.ACLOperationTypeDescribe: + return "describe" + case kafka.ACLOperationTypeClusterAction: + return "clusteraction" + case kafka.ACLOperationTypeDescribeConfigs: + return "describeconfigs" + case kafka.ACLOperationTypeAlterConfigs: + return "alterconfigs" + case kafka.ACLOperationTypeIdempotentWrite: + return "idempotentwrite" + default: + return "unknown" + } +} + +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (r *ACLOperationType) Set(v string) error { + rt, ok := aclOperationTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`) + } + *r = ACLOperationType(rt) + return nil +} + +// Type is used by Cobra in help text. +func (r *ACLOperationType) Type() string { + return "ACLOperationType" +} + // func (o kafka.ACLOperationType) String() string { // switch o { // case kafka.ACLOperationTypeUnknown: From 62671b0b960411e1ae81532d06cf9ffe8ddc55e3 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 10:55:31 -0400 Subject: [PATCH 21/57] improve descriptions --- cmd/topicctl/subcmd/get.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 9df654cd..e94ca3db 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -264,11 +264,13 @@ func aclsCmd() *cobra.Command { } filter := kafka.ACLFilter{ + ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), //ResourceNameFilter: "", - ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), - Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), - PermissionType: kafka.ACLPermissionTypeAny, + //PrincipalFilter: "*", + //HostFilter:"*". + Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), + PermissionType: kafka.ACLPermissionTypeAny, } return cliRunner.GetACLs(ctx, filter) }, @@ -276,18 +278,19 @@ func aclsCmd() *cobra.Command { cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", - `Resource type. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", // TODO: document the behavior of each of these - `Resource pattern type. allowed: "any", "match", "literal", "prefixed"`, + // TODO: match isn't really supported right now, look into that + `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, ) cmd.Flags().Var( &aclsConfig.aclOperationType, "operations", - `ACL operation type. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, + `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, ) return cmd } From 3f050ca4e9eb26c3d1faf551628412fff82a148b Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 11:17:26 -0400 Subject: [PATCH 22/57] support permissiontype and host filters --- cmd/topicctl/subcmd/get.go | 37 +++++++++---- pkg/admin/brokerclient.go | 2 +- pkg/admin/types.go | 107 +++++++++++++++++-------------------- 3 files changed, 78 insertions(+), 68 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index e94ca3db..2d5ed595 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -239,15 +239,21 @@ func topicsCmd() *cobra.Command { } type aclsCmdConfig struct { + hostFilter string + operationType admin.ACLOperationType + permissionType admin.ACLPermissionType resourceType admin.ResourceType resourcePatternType admin.PatternType - aclOperationType admin.ACLOperationType } +// aclsConfig defines the default values if a flag is not provided. These all default +// to doing no filtering (e.g. "all") var aclsConfig = aclsCmdConfig{ + hostFilter: "", + operationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), + permissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAny), resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), - aclOperationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), } func aclsCmd() *cobra.Command { @@ -268,13 +274,29 @@ func aclsCmd() *cobra.Command { //ResourceNameFilter: "", ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), //PrincipalFilter: "*", - //HostFilter:"*". - Operation: kafka.ACLOperationType(aclsConfig.aclOperationType), - PermissionType: kafka.ACLPermissionTypeAny, + HostFilter: aclsConfig.hostFilter, + Operation: kafka.ACLOperationType(aclsConfig.operationType), + PermissionType: kafka.ACLPermissionType(aclsConfig.permissionType), } return cliRunner.GetACLs(ctx, filter) }, } + cmd.Flags().StringVar( + &aclsConfig.hostFilter, + "host", + "", + `The host to filter on.`, + ) + cmd.Flags().Var( + &aclsConfig.operationType, + "operations", + `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, + ) + cmd.Flags().Var( + &aclsConfig.permissionType, + "permission-type", + `The permission type to filter on. allowed: "any", "allow", or "deny"`, + ) cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", @@ -287,10 +309,5 @@ func aclsCmd() *cobra.Command { // TODO: match isn't really supported right now, look into that `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, ) - cmd.Flags().Var( - &aclsConfig.aclOperationType, - "operations", - `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, - ) return cmd } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index c6910c41..329fa1b3 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -732,7 +732,7 @@ func (c *BrokerAdminClient) GetACLs( Principal: acl.Principal, Host: acl.Host, Operation: ACLOperationType(acl.Operation), - PermissionType: acl.PermissionType, + PermissionType: ACLPermissionType(acl.PermissionType), }) } } diff --git a/pkg/admin/types.go b/pkg/admin/types.go index a336bcfe..14b7c744 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -84,7 +84,7 @@ type ACLInfo struct { Principal string Host string Operation ACLOperationType - PermissionType kafka.ACLPermissionType + PermissionType ACLPermissionType } // ResourceType presents the Kafka resource type. @@ -137,7 +137,7 @@ func (r *ResourceType) Type() string { return "ResourceType" } -// PatternType presents the Kafka resource type. +// PatternType presents the Kafka pattern type. // We need to subtype this to be able to define methods to // satisfy the Value interface from Cobra so we can use it // as a Cobra flag. @@ -167,12 +167,12 @@ func (p *PatternType) String() string { } // Set is used by Cobra to set the value of a variable from a Cobra flag. -func (r *PatternType) Set(v string) error { - rt, ok := patternTypeMap[strings.ToLower(v)] +func (p *PatternType) Set(v string) error { + pt, ok := patternTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "match", "literal", or "prefixed"`) } - *r = PatternType(rt) + *p = PatternType(pt) return nil } @@ -181,7 +181,7 @@ func (r *PatternType) Type() string { return "PatternType" } -// ACLOperationType presents the Kafka resource type. +// ACLOperationType presents the Kafka operation type. // We need to subtype this to be able to define methods to // satisfy the Value interface from Cobra so we can use it // as a Cobra flag. @@ -235,67 +235,60 @@ func (o *ACLOperationType) String() string { } // Set is used by Cobra to set the value of a variable from a Cobra flag. -func (r *ACLOperationType) Set(v string) error { - rt, ok := aclOperationTypeMap[strings.ToLower(v)] +func (o *ACLOperationType) Set(v string) error { + ot, ok := aclOperationTypeMap[strings.ToLower(v)] if !ok { return errors.New(`must be one of "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`) } - *r = ACLOperationType(rt) + *o = ACLOperationType(ot) return nil } // Type is used by Cobra in help text. -func (r *ACLOperationType) Type() string { +func (o *ACLOperationType) Type() string { return "ACLOperationType" } -// func (o kafka.ACLOperationType) String() string { -// switch o { -// case kafka.ACLOperationTypeUnknown: -// return "Unknown" -// case kafka.ACLOperationTypeAny: -// return "Any" -// case kafka.ACLOperationTypeAll: -// return "All" -// case kafka.ACLOperationTypeRead: -// return "Read" -// case kafka.ACLOperationTypeWrite: -// return "Write" -// case kafka.ACLOperationTypeCreate: -// return "Create" -// case kafka.ACLOperationTypeDelete: -// return "Delete" -// case kafka.ACLOperationTypeAlter: -// return "Alter" -// case kafka.ACLOperationTypeDescribe: -// return "Describe" -// case kafka.ACLOperationTypeClusterAction: -// return "ClusterAction" -// case kafka.ACLOperationTypeDescribeConfigs: -// return "DescribeConfigs" -// case kafka.ACLOperationTypeAlterConfigs: -// return "AlterConfigs" -// case kafka.ACLOperationTypeIdempotentWrite: -// return "IdempotentWrite" -// default: -// return "Invalid OperationType" -// } -// } - -// func (p kafka.ACLPermissionType) String() string { -// switch p { -// case kafka.ACLPermissionTypeUnknown: -// return "Unknown" -// case kafka.ACLPermissionTypeAny: -// return "Any" -// case kafka.ACLPermissionTypeDeny: -// return "Deny" -// case kafka.ACLPermissionTypeAllow: -// return "Allow" -// default: -// return "Invalid PermissionType" -// } -// } +// ACLPermissionType presents the Kafka operation type. +// We need to subtype this to be able to define methods to +// satisfy the Value interface from Cobra so we can use it +// as a Cobra flag. +type ACLPermissionType kafka.ACLPermissionType + +var aclPermissionTypeMap = map[string]kafka.ACLPermissionType{ + "any": kafka.ACLPermissionTypeAny, + "allow": kafka.ACLPermissionTypeAllow, + "deny": kafka.ACLPermissionTypeDeny, +} + +// String is used both by fmt.Print and by Cobra in help text. +func (p *ACLPermissionType) String() string { + switch kafka.ACLPermissionType(*p) { + case kafka.ACLPermissionTypeAny: + return "any" + case kafka.ACLPermissionTypeDeny: + return "deny" + case kafka.ACLPermissionTypeAllow: + return "allow" + default: + return "unknown" + } +} + +// Set is used by Cobra to set the value of a variable from a Cobra flag. +func (p *ACLPermissionType) Set(v string) error { + pt, ok := aclPermissionTypeMap[strings.ToLower(v)] + if !ok { + return errors.New(`must be one of "any", "allow", or "deny"`) + } + *p = ACLPermissionType(pt) + return nil +} + +// Type is used by Cobra in help text. +func (p *ACLPermissionType) Type() string { + return "ACLPermissionType" +} type zkClusterID struct { Version string `json:"version"` From 925670ce87e257fd22671f5e3da64ba99221ba53 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 13:37:31 -0400 Subject: [PATCH 23/57] add resource name filter and fix permission type formatting --- cmd/topicctl/subcmd/get.go | 22 +++++++++++++++------- pkg/admin/format.go | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 2d5ed595..b8163076 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -242,8 +242,9 @@ type aclsCmdConfig struct { hostFilter string operationType admin.ACLOperationType permissionType admin.ACLPermissionType - resourceType admin.ResourceType + resourceNameFilter string resourcePatternType admin.PatternType + resourceType admin.ResourceType } // aclsConfig defines the default values if a flag is not provided. These all default @@ -252,6 +253,7 @@ var aclsConfig = aclsCmdConfig{ hostFilter: "", operationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), permissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAny), + resourceNameFilter: "", resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), } @@ -270,8 +272,8 @@ func aclsCmd() *cobra.Command { } filter := kafka.ACLFilter{ - ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), - //ResourceNameFilter: "", + ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), + ResourceNameFilter: aclsConfig.resourceNameFilter, ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), //PrincipalFilter: "*", HostFilter: aclsConfig.hostFilter, @@ -297,10 +299,11 @@ func aclsCmd() *cobra.Command { "permission-type", `The permission type to filter on. allowed: "any", "allow", or "deny"`, ) - cmd.Flags().Var( - &aclsConfig.resourceType, - "resource-type", - `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + cmd.Flags().StringVar( + &aclsConfig.resourceNameFilter, + "resource-name", + "", + `The resource name to filter on.`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, @@ -309,5 +312,10 @@ func aclsCmd() *cobra.Command { // TODO: match isn't really supported right now, look into that `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, ) + cmd.Flags().Var( + &aclsConfig.resourceType, + "resource-type", + `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + ) return cmd } diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 6cfa33aa..1173a7ea 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -792,7 +792,7 @@ func FormatACLs(acls []ACLInfo) string { acl.Principal, acl.Host, acl.Operation.String(), - fmt.Sprintf("%d", acl.PermissionType), + acl.PermissionType.String(), } table.Append(row) From 5cff3325d1e20f7ae0747d27721e50f2cf3442ac Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 13:41:26 -0400 Subject: [PATCH 24/57] support principal filtering --- cmd/topicctl/subcmd/get.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index b8163076..7b7b731c 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -242,17 +242,19 @@ type aclsCmdConfig struct { hostFilter string operationType admin.ACLOperationType permissionType admin.ACLPermissionType + principalFilter string resourceNameFilter string resourcePatternType admin.PatternType resourceType admin.ResourceType } // aclsConfig defines the default values if a flag is not provided. These all default -// to doing no filtering (e.g. "all") +// to doing no filtering (e.g. "all" or null) var aclsConfig = aclsCmdConfig{ hostFilter: "", operationType: admin.ACLOperationType(kafka.ACLOperationTypeAny), permissionType: admin.ACLPermissionType(kafka.ACLPermissionTypeAny), + principalFilter: "", resourceNameFilter: "", resourceType: admin.ResourceType(kafka.ResourceTypeAny), resourcePatternType: admin.PatternType(kafka.PatternTypeAny), @@ -275,10 +277,10 @@ func aclsCmd() *cobra.Command { ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), ResourceNameFilter: aclsConfig.resourceNameFilter, ResourcePatternTypeFilter: kafka.PatternType(aclsConfig.resourcePatternType), - //PrincipalFilter: "*", - HostFilter: aclsConfig.hostFilter, - Operation: kafka.ACLOperationType(aclsConfig.operationType), - PermissionType: kafka.ACLPermissionType(aclsConfig.permissionType), + PrincipalFilter: aclsConfig.principalFilter, + HostFilter: aclsConfig.hostFilter, + Operation: kafka.ACLOperationType(aclsConfig.operationType), + PermissionType: kafka.ACLPermissionType(aclsConfig.permissionType), } return cliRunner.GetACLs(ctx, filter) }, @@ -299,6 +301,12 @@ func aclsCmd() *cobra.Command { "permission-type", `The permission type to filter on. allowed: "any", "allow", or "deny"`, ) + cmd.Flags().StringVar( + &aclsConfig.principalFilter, + "principal", + "", + `The principal to filter on in principalType:name format (e.g. User:alice).`, + ) cmd.Flags().StringVar( &aclsConfig.resourceNameFilter, "resource-name", From e6e8c634a44b4206390887f935dfe64382ae2e33 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Thu, 14 Sep 2023 14:17:26 -0400 Subject: [PATCH 25/57] improve docs --- cmd/topicctl/subcmd/get.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 7b7b731c..4deb0026 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -289,7 +289,7 @@ func aclsCmd() *cobra.Command { &aclsConfig.hostFilter, "host", "", - `The host to filter on.`, + `The host to filter on. (e.g. 198.51.100.0)`, ) cmd.Flags().Var( &aclsConfig.operationType, @@ -311,14 +311,14 @@ func aclsCmd() *cobra.Command { &aclsConfig.resourceNameFilter, "resource-name", "", - `The resource name to filter on.`, + `The resource name to filter on. (e.g. my-topic)`, ) cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", // TODO: document the behavior of each of these // TODO: match isn't really supported right now, look into that - `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed"`, + `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed". "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, ) cmd.Flags().Var( &aclsConfig.resourceType, From e28cb01d9be4d86e56aef4eb41b0bfe3a14ab83c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 10:58:13 -0400 Subject: [PATCH 26/57] add examples --- cmd/topicctl/subcmd/get.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 4deb0026..9d810ffd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -265,8 +265,18 @@ func aclsCmd() *cobra.Command { Use: "acls", Short: "Displays information for ACLs in the cluster. Supports filtering with flags.", Args: cobra.NoArgs, - // TODO: make common examples here - // Example: + Example: `List all acls +$ topicctl get acls + +List read acls for topic my-topic +$ topicctl get acls --resource-type topic --resource-name my-topic --operations read + +List acls for user Alice with permission allow +$ topicctl get acls --principal User:alice --permission-type allow + +List acls for host 198.51.100.0 +$ topicctl get acls --host 198.51.100.0 +`, RunE: func(cmd *cobra.Command, args []string) error { ctx, cliRunner, err := getCliRunnerAndCtx() if err != nil { @@ -291,6 +301,7 @@ func aclsCmd() *cobra.Command { "", `The host to filter on. (e.g. 198.51.100.0)`, ) + // TODO: support multiple comma separated ones cmd.Flags().Var( &aclsConfig.operationType, "operations", @@ -316,8 +327,6 @@ func aclsCmd() *cobra.Command { cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", - // TODO: document the behavior of each of these - // TODO: match isn't really supported right now, look into that `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed". "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, ) cmd.Flags().Var( From 9735b1b104b3a895c4386acca51c93137adbc476 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:07:10 -0400 Subject: [PATCH 27/57] remove comment --- cmd/topicctl/subcmd/get.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 9d810ffd..5461397a 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -301,7 +301,6 @@ $ topicctl get acls --host 198.51.100.0 "", `The host to filter on. (e.g. 198.51.100.0)`, ) - // TODO: support multiple comma separated ones cmd.Flags().Var( &aclsConfig.operationType, "operations", From b19a4e145052593a466a48000fb678cd72010483 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:08:05 -0400 Subject: [PATCH 28/57] remove TODOs that are complete --- pkg/admin/format.go | 1 - pkg/cli/cli.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/admin/format.go b/pkg/admin/format.go index 1173a7ea..fe0625d0 100644 --- a/pkg/admin/format.go +++ b/pkg/admin/format.go @@ -785,7 +785,6 @@ func FormatACLs(acls []ACLInfo) string { for _, acl := range acls { row := []string{ - // TODO: convert ints to something human readable acl.ResourceType.String(), acl.PatternType.String(), acl.ResourceName, diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 40e4f901..84b37bd6 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -553,7 +553,6 @@ func (c *CLIRunner) Tail( return err } -// TODO add options for filtering // GetACLs fetches the details of each acl in the cluster and prints out a summary. func (c *CLIRunner) GetACLs( ctx context.Context, From 43806c0a86a2fbb1a8ae79e90dd9f94b50bf8e5c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:08:41 -0400 Subject: [PATCH 29/57] remove TODOs that are complete --- pkg/admin/brokerclient.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 329fa1b3..9fbf5e7f 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -697,10 +697,6 @@ func configEntriesToAPIConfigs( return apiConfigs } -// TODO: what fields should we let people filter on / how best to support that? -// It could be really useful to be able to use this to answer questions like what services have access topics x,y,z or -// who has write access to topic b? -// Is GetACL (single ACL) even applicable? // GetACLs gets full information about each ACL in the cluster. func (c *BrokerAdminClient) GetACLs( ctx context.Context, From 2fb8c8e531883df80b6650777675c04a4199e7f1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:15:12 -0400 Subject: [PATCH 30/57] update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4ae37db1..5701dbff 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ resource type in the cluster. Currently, the following operations are supported: | `get partitions [topic]` | All partitions in a topic | | `get offsets [topic]` | Number of messages per partition along with start and end times | | `get topics` | All topics in the cluster | +| `get acls [flags]` | Describe access control levels (ACLs) in the cluster | #### rebalance From 45d403d8728ecd03a7a6784847d25c147c715794 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 11:16:26 -0400 Subject: [PATCH 31/57] fix test --- pkg/admin/brokerclient_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 9a9c77c3..66f25ab6 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -647,7 +647,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { Principal: principal, Host: "*", Operation: ACLOperationType(kafka.ACLOperationTypeRead), - PermissionType: kafka.ACLPermissionTypeAllow, + PermissionType: ACLPermissionType(kafka.ACLPermissionTypeAllow), }, } assert.Equal(t, expected, aclsInfo) From b3a5ef89cd2907a9b1fd2e40baf8238c889f5e20 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 12:04:58 -0400 Subject: [PATCH 32/57] wip --- cmd/topicctl/subcmd/get.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 33d34fed..5461397a 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -208,7 +208,6 @@ func partitionsCmd() *cobra.Command { } } ->>>>>>> master func offsetsCmd() *cobra.Command { return &cobra.Command{ Use: "offsets [topic]", From 6c1f7f11ddafc58b5e30d97321ac1015e62b8d13 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 12:08:01 -0400 Subject: [PATCH 33/57] fix error handling --- pkg/admin/brokerclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 9fbf5e7f..14b73e26 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -758,7 +758,7 @@ func (c *BrokerAdminClient) CreateACL( return err } if len(resp.Errors) > 0 { - fmt.Errorf("%+v", resp.Errors) + return fmt.Errorf("%+v", resp.Errors) } return nil } From cd3a1f66339f9a788fcf6cc93cb3a07a2b9905fe Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:16:16 -0400 Subject: [PATCH 34/57] error handling for zk --- pkg/admin/zkclient.go | 2 +- pkg/admin/zkclient_test.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index fa032764..13e604cb 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -426,7 +426,7 @@ func (c *ZKAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, ) ([]ACLInfo, error) { - return nil, nil + return nil, errors.New("Zookeeper client does not yet support GetACLs. Remove zk-addr and use the broker client instead") } // UpdateTopicConfig updates the config JSON for a topic and sets a change diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 210b23fb..d2adf21d 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1074,3 +1074,20 @@ func TestZkClientLocking(t *testing.T) { func testClusterID(name string) string { return util.RandomString(fmt.Sprintf("cluster-%s-", name), 6) } + +func TestZkGetACLs(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + BootstrapAddrs: []string{util.TestKafkaAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + acls, err := adminClient.GetACLs(ctx, kafka.ACLFilter{}) + assert.Empty(t, acls) + assert.Error(t, err) +} From e0c8c635af965059f90311c32b8ea208300dcca9 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:20:27 -0400 Subject: [PATCH 35/57] more consistent error msg --- pkg/admin/zkclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 13e604cb..5a6f115e 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -426,7 +426,7 @@ func (c *ZKAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, ) ([]ACLInfo, error) { - return nil, errors.New("Zookeeper client does not yet support GetACLs. Remove zk-addr and use the broker client instead") + return nil, errors.New("GetACLs not yet supported with zk access mode; omit zk addresses to fix.") } // UpdateTopicConfig updates the config JSON for a topic and sets a change From 90147f39dc7507d51097119c8be088e595ec3df7 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:40:56 -0400 Subject: [PATCH 36/57] clean up createacl --- pkg/admin/client.go | 6 ++++++ pkg/admin/zkclient.go | 9 ++++++++- pkg/admin/zkclient_test.go | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pkg/admin/client.go b/pkg/admin/client.go index c3b4bf8c..5ec1f731 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -68,6 +68,12 @@ type Client interface { config kafka.TopicConfig, ) error + // Create ACL creates an ACL in the cluster. + CreateACL( + ctx context.Context, + entry kafka.ACLEntry, + ) error + // AssignPartitions sets the replica broker IDs for one or more partitions in a topic. AssignPartitions( ctx context.Context, diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 5a6f115e..0aca8e77 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -426,7 +426,14 @@ func (c *ZKAdminClient) GetACLs( ctx context.Context, filter kafka.ACLFilter, ) ([]ACLInfo, error) { - return nil, errors.New("GetACLs not yet supported with zk access mode; omit zk addresses to fix.") + return nil, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") +} + +func (c *ZKAdminClient) CreateACL( + ctx context.Context, + entry kafka.ACLEntry, +) error { + return errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } // UpdateTopicConfig updates the config JSON for a topic and sets a change diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index d2adf21d..f0a692c4 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1091,3 +1091,19 @@ func TestZkGetACLs(t *testing.T) { assert.Empty(t, acls) assert.Error(t, err) } + +func TestZkCreateACL(t *testing.T) { + ctx := context.Background() + adminClient, err := NewZKAdminClient( + ctx, + ZKAdminClientConfig{ + ZKAddrs: []string{util.TestZKAddr()}, + BootstrapAddrs: []string{util.TestKafkaAddr()}, + }, + ) + require.NoError(t, err) + defer adminClient.Close() + + err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) + assert.Error(t, err) +} From 7534ecfc900bf7d308bbf2ce132b6cc8903538fe Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:47:29 -0400 Subject: [PATCH 37/57] add TestBrokerClientCreateACLReadOnly --- pkg/admin/brokerclient_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 66f25ab6..ea159fa9 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -2,6 +2,7 @@ package admin import ( "context" + "errors" "fmt" "testing" "time" @@ -652,3 +653,21 @@ func TestBrokerClientCreateGetACL(t *testing.T) { } assert.Equal(t, expected, aclsInfo) } + +func TestBrokerClientCreateACLReadOnly(t *testing.T) { + ctx := context.Background() + client, err := NewBrokerAdminClient( + ctx, + BrokerAdminClientConfig{ + ConnectorConfig: ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ReadOnly: true, + }, + ) + require.NoError(t, err) + + err = client.CreateACL(ctx, kafka.ACLEntry{}) + assert.Equal(t, err, errors.New("Cannot create ACL in read-only mode")) + +} From 7551ece928980a497a8112756980d69e2554b0c1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 14:50:52 -0400 Subject: [PATCH 38/57] improve zk tests --- pkg/admin/zkclient_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index f0a692c4..82a873c7 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -2,6 +2,7 @@ package admin import ( "context" + "errors" "fmt" "testing" "time" @@ -1089,7 +1090,7 @@ func TestZkGetACLs(t *testing.T) { acls, err := adminClient.GetACLs(ctx, kafka.ACLFilter{}) assert.Empty(t, acls) - assert.Error(t, err) + assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } func TestZkCreateACL(t *testing.T) { @@ -1105,5 +1106,5 @@ func TestZkCreateACL(t *testing.T) { defer adminClient.Close() err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) - assert.Error(t, err) + assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } From df19f187bb242199741398356bb61a4ec1b2f554 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:14:21 -0400 Subject: [PATCH 39/57] run acl tests in ci --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44329a7b..5cd5b31a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,7 @@ jobs: env: KAFKA_TOPICS_TEST_ZK_ADDR: zookeeper:2181 KAFKA_TOPICS_TEST_KAFKA_ADDR: kafka1:9092 + KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY: 1 services: zookeeper: From e799c4019d33d7ceed6746d99d977ecbff68ee7f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:16:33 -0400 Subject: [PATCH 40/57] enable acls for kafka 2.4.1 in ci --- .github/workflows/ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cd5b31a..ac9d46ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka1 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka2: image: wurstmeister/kafka:2.12-2.4.1 @@ -168,6 +170,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka2 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka3: image: wurstmeister/kafka:2.12-2.4.1 @@ -179,6 +183,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka3 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka4: image: wurstmeister/kafka:2.12-2.4.1 @@ -190,6 +196,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka4 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka5: image: wurstmeister/kafka:2.12-2.4.1 @@ -201,6 +209,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka5 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true kafka6: image: wurstmeister/kafka:2.12-2.4.1 @@ -212,7 +222,8 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka6 KAFKA_ADVERTISED_PORT: 9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true snyk: runs-on: ubuntu-latest From cf690ee6bc9f0ac670ebddb4a1fe48c4383094f5 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:26:48 -0400 Subject: [PATCH 41/57] fix zk tests --- pkg/admin/zkclient_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 82a873c7..34383b7d 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1081,8 +1081,7 @@ func TestZkGetACLs(t *testing.T) { adminClient, err := NewZKAdminClient( ctx, ZKAdminClientConfig{ - ZKAddrs: []string{util.TestZKAddr()}, - BootstrapAddrs: []string{util.TestKafkaAddr()}, + ZKAddrs: []string{util.TestZKAddr()}, }, ) require.NoError(t, err) @@ -1098,8 +1097,7 @@ func TestZkCreateACL(t *testing.T) { adminClient, err := NewZKAdminClient( ctx, ZKAdminClientConfig{ - ZKAddrs: []string{util.TestZKAddr()}, - BootstrapAddrs: []string{util.TestKafkaAddr()}, + ZKAddrs: []string{util.TestZKAddr()}, }, ) require.NoError(t, err) From 41283c7b7f49465ed3df8b41ab3aa74052aa7627 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:35:19 -0400 Subject: [PATCH 42/57] skip TestBrokerClientCreateACLReadOnly on old versions of kafka --- pkg/admin/brokerclient_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index ea159fa9..7f17ced3 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -655,6 +655,9 @@ func TestBrokerClientCreateGetACL(t *testing.T) { } func TestBrokerClientCreateACLReadOnly(t *testing.T) { + if !util.CanTestBrokerAdmin() { + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + } ctx := context.Background() client, err := NewBrokerAdminClient( ctx, From b553d3d9f44d2e896e23b54ad3465774ab483dd2 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:38:40 -0400 Subject: [PATCH 43/57] try to debug --- pkg/admin/brokerclient.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 14b73e26..8a593650 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -745,6 +745,8 @@ func (c *BrokerAdminClient) CreateACL( return errors.New("Cannot create ACL in read-only mode") } + log.SetLevel(log.DebugLevel) + req := kafka.CreateACLsRequest{ ACLs: []kafka.ACLEntry{ entry, From 14811f7ebbf07c3d13c260bce57c7ddca4404b3e Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 15:51:11 -0400 Subject: [PATCH 44/57] handle nested errors from createacls --- pkg/admin/brokerclient.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 8a593650..b6df16c9 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -759,8 +759,14 @@ func (c *BrokerAdminClient) CreateACL( if err != nil { return err } - if len(resp.Errors) > 0 { - return fmt.Errorf("%+v", resp.Errors) + var errors []error + for _, err := range resp.Errors { + if err != nil { + errors = append(errors, err) + } + } + if len(errors) > 0 { + return fmt.Errorf("%+v", errors) } return nil } From 7cc16a652a7a8a047cce5fa72843931e527c6ca1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 16:08:10 -0400 Subject: [PATCH 45/57] operations -> operation --- cmd/topicctl/subcmd/get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 5461397a..ba3b946e 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -303,7 +303,7 @@ $ topicctl get acls --host 198.51.100.0 ) cmd.Flags().Var( &aclsConfig.operationType, - "operations", + "operation", `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, ) cmd.Flags().Var( From 2d126423a28ff30e5541b735972d3d0d30801cc1 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 16:08:25 -0400 Subject: [PATCH 46/57] operations -> operation --- cmd/topicctl/subcmd/get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index ba3b946e..068d35b6 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -269,7 +269,7 @@ func aclsCmd() *cobra.Command { $ topicctl get acls List read acls for topic my-topic -$ topicctl get acls --resource-type topic --resource-name my-topic --operations read +$ topicctl get acls --resource-type topic --resource-name my-topic --operation read List acls for user Alice with permission allow $ topicctl get acls --principal User:alice --permission-type allow From fdb8288a2f4c502835242e9de628540e7101a76d Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Fri, 15 Sep 2023 16:23:16 -0400 Subject: [PATCH 47/57] remove setting log level in test --- pkg/admin/brokerclient.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index b6df16c9..5211e87b 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -745,8 +745,6 @@ func (c *BrokerAdminClient) CreateACL( return errors.New("Cannot create ACL in read-only mode") } - log.SetLevel(log.DebugLevel) - req := kafka.CreateACLsRequest{ ACLs: []kafka.ACLEntry{ entry, From 96dedfd7c4322789bfba93722ca8a907715623c0 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 18 Sep 2023 14:51:43 -0400 Subject: [PATCH 48/57] clean up allowed types in help command --- cmd/topicctl/subcmd/get.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index e3f0390d..53e369fd 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -342,12 +342,12 @@ $ topicctl get acls --host 198.51.100.0 cmd.Flags().Var( &aclsConfig.operationType, "operation", - `The operation that is being allowed or denied to filter on. allowed: "any", "all", "read", "write", "create", "delete", "alter", "describe", "clusteraction", "describeconfigs", "alterconfigs" or "idempotentwrite"`, + `The operation that is being allowed or denied to filter on. allowed: [any, all, read, write, create, delete, alter, describe, clusteraction, describeconfigs, alterconfigs, idempotentwrite]`, ) cmd.Flags().Var( &aclsConfig.permissionType, "permission-type", - `The permission type to filter on. allowed: "any", "allow", or "deny"`, + `The permission type to filter on. allowed: [any, allow, deny]`, ) cmd.Flags().StringVar( &aclsConfig.principalFilter, @@ -364,12 +364,12 @@ $ topicctl get acls --host 198.51.100.0 cmd.Flags().Var( &aclsConfig.resourcePatternType, "resource-pattern-type", - `The type of the resource pattern or filter. allowed: "any", "match", "literal", "prefixed". "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, + `The type of the resource pattern or filter. allowed: [any, match, literal, prefixed]. "any" will match any pattern type (literal or prefixed), but will match the resource name exactly, where as "match" will perform pattern matching to list all acls that affect the supplied resource(s).`, ) cmd.Flags().Var( &aclsConfig.resourceType, "resource-type", - `The type of resource to filter on. allowed: "any", "topic", "group", "cluster", "transactionalid", "delegationtoken"`, + `The type of resource to filter on. allowed: [any, topic, group, cluster, transactionalid, delegationtoken]`, ) return cmd } From d65759d2b38323fd7fdc6b393a3578499b6a1311 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Mon, 18 Sep 2023 14:57:34 -0400 Subject: [PATCH 49/57] fix merge conflict --- cmd/topicctl/subcmd/get.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 53e369fd..4c00e7d8 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -316,10 +316,16 @@ List acls for host 198.51.100.0 $ topicctl get acls --host 198.51.100.0 `, RunE: func(cmd *cobra.Command, args []string) error { - ctx, cliRunner, err := getCliRunnerAndCtx() + ctx := context.Background() + sess := session.Must(session.NewSession()) + + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) if err != nil { return err } + defer adminClient.Close() + + cliRunner := cli.NewCLIRunner(adminClient, log.Infof, !noSpinner) filter := kafka.ACLFilter{ ResourceTypeFilter: kafka.ResourceType(aclsConfig.resourceType), From 36d3de99ca10825a476d9abfbe551919ab4f53fc Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 13:36:42 -0400 Subject: [PATCH 50/57] fix test --- pkg/admin/brokerclient_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 7f17ced3..e88eea25 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -656,7 +656,7 @@ func TestBrokerClientCreateGetACL(t *testing.T) { func TestBrokerClientCreateACLReadOnly(t *testing.T) { if !util.CanTestBrokerAdmin() { - t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN_SECURITY is not set") + t.Skip("Skipping because KAFKA_TOPICS_TEST_BROKER_ADMIN is not set") } ctx := context.Background() client, err := NewBrokerAdminClient( From 9b1262a71ebdd9e5f3d00af8a7bce2801b3a5208 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 19 Sep 2023 15:24:27 -0400 Subject: [PATCH 51/57] add json annotations --- pkg/admin/types.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/admin/types.go b/pkg/admin/types.go index 14b7c744..945c0999 100644 --- a/pkg/admin/types.go +++ b/pkg/admin/types.go @@ -78,13 +78,13 @@ type PartitionAssignment struct { // PartitionInfo represents the information stored about an ACL // in zookeeper. type ACLInfo struct { - ResourceType ResourceType - ResourceName string - PatternType PatternType - Principal string - Host string - Operation ACLOperationType - PermissionType ACLPermissionType + ResourceType ResourceType `json:"resourceType"` + ResourceName string `json:"resourceName"` + PatternType PatternType `json:"patternType"` + Principal string `json:"principal"` + Host string `json:"host"` + Operation ACLOperationType `json:"operation"` + PermissionType ACLPermissionType `json:"permissionType"` } // ResourceType presents the Kafka resource type. From cd2c1f215ea1ff2afc23fa1c5c130c27015d4e33 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 10 Oct 2023 10:18:45 -0400 Subject: [PATCH 52/57] use released version of kafka-go --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 66a35960..26598df8 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/olekukonko/tablewriter v0.0.5 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da - github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 + github.com/segmentio/kafka-go v0.4.43 github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index 6e87f76d..d9ab683e 100644 --- a/go.sum +++ b/go.sum @@ -74,12 +74,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg= -github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= -github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= -github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965 h1:fp6S1UnoT4Tq7N+T30m/2WtvRacFFGMlOcpvkNcoeVI= -github.com/segmentio/kafka-go v0.4.43-0.20230728165410-f4ca0b482965/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= -github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5 h1:Gok6q1P5yUVLYt+auyZgcRBP89thqCmU9MNT65Ms1SI= -github.com/segmentio/kafka-go v0.4.43-0.20230913165112-9ecb9d2f7da5/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= +github.com/segmentio/kafka-go v0.4.43 h1:yKVQ/i6BobbX7AWzwkhulsEn47wpLA8eO6H03bCMqYg= +github.com/segmentio/kafka-go v0.4.43/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070 h1:ng1Z/x5LLOIrzgWUOtypsCkR+dHTux7slqOCVkuwQBo= github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20220211180808-78889264d070/go.mod h1:IjMUGcOJoATsnlqAProGN1ezXeEgU5GCWr1/EzmkEMA= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= From 4b68dff65e9708d70ad7b97847b594ecb7a391bf Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 10 Oct 2023 13:39:06 -0400 Subject: [PATCH 53/57] createacl -> createacls --- pkg/admin/brokerclient.go | 10 ++++------ pkg/admin/brokerclient_test.go | 22 ++++++++++++---------- pkg/admin/client.go | 6 +++--- pkg/admin/zkclient.go | 4 ++-- pkg/admin/zkclient_test.go | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 5211e87b..d4c40609 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -736,19 +736,17 @@ func (c *BrokerAdminClient) GetACLs( return aclinfos, nil } -// CreateACL creates an ACL in the cluster. -func (c *BrokerAdminClient) CreateACL( +// CreateACLs creates an ACL in the cluster. +func (c *BrokerAdminClient) CreateACLs( ctx context.Context, - entry kafka.ACLEntry, + acls []kafka.ACLEntry, ) error { if c.config.ReadOnly { return errors.New("Cannot create ACL in read-only mode") } req := kafka.CreateACLsRequest{ - ACLs: []kafka.ACLEntry{ - entry, - }, + ACLs: acls, } log.Debugf("CreateACLs request: %+v", req) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index e88eea25..85587e45 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -616,16 +616,18 @@ func TestBrokerClientCreateGetACL(t *testing.T) { } }() - err = client.CreateACL( + err = client.CreateACLs( ctx, - kafka.ACLEntry{ - Principal: principal, - PermissionType: kafka.ACLPermissionTypeAllow, - Operation: kafka.ACLOperationTypeRead, - ResourceType: kafka.ResourceTypeTopic, - ResourcePatternType: kafka.PatternTypeLiteral, - ResourceName: topicName, - Host: "*", + []kafka.ACLEntry{ + { + Principal: principal, + PermissionType: kafka.ACLPermissionTypeAllow, + Operation: kafka.ACLOperationTypeRead, + ResourceType: kafka.ResourceTypeTopic, + ResourcePatternType: kafka.PatternTypeLiteral, + ResourceName: topicName, + Host: "*", + }, }, ) require.NoError(t, err) @@ -670,7 +672,7 @@ func TestBrokerClientCreateACLReadOnly(t *testing.T) { ) require.NoError(t, err) - err = client.CreateACL(ctx, kafka.ACLEntry{}) + err = client.CreateACLs(ctx, []kafka.ACLEntry{}) assert.Equal(t, err, errors.New("Cannot create ACL in read-only mode")) } diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 5ec1f731..5c367b04 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -68,10 +68,10 @@ type Client interface { config kafka.TopicConfig, ) error - // Create ACL creates an ACL in the cluster. - CreateACL( + // Create ACLs creates ACLs in the cluster. + CreateACLs( ctx context.Context, - entry kafka.ACLEntry, + acls []kafka.ACLEntry, ) error // AssignPartitions sets the replica broker IDs for one or more partitions in a topic. diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 0aca8e77..a805db8a 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -429,9 +429,9 @@ func (c *ZKAdminClient) GetACLs( return nil, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } -func (c *ZKAdminClient) CreateACL( +func (c *ZKAdminClient) CreateACLs( ctx context.Context, - entry kafka.ACLEntry, + acls []kafka.ACLEntry, ) error { return errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.") } diff --git a/pkg/admin/zkclient_test.go b/pkg/admin/zkclient_test.go index 34383b7d..650e410c 100644 --- a/pkg/admin/zkclient_test.go +++ b/pkg/admin/zkclient_test.go @@ -1103,6 +1103,6 @@ func TestZkCreateACL(t *testing.T) { require.NoError(t, err) defer adminClient.Close() - err = adminClient.CreateACL(ctx, kafka.ACLEntry{}) + err = adminClient.CreateACLs(ctx, []kafka.ACLEntry{}) assert.Equal(t, err, errors.New("ACLs not yet supported with zk access mode; omit zk addresses to fix.")) } From 3f79c7e1feb8b0703159b5229b5b0ccce52464de Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 10 Oct 2023 16:25:43 -0400 Subject: [PATCH 54/57] add minimal repl support --- pkg/cli/repl.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/cli/repl.go b/pkg/cli/repl.go index 77096191..7ea786c8 100644 --- a/pkg/cli/repl.go +++ b/pkg/cli/repl.go @@ -39,6 +39,10 @@ var ( } getSuggestions = []prompt.Suggest{ + { + Text: "acls", + Description: "Get all acls", + }, { Text: "balance", Description: "Get positions of all brokers in a topic or across entire cluster", @@ -226,6 +230,16 @@ func (r *Repl) executor(in string) { } switch command.args[1] { + case "acls": + if err := command.checkArgs(2, 2, nil); err != nil { + log.Errorf("Error: %+v", err) + return + } + if err := r.cliRunner.GetACLs(ctx, kafka.ACLFilter{}); err != nil { + log.Errorf("Error: %+v", err) + return + } + case "balance": if err := command.checkArgs(2, 3, nil); err != nil { log.Errorf("Error: %+v", err) @@ -432,6 +446,10 @@ func helpTable() string { table.AppendBulk( [][]string{ + { + " get acls", + "Get all acls", + }, { " get balance [optional topic]", "Get positions of all brokers in topic or across cluster", From ae5de4717a964155ed2f3c440e17e27b0c44673f Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 10 Oct 2023 16:41:38 -0400 Subject: [PATCH 55/57] add sleep to stop flaky test failures --- pkg/admin/brokerclient_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 85587e45..c0030e93 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -434,6 +434,8 @@ func TestBrokerClientAlterAssignments(t *testing.T) { return err }) + time.Sleep(250 * time.Millisecond) + err = client.AssignPartitions( ctx, topicName, From 41c6acd907f11004e1174e2bf99812889f398237 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Tue, 10 Oct 2023 17:22:08 -0400 Subject: [PATCH 56/57] remove sleep --- pkg/admin/brokerclient_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index aa22e8d7..b75e5491 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -434,8 +434,6 @@ func TestBrokerClientAlterAssignments(t *testing.T) { return err }) - time.Sleep(250 * time.Millisecond) - err = client.AssignPartitions( ctx, topicName, From e31e066c7071a7cc7692b74d42e8f9c8160aee3e Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 11 Oct 2023 12:29:43 -0400 Subject: [PATCH 57/57] capitalize ACls --- pkg/cli/repl.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cli/repl.go b/pkg/cli/repl.go index 37ea8b31..0fcdb2d0 100644 --- a/pkg/cli/repl.go +++ b/pkg/cli/repl.go @@ -41,7 +41,7 @@ var ( getSuggestions = []prompt.Suggest{ { Text: "acls", - Description: "Get all acls", + Description: "Get all ACLs", }, { Text: "balance", @@ -468,7 +468,7 @@ func helpTable() string { [][]string{ { " get acls", - "Get all acls", + "Get all ACLs", }, { " get balance [optional topic]",