Merge branch 'master' into feat/delete-acl

segmentio · Dec 13, 2023 · 95d73d6 · 95d73d6
2 parents aa79335 + e9241f4
commit 95d73d6
Show file tree

Hide file tree

Showing 7 changed files with 654 additions and 149 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,6 +1,10 @@
 name: ci
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
 
 jobs:
@@ -46,78 +50,86 @@ jobs:
 
     services:
       zookeeper:
-        image: wurstmeister/zookeeper
+        image: bitnami/zookeeper:latest
         ports:
           - "2181:2181"
+        env:
+          ALLOW_ANONYMOUS_LOGIN: yes
 
       kafka1:
-        image: wurstmeister/kafka:2.11-0.10.2.2
+        image: bitnami/kafka:0.10.2.1
         ports:
           - "9092:9092"
         env:
           KAFKA_BROKER_ID: 1
           KAFKA_BROKER_RACK: zone1
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          ALLOW_PLAINTEXT_LISTENER: yes
           KAFKA_ADVERTISED_HOST_NAME: kafka1
           KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 
       kafka2:
-        image: wurstmeister/kafka:2.11-0.10.2.2
+        image: bitnami/kafka:0.10.2.1
         ports:
           - "9093:9092"
         env:
           KAFKA_BROKER_ID: 2
           KAFKA_BROKER_RACK: zone1
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          ALLOW_PLAINTEXT_LISTENER: yes
           KAFKA_ADVERTISED_HOST_NAME: kafka2
           KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 
       kafka3:
-        image: wurstmeister/kafka:2.11-0.10.2.2
+        image: bitnami/kafka:0.10.2.1
         ports:
           - "9094:9092"
         env:
           KAFKA_BROKER_ID: 3
           KAFKA_BROKER_RACK: zone2
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          ALLOW_PLAINTEXT_LISTENER: yes
           KAFKA_ADVERTISED_HOST_NAME: kafka3
           KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 
       kafka4:
-        image: wurstmeister/kafka:2.11-0.10.2.2
+        image: bitnami/kafka:0.10.2.1
         ports:
           - "9095:9092"
-        env: 
+        env:
           KAFKA_BROKER_ID: 4
           KAFKA_BROKER_RACK: zone2
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          ALLOW_PLAINTEXT_LISTENER: yes
           KAFKA_ADVERTISED_HOST_NAME: kafka4
           KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 
       kafka5:
-        image: wurstmeister/kafka:2.11-0.10.2.2
+        image: bitnami/kafka:0.10.2.1
         ports:
           - "9096:9092"
-        env: 
+        env:
           KAFKA_BROKER_ID: 5
           KAFKA_BROKER_RACK: zone3
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          ALLOW_PLAINTEXT_LISTENER: yes
           KAFKA_ADVERTISED_HOST_NAME: kafka5
           KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 
       kafka6:
-        image: wurstmeister/kafka:2.11-0.10.2.2
+        image: bitnami/kafka:0.10.2.1
         ports:
           - "9097:9092"
-        env: 
+        env:
           KAFKA_BROKER_ID: 6
           KAFKA_BROKER_RACK: zone3
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          ALLOW_PLAINTEXT_LISTENER: yes
           KAFKA_ADVERTISED_HOST_NAME: kafka6
           KAFKA_ADVERTISED_PORT: 9092
-          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
 
 
-  test271:
+  test270:
     runs-on: ubuntu-latest
     container:
       image: cimg/go:1.19
@@ -143,91 +155,99 @@ jobs:
 
     services:
       zookeeper:
-        image: wurstmeister/zookeeper
+        image: bitnami/zookeeper:latest
         ports:
           - "2181:2181"
+        env:
+          ALLOW_ANONYMOUS_LOGIN: yes
 
       kafka1:
-        image: wurstmeister/kafka:2.13-2.7.1
+        image: bitnami/kafka:2.7.0
         ports:
           - "9092:9092"
         env:
-          KAFKA_BROKER_ID: 1
-          KAFKA_BROKER_RACK: zone1
-          KAFKA_ADVERTISED_HOST_NAME: kafka1
-          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_CFG_BROKER_ID: 1
+          KAFKA_CFG_BROKER_RACK: zone1
           KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-          KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
-          KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+          ALLOW_PLAINTEXT_LISTENER: yes
+          KAFKA_CFG_ADVERTISED_HOST_NAME: kafka1
+          KAFKA_CFG_ADVERTISED_PORT: 9092
+          KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
+          KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
 
       kafka2:
-        image: wurstmeister/kafka:2.13-2.7.1
+        image: bitnami/kafka:2.7.0
         ports:
           - "9093:9092"
         env:
-          KAFKA_BROKER_ID: 2
-          KAFKA_BROKER_RACK: zone1
-          KAFKA_ADVERTISED_HOST_NAME: kafka2
-          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_CFG_BROKER_ID: 2
+          KAFKA_CFG_BROKER_RACK: zone1
           KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-          KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
-          KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+          ALLOW_PLAINTEXT_LISTENER: yes
+          KAFKA_CFG_ADVERTISED_HOST_NAME: kafka2
+          KAFKA_CFG_ADVERTISED_PORT: 9092
+          KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
+          KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
 
       kafka3:
-        image: wurstmeister/kafka:2.13-2.7.1
+        image: bitnami/kafka:2.7.0
         ports:
           - "9094:9092"
         env:
-          KAFKA_BROKER_ID: 3
-          KAFKA_BROKER_RACK: zone2
-          KAFKA_ADVERTISED_HOST_NAME: kafka3
-          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_CFG_BROKER_ID: 3
+          KAFKA_CFG_BROKER_RACK: zone2
           KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-          KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
-          KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+          ALLOW_PLAINTEXT_LISTENER: yes
+          KAFKA_CFG_ADVERTISED_HOST_NAME: kafka3
+          KAFKA_CFG_ADVERTISED_PORT: 9092
+          KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
+          KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
 
       kafka4:
-        image: wurstmeister/kafka:2.13-2.7.1
+        image: bitnami/kafka:2.7.0
         ports:
           - "9095:9092"
-        env: 
-          KAFKA_BROKER_ID: 4
-          KAFKA_BROKER_RACK: zone2
-          KAFKA_ADVERTISED_HOST_NAME: kafka4
-          KAFKA_ADVERTISED_PORT: 9092
+        env:
+          KAFKA_CFG_BROKER_ID: 4
+          KAFKA_CFG_BROKER_RACK: zone2
           KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-          KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
-          KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+          ALLOW_PLAINTEXT_LISTENER: yes
+          KAFKA_CFG_ADVERTISED_HOST_NAME: kafka4
+          KAFKA_CFG_ADVERTISED_PORT: 9092
+          KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
+          KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
 
       kafka5:
-        image: wurstmeister/kafka:2.13-2.7.1
+        image: bitnami/kafka:2.7.0
         ports:
           - "9096:9092"
-        env: 
-          KAFKA_BROKER_ID: 5
-          KAFKA_BROKER_RACK: zone3
-          KAFKA_ADVERTISED_HOST_NAME: kafka5
-          KAFKA_ADVERTISED_PORT: 9092
+        env:
+          KAFKA_CFG_BROKER_ID: 5
+          KAFKA_CFG_BROKER_RACK: zone3
           KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-          KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
-          KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+          ALLOW_PLAINTEXT_LISTENER: yes
+          KAFKA_CFG_ADVERTISED_HOST_NAME: kafka5
+          KAFKA_CFG_ADVERTISED_PORT: 9092
+          KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
+          KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
 
       kafka6:
-        image: wurstmeister/kafka:2.13-2.7.1
+        image: bitnami/kafka:2.7.0
         ports:
           - "9097:9092"
-        env: 
-          KAFKA_BROKER_ID: 6
-          KAFKA_BROKER_RACK: zone3
-          KAFKA_ADVERTISED_HOST_NAME: kafka6
-          KAFKA_ADVERTISED_PORT: 9092
+        env:
+          KAFKA_CFG_BROKER_ID: 6
+          KAFKA_CFG_BROKER_RACK: zone3
           KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-          KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
-          KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
+          ALLOW_PLAINTEXT_LISTENER: yes
+          KAFKA_CFG_ADVERTISED_HOST_NAME: kafka6
+          KAFKA_CFG_ADVERTISED_PORT: 9092
+          KAFKA_CFG_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
+          KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
 
   snyk:
     runs-on: ubuntu-latest
-    needs: [test010, test271]
+    needs: [test010, test270]
     steps:
     - uses: actions/checkout@v3
     - name: Run Snyk to check for vulnerabilities

diff --git a/README.md b/README.md
@@ -165,7 +165,6 @@ Currently, only ACLs are supported. The create command is separate from the appl
 command as it is intended for usage with immutable resources managed by topicctl.
 
 #### delete
-
 ```
 topicctl delete [flags] [operation]
 ```
@@ -504,7 +503,7 @@ The `apply` subcommand can make changes, but under the following conditions:
 The `reset-offsets` command can also make changes in the cluster and should be used carefully.
 
 The `create` command can be used to create new resources in the cluster. It cannot be used with
-mutuable resources.
+mutable resources.
 
 ### Idempotency
 
@@ -619,7 +618,7 @@ make test
 
 You can change the Kafka version of the local cluster by setting the
 `KAFKA_IMAGE_TAG` environment variable when running `docker-compose up -d`. See the
-[`wurstmeister/kafka` dockerhub page](https://hub.docker.com/r/wurstmeister/kafka/tags) for more
+[`bitnami/kafka` dockerhub page](https://hub.docker.com/r/bitnami/kafka/tags) for more
 details on the available versions.
 
 #### Run against local cluster

diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml
@@ -1,3 +1,8 @@
+# By default, this docker-compose setup uses Kafka 2.7.0. This version can
+# be overwritten by setting the KAFKA_IMAGE_TAG environment variable.
+#
+# See https://hub.docker.com/r/bitnami/kafka/tags for the complete list.
+#
 # This config sets up a simple, single-node cluster that's equipped to use SSL/TLS and/or SASL.
 # It exposes access on four separate ports:
 #
@@ -7,46 +12,57 @@
 #    4. 9095: SASL over SSL
 #
 # See examples/auth for the associated cluster configs and certs.
-version: '2'
+version: '3'
 
 services:
   zookeeper:
-    image: "wurstmeister/zookeeper:latest"
+    container_name: zookeeper
+    hostname: zookeeper
+    image: bitnami/zookeeper:latest
     ports:
       - "2181:2181"
+    environment:
+      ALLOW_ANONYMOUS_LOGIN: yes
 
   kafka:
-    image: wurstmeister/kafka:2.13-2.7.1
+    container_name: kafka
+    hostname: kafka
+    image: bitnami/kafka:${KAFKA_IMAGE_TAG:-2.7.0}
+    depends_on:
+      - zookeeper
     restart: on-failure:3
-    links:
-    - zookeeper
     ports:
     - 9092:9092
     - 9093:9093
     - 9094:9094
     - 9095:9095
     environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_MESSAGE_MAX_BYTES: 200000000
-      KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095"
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095"
-      KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512"
-      KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer'
-      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
-      KAFKA_SSL_KEYSTORE_LOCATION: /certs/kafka.keystore.jks
-      KAFKA_SSL_KEYSTORE_PASSWORD: test123
-      KAFKA_SSL_KEY_PASSWORD: test123
-      KAFKA_SSL_TRUSTSTORE_LOCATION: /certs/kafka.truststore.jks
-      KAFKA_SSL_TRUSTSTORE_PASSWORD: test123
-      KAFKA_SSL_CLIENT_AUTH: none
-      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
-      KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf"
-      CUSTOM_INIT_SCRIPT: |-
-        echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n  };' > /opt/kafka/config/kafka_server_jaas.conf;
-        /opt/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config 'SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]' --entity-type users --entity-name adminscram
+      KAFKA_CFG_BROKER_ID: 1
+      KAFKA_CFG_BROKER_RACK: zone1
+      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CFG_MESSAGE_MAX_BYTES: 200000000
+      KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095"
+      KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095"
+      KAFKA_CFG_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512"
+      KAFKA_CFG_AUTHORIZER_CLASS_NAME: "kafka.security.auth.SimpleAclAuthorizer"
+
+      KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
+
+      KAFKA_CFG_SSL_KEYSTORE_LOCATION: /opt/bitnami/kafka/config/certs/kafka.truststore.jks
+      KAFKA_CFG_SSL_KEYSTORE_PASSWORD: test123
+
+      KAFKA_CFG_SSL_TRUSTSTORE_LOCATION: /opt/bitnami/kafka/config/certs/kafka.truststore.jks
+      KAFKA_CFG_SSL_TRUSTSTORE_PASSWORD: test123
+
+
+      KAFKA_CFG_SSL_CLIENT_AUTH: none
+      KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
+      KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf"
+      ALLOW_PLAINTEXT_LISTENER: "yes"
+    entrypoint:
+      - "/bin/bash"
+      - "-c"
+      - echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n  };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - ./examples/auth/certs:/certs
+      - ./examples/auth/certs:/opt/bitnami/kafka/config/certs