docker-compose.yml

version: '3.9'

services:

  zookeeper:
    image: 'docker.io/bitnami/zookeeper:3-debian-10'
    networks:
      - kaf
    ports:
      - '2181:2181'
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes

  kafka1:
    image: 'docker.io/bitnami/kafka:2-debian-10'
    networks:
      - kaf
    ports:
      - '9091:9093'
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_LISTENERS=INTERNAL://:9092,EXTERNAL://:9093
      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_ADVERTISED_LISTENERS=EXTERNAL://localhost:9091,INTERNAL://kafka1:9092
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_MIN_ISR=3
    depends_on:
      - zookeeper

  kafka2:
    image: 'docker.io/bitnami/kafka:2-debian-10'
    networks:
      - kaf
    ports:
      - '9092:9093'
    environment:
      - KAFKA_BROKER_ID=2
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_LISTENERS=INTERNAL://:9092,EXTERNAL://:9093
      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_ADVERTISED_LISTENERS=EXTERNAL://localhost:9092,INTERNAL://kafka2:9092
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    depends_on:
      - zookeeper

  kafka3:
    image: 'docker.io/bitnami/kafka:2-debian-10'
    networks:
        - kaf
    ports:
      - '9093:9093'
    environment:
      - KAFKA_BROKER_ID=3
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_LISTENERS=INTERNAL://:9092,EXTERNAL://:9093
      - KAFKA_ADVERTISED_LISTENERS=EXTERNAL://localhost:9093,INTERNAL://kafka3:9092
      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    depends_on:
      - zookeeper

  kafka4:
    image: 'docker.io/bitnami/kafka:2-debian-10'
    networks:
      - kaf
    ports:
      - '9094:9093'
    environment:
      - KAFKA_BROKER_ID=4
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_LISTENERS=INTERNAL://:9092,EXTERNAL://:9093
      - KAFKA_ADVERTISED_LISTENERS=EXTERNAL://localhost:9094,INTERNAL://kafka4:9092
      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
    depends_on:
      - zookeeper

  jobmanager1:
    image: flink:1.12.0-scala_2.11
    depends_on:
      - zookeeper
    ports:
      - "8081:8081"
    command: jobmanager
    volumes:
      - "./recovery:/tmp/flink/recovery:rw"
    networks:
      - kaf
    environment:
      - |
        FLINK_PROPERTIES=
        restart-strategy: fixeddelay
        restart-strategy.fixed-delay.attempts: 3
        restart-strategy.fixed-delay.delay: 10 s
        high-availability: zookeeper
        high-availability.storageDir: file:///tmp/flink/recovery
        high-availability.zookeeper.quorum: zookeeper:2181
        high-availability.cluster-id: /default_flink

  jobmanager2:
    image: flink:1.12.0-scala_2.11
    depends_on:
      - zookeeper
    ports:
      - "8082:8081"
    command: jobmanager
    volumes:
      - "./recovery:/tmp/flink/recovery:rw"
    networks:
      - kaf
    environment:
      - |
        FLINK_PROPERTIES=
        restart-strategy: fixeddelay
        restart-strategy.fixed-delay.attempts: 3
        restart-strategy.fixed-delay.delay: 10 s
        high-availability: zookeeper
        high-availability.storageDir: file:///tmp/flink/recovery
        high-availability.zookeeper.quorum: zookeeper:2181
        high-availability.cluster-id: /default_flink

  taskmanager:
    image: flink:1.12.0-scala_2.11
    depends_on:
      - zookeeper
      - jobmanager1
      - jobmanager2
    command: taskmanager
    networks:
      - kaf
    scale: 2
    environment:
      - |
        FLINK_PROPERTIES=
        taskmanager.numberOfTaskSlots: 10
        restart-strategy: fixeddelay
        restart-strategy.fixed-delay.attempts: 3
        restart-strategy.fixed-delay.delay: 10 s
        high-availability: zookeeper
        high-availability.storageDir: file:///tmp/flink/recovery
        high-availability.zookeeper.quorum: zookeeper:2181
        high-availability.cluster-id: /default_flink

networks:
  kaf:
    name: kaf
    driver: bridge