docker-compose.yml

version: '3.6'

volumes:
  zookeeper-data:
    driver: local
  zookeeper-log:
    driver: local
  kafka-data:
    driver: local


services:

  # ----------------- #
  # ---Apache Spark---#
  # ----------------- #
  spark-master:
    image: bitnami/sparknltk:latest
    environment:
      - SPARK_MODE=master
      - SPARK_LOCAL_IP=spark-master
    ports:
      - '9093:8080'
      - '4040:4040'
      - '7077:7077'
    volumes:
      - ./apps:/apps
      - ./nltk_data:/nltk_data 

  spark-worker-1:
    image: bitnami/sparknltk:latest
    ports:
      - "9091:8081"
      - "7000:7000"
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_EXECUTOR_MEMORY=1G
      - SPARK_WORKER_CORES=1
    depends_on:
      - spark-master
    volumes:
      - ./apps:/apps 
      - ./nltk_data:/nltk_data 

  spark-worker-2:
    image: bitnami/sparknltk:latest
    ports:
      - "9094:8081"
      - "7001:7000"
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_EXECUTOR_MEMORY=1G
      - SPARK_WORKER_CORES=1

    depends_on:
      - spark-master
    volumes:
      - ./apps:/apps 
      - ./nltk_data:/nltk_data 


  # ----------------------- #
  # Apache Kafka with tools #
  # ------------------------#

   
  zookeeper:
    image: confluentinc/cp-zookeeper:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    volumes:
      - zookeeper-data:/var/lib/zookeeper/data:Z
      - zookeeper-log:/var/lib/zookeeper/log:Z
    environment:
      ZOOKEEPER_CLIENT_PORT: '2181'
      ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false'

  kafka:
    image: confluentinc/cp-kafka:${CONFLUENT_VERSION:-latest}
    restart: unless-stopped
    volumes:
      - kafka-data:/var/lib/kafka/data:Z
    environment:
      KAFKA_BROKER_ID: '0'
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_NUM_PARTITIONS: '12'
      KAFKA_COMPRESSION_TYPE: 'gzip'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1'
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092'
      KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
    links:
      - zookeeper

  akhq:
    image: tchiotludo/akhq
    restart: unless-stopped
    environment:
      AKHQ_CONFIGURATION: |
        akhq:
          connections:
            docker-kafka-server:
              properties:
                bootstrap.servers: "kafka:9092"
              schema-registry:
                url: "http://schema-registry:8085"
              connect:
                - name: "connect"
                  url: "localhost:8083"
    ports:
      - 8080:8080
    links:
      - kafka
      
  # ----------------------- #
  # ----- Apache Nifi------ #
  # ------------------------#

  nifi:
    image: apache/nifi:latest
    container_name: nifi
    ports:
      - 8000:8000  # Map host port 8000 to container port 8000
    environment:
      - NIFI_WEB_HTTP_PORT=8000  # NiFi HTTP port
    volumes:
      - ./nifi-data:/opt/nifi/nifi-current/data  # Persist NiFi data outside the container
    depends_on:
      - zookeeper