-
Notifications
You must be signed in to change notification settings - Fork 1
/
compose-dev.yaml
79 lines (72 loc) · 2.8 KB
/
compose-dev.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
services:
fluentbit:
image: fluent/fluent-bit
container_name: fluentbit
volumes:
# - ./fluentbit/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
- ./fluentbit/allData.conf:/fluent-bit/etc/fluent-bit.conf
# - ./fluentbit/parsers.conf:/fluent-bit/etc/parsers.conf
- ./producer/logs:/var/log
depends_on:
- broker
command: /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
broker:
image: apache/kafka:latest
hostname: broker
container_name: broker
ports:
- '9092:9092'
environment:
KAFKA_NODE_ID: 1
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://broker:9092,PLAINTEXT://broker:19092'
KAFKA_PROCESS_ROLES: 'broker,controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092'
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
consumer:
image: apache/kafka:latest
hostname: consumer
container_name: consumer
command: >
bash -c "
sleep 10;
/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server broker:9092 --topic room --from-beginning
"
spark:
build:
context: ./spark
dockerfile: Dockerfile
volumes:
- ./spark/python:/opt/spark/work-dir
command: "/opt/spark/bin/spark-submit --conf spark.driver.extraJavaOptions='-Divy.cache.dir=/tmp -Divy.home=/tmp' --master local --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.0.1,org.elasticsearch:elasticsearch-spark-30_2.12:8.13.4 /opt/spark/work-dir/spark_KtoE_train_pred.py"
# sparkEnv:
# image: apache/spark:3.4.2
# volumes:
# - ./spark/python:/opt/spark/work-dir
# command: "/opt/spark/bin/spark-submit --conf spark.driver.extraJavaOptions='-Divy.cache.dir=/tmp -Divy.home=/tmp' --master local --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.0.1,org.elasticsearch:elasticsearch-spark-30_2.12:8.13.4 /opt/spark/work-dir/readkafka.py"
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4
ports:
- 9200:9200
environment:
- discovery.type=single-node
- xpack.security.enabled=false
mem_limit: 1 GB
volumes:
- esdata:/usr/share/elasticsearch/data
kibana:
image: docker.elastic.co/kibana/kibana:8.13.4
ports:
- 5601:5601
depends_on:
- elasticsearch
volumes:
esdata: