-
Notifications
You must be signed in to change notification settings - Fork 1
/
docker-compose.yml
95 lines (88 loc) · 1.84 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
version: '3.9'
services:
# Airflow
airflow:
build: .
environment:
AIRFLOW_HOME: /opt/airflow
restart: unless-stopped
volumes:
- .:/opt/airflow
# Spark
spark:
image: bitnami/spark:3.3.0
environment:
SPARK_MODE: master
restart: unless-stopped
ports:
- 7077:7077
spark-worker:
image: bitnami/spark:3.3.0
environment:
SPARK_MODE: worker
SPARK_MASTER_URL: spark://spark:7077
restart: unless-stopped
deploy:
resources:
limits:
memory: 5G
replicas: 3
depends_on:
- spark
# Databases
postgres:
image: postgres:14-alpine
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- ./.docker/postgresql:/var/lib/postgresql/data
restart: unless-stopped
expose:
- 5432
elasticsearch:
image: elasticsearch:8.4.3
environment:
- xpack.security.enabled=false
- action.destructive_requires_name=false
- discovery.type=single-node
restart: unless-stopped
ports:
- 9200:9200
deploy:
resources:
limits:
cpus: '2.5'
memory: 4G
volumes:
- ./.docker/elasticsearch:/usr/share/elasticsearch/data
# Data Lake
minio:
image: minio/minio
restart: unless-stopped
command: server /data --console-address ":9001"
deploy:
resources:
limits:
cpus: '4'
memory: 2G
volumes:
- ./.docker/minio:/data
ports:
- 9001:9001
# Observability
kibana:
image: kibana:8.4.3
environment:
- elasticsearch.hosts=["http://elasticsearch:9200"]
restart: unless-stopped
deploy:
resources:
limits:
cpus: '1'
memory: 1G
ports:
- 5601:5601
depends_on:
- elasticsearch