docker-compose-extra.yml

version: '3.4'

x-cassandra-oai: &cassandra-oai-image
  cassandra:4.1
x-cassandra-backup-image: &cassandra-backup-image
  docker.dev.fiz-karlsruhe.de/cassandra-backup:5.2
x-elasticsearch-oai-image: &elasticsearch-oai-image
  docker.elastic.co/elasticsearch/elasticsearch:7.17.13
x-elasticsearch-oai-setup-image: &elasticsearch-oai-setup-image
  centos
x-goaccess-image: &goaccess-image
  ghcr.io/mardi4nfdi/docker-goaccess-cron:main
x-grafana-image: &grafana-image
  grafana/grafana
x-jaegertracing-image: &jaegertracing-image
  jaegertracing/all-in-one:latest
x-latexml-image: &latexml-image
  physikerwelt/latexml
x-mardi-backup-image: &mardi-backup-image
  ghcr.io/mardi4nfdi/docker-backup
x-mardi-importer-image: &mardi-importer-image
  ghcr.io/mardi4nfdi/docker-importer:main
x-mardi-importer-api-image: &mardi-importer-api-image
  ghcr.io/mardi4nfdi/importer-api:main
x-matomo-image: &matomo-image
  matomo
x-nginx-image: &nginx-image
  nginx
x-node-exporter-image: &node-exporter-image
  prom/node-exporter:latest
x-oai-backend-image: &oai-backend-image
  docker.dev.fiz-karlsruhe.de/oai-backend:1.3.0
x-oai-provider-image: &oai-provider-image
  docker.dev.fiz-karlsruhe.de/oai-provider:1.3.0
x-prometheus-image: &prometheus-image
  prom/prometheus
x-python-zbmathrest2oai-image: &python-zbmathrest2oai-image
  ghcr.io/mardi4nfdi/python-zbmathrest2oai:main
x-statsd: &statsd-image
  ghcr.io/statsd/statsd
x-scholia-image: &scholia-image
  ghcr.io/mardi4nfdi/scholia:nightly
x-setup-prometheus-grafana: &setup-prometheus-grafana
  ghcr.io/mardi4nfdi/docker-alpine-ext:main
x-uptime-kuma-image: &uptime-kuma-image
  louislam/uptime-kuma:1
x-watchtower-image: &watchtower-image
  containrrr/watchtower
x-whoami: &whoami-image
  containous/whoami
x-flask: &flask-image
  tiangolo/uwsgi-nginx-flask:latest-2024-12-02

services:
  statsd:
    image: *statsd-image

  cassandra-oai:
    hostname: cassandra-oai
    image: *cassandra-oai-image
    environment:
      LOG4J_FORMAT_MSG_NO_LOOKUPS: "true"
    volumes:
    - cassandra-data:/var/lib/cassandra/
    - ./oaipmh/cassandra.yaml:/etc/cassandra/cassandra.yaml
    - ./oaipmh/cassandra-env.sh:/etc/cassandra/cassandra-env.sh
    - ./oaipmh/jmxremote.access:/opt/java/openjdk/lib/management/jmxremote.access
    - ./oaipmh/jmxremote.password:/etc/cassandra/jmxremote.password

  cassandra-oai-setup:
    hostname: cassandra-oai-setup
    image: *cassandra-oai-image
    depends_on:
    - cassandra-oai
    command: ["/wait-for-it.sh","cassandra-oai:9042","--", "sh", "/init-fizoai-database.sh"]
    volumes:
    - ./oaipmh/init-fizoai-database.sh:/init-fizoai-database.sh:ro
    - ./oaipmh/wait-for-it.sh:/wait-for-it.sh:ro

  cassandra-backup:
    hostname: cassandra-backup
    image: *cassandra-backup-image
    environment:
      JAVA_OPTS: "-Dlog4j2.formatMsgNoLookups=true"
      LOG4J_FORMAT_MSG_NO_LOOKUPS: "true"
    env_file:
    - ./oaipmh/.cassandra_dump_env
    volumes:
    - backup-logs:/logs
    - cassandra-data:/source_data
    - ./cassandra-backup:/backup
    depends_on:
    - cassandra-oai

  elasticsearch-oai:
    hostname: elasticsearch-oai
    image: *elasticsearch-oai-image
    environment:
      # - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Dlog4j2.formatMsgNoLookups=true -Xms2g -Xmx2g"
      - "LOG4J_FORMAT_MSG_NO_LOOKUPS=true"
      - discovery.type=single-node
      - xpack.security.enabled=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
    - es-logs:/usr/share/elasticsearch/logs
    - es-data:/usr/share/elasticsearch/data
    # - ./oaipmh/oai-elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro

  elasticsearch-oai-setup:
    hostname: elasticsearch-oai-setup
    image: *elasticsearch-oai-setup-image
    depends_on:
    - elasticsearch-oai
    command: ["/wait-for-it.sh","elasticsearch-oai:9200","--", "sh", "/init-fizoai-elasticsearch.sh"]
    volumes:
    - ./oaipmh/init-fizoai-elasticsearch.sh:/init-fizoai-elasticsearch.sh:ro
    - ./oaipmh/item_mapping_es_v7:/item_mapping_es_v7:ro
    - ./oaipmh/wait-for-it.sh:/wait-for-it.sh:ro

  oai-backend:
    hostname: oai-backend
    image: *oai-backend-image
    environment:
    - "LOG4J_FORMAT_MSG_NO_LOOKUPS=true"
    - "CATALINA_OPTS=-Dlog4j2.formatMsgNoLookups=true -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
    depends_on:
    - cassandra-oai
    - elasticsearch-oai
    links: 
    - "cassandra-oai"
    - "elasticsearch-oai"
    volumes:
    - ./oaipmh/fiz-oai-backend.properties:/usr/local/tomcat/conf/fiz-oai-backend.properties:ro
    - backend-logs:/usr/local/tomcat/logs
    labels:
      - traefik.http.routers.service-oai-backend.rule=Host(`oai-input.${WIKIBASE_HOST}`)
      - traefik.http.routers.service-oai-backend.entrypoints=websecure
      - traefik.http.routers.service-oai-backend.tls.certResolver=le
      - traefik.http.middlewares.oai-auth.basicauth.users=swmath:$$2y$$05$$w37F7z0zZb0ytCvCrxRohOSlcl3lSx9OV9mGTOWOBuMsRRht7LLaO
      - traefik.http.routers.service-oai-backend.middlewares=oai-auth

  oai-provider:
    hostname: oai-provider
    image: *oai-provider-image
    environment:
    - "LOG4J_FORMAT_MSG_NO_LOOKUPS=true"
    - "CATALINA_OPTS=-Dlog4j2.formatMsgNoLookups=true -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true"
    depends_on:
    - oai-backend
    links: 
    - "oai-backend"
    volumes:
    - ./oaipmh/oaicat.properties:/usr/local/tomcat/conf/oaicat.properties:ro
    - provider-logs:/usr/local/tomcat/logs
    labels:
    - traefik.http.routers.oai-provider.rule=Host(`oai.${WIKIBASE_HOST}`)
    - traefik.http.routers.oai-provider.entrypoints=websecure
    - traefik.http.routers.oai-provider.tls.certResolver=le
  
  oai-importer:
    hostname: oai-importer
    image: *python-zbmathrest2oai-image
    environment:
    # start over on each run
    - OAI_STATE_PATH=/tmp/state.json
    depends_on:
    - oai-backend
    restart: always
    
  # A container that exposes an API to show its IP address
  whoami:
    image: *whoami-image 
    restart: always

  mardi-importer:
    image: *mardi-importer-image
    container_name: mardi-importer
    links:
      - wikibase
      - mysql
    depends_on:
      - wikibase
    restart: always
    volumes:
      - shared_mardi_wikibase:/shared/:ro
      #- ./config/:/config
    environment:
      - DB_HOST=mysql.svc
      - DB_NAME=${DB_NAME}
      - DB_USER=${DB_USER}
      - DB_PASS=${DB_PASS}
      - IMPORT_SCHEDULE=${IMPORT_SCHEDULE}
      - IMPORTER_CRON_ENABLE=${IMPORTER_CRON_ENABLE:-false}
      - IMPORTER_AGENT=${IMPORTER_AGENT}
      - MEDIAWIKI_API_URL=http://mardi-wikibase/w/api.php
      - SPARQL_ENDPOINT_URL=http://query.${WIKIBASE_HOST}/proxy/wdqs/bigdata/namespace/wdq/sparql
      - WIKIBASE_URL=http://mardi-wikibase
      - IMPORTER_USER=${IMPORTER_USER}
      - IMPORTER_PASS=${IMPORTER_PASS}
    entrypoint: "/app/start.sh"

  mardi-importer-api:
    image: *mardi-importer-api-image
    container_name: importer-api
    restart: unless-stopped
    links:
      - wikibase
      - mysql
    depends_on:
      - wikibase
    networks:
      default:
        aliases:
         - importer-api.svc
    environment:
      DB_HOST: mardi-mysql
      MYSQL_USER: ${DB_API_USER}
      MYSQL_PASSWORD: ${DB_API_PASS}
      MYSQL_DATABASE: ${DB_NAME}
    labels:
      - traefik.http.routers.importer-api.rule=Host(`importer.${WIKIBASE_HOST}`)
      - traefik.http.routers.importer-api.entrypoints=websecure
      - traefik.http.routers.importer-api.tls.certResolver=le

  mardi-backup:
    image: *mardi-backup-image
    container_name: mardi-backup
    links:
      - mysql
    depends_on:
      - mysql
      - wikibase
    restart: always
    volumes:
      # shared from wikibase, to run dumpBackup.php and importBackup.php
      - shared_mardi_wikibase:/shared/:ro
      - ./mediawiki/LocalSettings.d:/shared/LocalSettings.d
      - shared_mediawiki_images:/var/www/html/images/
      # dir on host where to store the backups
      - ${BACKUP_DIR:-./backup}:/data
    labels:
      - traefik.enable=false
    environment:
      DB_HOST: mysql.svc # internal docker hostname (alias) of the database service
      DB_NAME: ${DB_NAME}
      DB_USER: ${DB_USER}
      DB_PASS: ${DB_PASS}
      BACKUP_SCHEDULE: ${BACKUP_SCHEDULE}
      KEEP_DAYS: ${KEEP_DAYS:-30}
      BACKUP_CRON_ENABLE: ${BACKUP_CRON_ENABLE:-true}
      GF_API_KEY: ${GF_API_KEY}
      GF_PUBLIC_HOST_AND_PORT: ${GF_PUBLIC_HOST_AND_PORT}
      WIKIBASE_SCHEME: ${WIKIBASE_SCHEME:-https}
    entrypoint: "/app/start.sh"

  latexml:
    container_name: latexml
    image: *latexml-image
    restart: always
    labels:
      - traefik.http.routers.latexml.rule=Host(`latexml.${WIKIBASE_HOST}`)
      - traefik.http.routers.latexml.entrypoints=websecure
      - traefik.http.routers.latexml.tls.certResolver=le

  setup-prometheus:
    image: *setup-prometheus-grafana
    volumes:
      - ./prometheus/:/etc/prometheus/:rw
    command: sh -c "envsubst < /etc/prometheus/prometheus.template.yml > /etc/prometheus/prometheus.yml"
    environment:
      - TRAEFIK_USER
      - TRAEFIK_PW
      - HOST_NETWORK_IP
      - WATCHTOWER_API_TOKEN

  prometheus:
    image: *prometheus-image
    container_name: prometheus
    depends_on:
      - setup-prometheus
    restart: unless-stopped
    volumes:
      - ./prometheus/:/etc/prometheus/:ro
      - prometheus_data:/prometheus
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --web.console.libraries=/usr/share/prometheus/console_libraries
      - --web.console.templates=/usr/share/prometheus/consoles
    labels:
      - traefik.http.routers.prometheus.rule=Host(`prometheus.${WIKIBASE_HOST}`)
      - traefik.http.routers.prometheus.entrypoints=websecure
      - traefik.http.routers.prometheus.tls.certResolver=le
      - traefik.http.routers.prometheus.middlewares=auth

  setup-grafana:
    image: *setup-prometheus-grafana
    volumes:
      - ./grafana/:/etc/grafana/:rw
    command: sh -c "envsubst < /etc/grafana/grafana.template.ini > /etc/grafana/grafana.ini"
    environment:
      - GF_MAIL_HOST
      - GF_MAIL_USER
      - GF_MAIL_PW
      - GF_MAIL_FROMADDRESS
      - GF_MAIL_FROMNAME

  grafana:
    image: *grafana-image
    depends_on:
      - setup-grafana
      - prometheus
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/:/etc/grafana/
    labels:
      - traefik.http.routers.grafana.rule=Host(`grafana.${WIKIBASE_HOST}`)
      - traefik.http.routers.grafana.entrypoints=websecure
      - traefik.http.routers.grafana.tls.certResolver=le

  # recommended setup from
  # https://github.com/prometheus/node_exporter/issues/671
  # shared processes and network with host
  node_exporter:
    image: *node-exporter-image
    container_name: node_exporter
    command:
      - '--path.rootfs=/host'
      - '--collector.textfile.directory=/backup_data'
      - '--web.listen-address=:9101'
    network_mode: host
    pid: host
    restart: unless-stopped
    volumes:
      - '/:/host:ro,rslave'
      - '${BACKUP_DIR:-./backup}:/backup_data:ro'

  jaeger:
    image: *jaegertracing-image
    container_name: jaeger
    environment:
      COLLECTOR_ZIPKIN_HTTP_PORT: 9411
    labels:
      - traefik.http.routers.jaeger.rule=Host(`jaeger.${WIKIBASE_HOST}`)
      - traefik.http.routers.jaeger.entrypoints=websecure
      - traefik.http.routers.jaeger.tls.certResolver=le
      - traefik.http.routers.jaeger.middlewares=auth
      - traefik.http.routers.jaeger.service=jaeger
      - traefik.http.services.jaeger.loadbalancer.server.port=16686

  goaccess:
    image: *goaccess-image
    container_name: goaccess
    restart: unless-stopped
    command:
      - /srv/log/access.log
      - /srv/log/access.log.1
      - --output=/srv/reports/index.html
      - --geoip-database=/srv/geoip/GeoLite2-City.mmdb
      - --db-path=/srv/data
      - --log-format='%h %^[%d:%t %^] "%r" %s %b "%R" "%u" %Lm'
      - --date-format=%d/%b/%Y
      - --time-format=%T
    environment:
      - GOACCESS_SCHEDULE=${GOACCESS_SCHEDULE:-0 0 * * *}
    volumes:
      - ./traefik-log:/srv/log:ro
      - goaccess_report:/srv/reports
      - goaccess_db:/srv/data
      - ./goaccess/goaccess.conf:/etc/goaccess/goaccess.conf
      - ./goaccess/GeoLite2-City.mmdb:/srv/geoip/GeoLite2-City.mmdb
    labels:
      - traefik.enable=false

  nginx:
    image: *nginx-image
    container_name: nginx-goaccess
    depends_on:
      - goaccess
    volumes:
      - goaccess_report:/usr/share/nginx/html
    labels:
      - traefik.http.routers.nginx.rule=Host(`stats.${WIKIBASE_HOST}`)
      - traefik.http.routers.nginx.entrypoints=websecure
      - traefik.http.routers.nginx.tls.certResolver=le
      - traefik.http.routers.nginx.middlewares=auth
      
  scholia:
    image: *scholia-image
    restart: unless-stopped
    labels:
      - traefik.http.routers.scholia.rule=Host(`scholia.${WIKIBASE_HOST}`)
      - traefik.http.routers.scholia.entrypoints=websecure
      - traefik.http.routers.scholia.tls.certResolver=le
  
  matomo:
    image: *matomo-image
    restart: always
    volumes:
#     - ./config:/var/www/html/config
#     - ./logs:/var/www/html/logs
      - matomo:/var/www/html
    environment:
      - MATOMO_DATABASE_HOST=${MATOMO_DATABASE_HOST}
      - MATOMO_DATABASE_USERNAME=${MATOMO_DATABASE_USERNAME}
      - MATOMO_DATABASE_PASSWORD=${MATOMO_DATABASE_PASSWORD}
      - MATOMO_DATABASE_DBNAME=${MATOMO_DATABASE_DBNAME}
    labels:
      - traefik.http.routers.matomo.rule=Host(`matomo.${WIKIBASE_HOST}`)
      - traefik.http.routers.matomo.entrypoints=websecure
      - traefik.http.routers.matomo.tls.certResolver=le
  matomo-database:
    image: mariadb:11
    restart: always
    volumes:
      - matomo-mysql-data:/var/lib/mysql
    environment:
      MYSQL_DATABASE: ${MATOMO_DATABASE_DBNAME}
      MYSQL_USER: ${MATOMO_DATABASE_USERNAME}
      MYSQL_PASSWORD: ${MATOMO_DATABASE_PASSWORD}
      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
    

  # Watchtower provides automatic updates for all containers
  # see https://containrrr.github.io/watchtower/arguments/
  watchtower:
    image: *watchtower-image
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    command: --include-stopped --revive-stopped --schedule "0 0 1 * * 1" --http-api-metrics --http-api-token ${WATCHTOWER_API_TOKEN}
    restart: always
    
  uptime-kuma:
    image: *uptime-kuma-image
    container_name: uptime-kuma
    volumes:
      - uptime-kuma:/app/data
    restart: always
    labels:
      - traefik.http.routers.uptime.rule=Host(`uptime.${WIKIBASE_HOST}`)
      - traefik.http.routers.uptime.entrypoints=websecure
      - traefik.http.routers.uptime.tls.certResolver=le

  # see https://github.com/ipfs/kubo/blob/master/docker-compose.yaml
  ipfs:
    image: ipfs/kubo
    restart: unless-stopped
    ports:
      - 4001:4001/tcp
      - 4001:4001/udp
    volumes:
      - ipfs-staging:/export
      - ipfs-data:/data/ipfs
    environment:
      - IPFS_PROFILE=server
    labels:
      - com.centurylinklabs.watchtower.enable=true

      # public gateway
      - traefik.http.services.kubo-service.loadbalancer.server.port=8080
      - traefik.http.routers.kubo-docker.service=kubo-service
      - traefik.http.routers.kubo-docker.rule=Host(`ipfs.${WIKIBASE_HOST}`)
      - traefik.http.routers.kubo-docker.entrypoints=websecure
      - traefik.http.routers.kubo-docker.tls.certResolver=le
      
      #internal management interface
      - traefik.http.services.kubo-rpc-service.loadbalancer.server.port=5001
      - traefik.http.routers.kubo-rpc-docker.service=kubo-rpc-service
      - traefik.http.routers.kubo-rpc-docker.middlewares=auth
      - traefik.http.routers.kubo-rpc-docker.rule=Host(`ipfs-admin.${WIKIBASE_HOST}`)
      - traefik.http.routers.kubo-rpc-docker.entrypoints=websecure
      - traefik.http.routers.kubo-rpc-docker.tls.certResolver=le
  
  mscexplainer:
    image: ghcr.io/gipplab/automscexplainer:latest
    restart: always
    labels:
      - com.centurylinklabs.watchtower.enable=true
      - traefik.http.routers.mscexplainer.rule=Host(`mscexplainer.${WIKIBASE_HOST}`)
      - traefik.http.routers.mscexplainer.entrypoints=websecure
      - traefik.http.routers.mscexplainer.tls.certResolver=le
      
  mscbackend:
    image: aggipp/automscbackend
    restart: always
    labels:
      - com.centurylinklabs.watchtower.enable=true
      - traefik.http.routers.mscbackend.rule=Host(`mscbackend.${WIKIBASE_HOST}`)
      - traefik.http.routers.mscbackend.entrypoints=websecure
      - traefik.http.routers.mscbackend.tls.certResolver=le
      
  msceval:
    image: aggipp/automsceval
    restart: always
    labels:
      - com.centurylinklabs.watchtower.enable=true
      - traefik.http.routers.msceval.rule=Host(`msceval.${WIKIBASE_HOST}`)
      - traefik.http.routers.msceval.entrypoints=websecure
      - traefik.http.routers.msceval.tls.certResolver=le

  flask:
    image: *flask-image
    labels:
      - com.centurylinklabs.watchtower.enable=true
      - traefik.http.routers.flask.rule=Host(`flask.${WIKIBASE_HOST}`)
      - traefik.http.routers.flask.entrypoints=websecure
      - traefik.http.routers.flask.tls.certResolver=le
      - traefik.http.services.flask.loadbalancer.server.port=80

      
volumes:
  backend-logs:
  backup-logs:
  cassandra-data:
  es-data:
  es-logs:
  grafana_data:
  goaccess_report:
  goaccess_db:
  matomo:
  matomo-mysql-data:
  prometheus_data:
  provider-logs:
  uptime-kuma:
  ipfs-staging: # IPFS data on /export cf https://docs.ipfs.tech/install/run-ipfs-inside-docker/#set-up
  ipfs-data: # IPFS data on /data/ipfs