diff --git a/CHANGES b/CHANGES
index 990caaa..5f3e2e7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,4 @@
+v0.3.7 Added Docker pieces.
v0.3.6 Fixed a bug with the slash on the end of the ES url option.
v0.3.5 Removed need for trailing slash on ES URL.
v0.3.4 Made datastream names consistent with ES expectations if -d is used without an index name.
diff --git a/Readme.md b/Readme.md
index 64c6715..2bfc4fa 100644
--- a/Readme.md
+++ b/Readme.md
@@ -7,6 +7,7 @@ logs into [ElasticSearch's bulk load JSON format](https://www.elastic.co/guide/e
- [Introduction](#introduction)
- [Installation](#installation)
- [Elastic v8.0+](#elastic80)
+ - [Docker](#docker)
- [Upgrading zeek2es](#upgradingzeek2es)
- [ES Ingest Pipeline](#esingestpipeline)
- [Filtering Data](#filteringdata)
@@ -98,6 +99,28 @@ You can use `zeek2es.py` with the `--user` and `--passwd` command line options t
credentials to ES. You can also supply these options via the extra command line arguments for the helper
scripts.
+### Docker
+
+Probably the easiest way to use this code is through Docker. First, you will want to edit the lines
+with `CHANGEME!!!` in the `.env` file to fit your environment. You will also need to edit the Elastic
+password in `docker/zeek2es/entrypoint.sh` to match. It can be found after the `--passwd` option.
+Next, you can change directory into the `docker` directory and type the following commands to bring
+up a zeek2es and Elasticsearch cluster:
+
+```
+docker-compose build
+dockr-compose up
+```
+
+Now you can put logs in the `VOLUME_MOUNT\logs` directory (`VOLUME_MOUNT` you set in the `.env` file).
+When logs are CREATED in this directory, zeek2es will begin processing them and pushing them into Elasticsearch.
+You can then login to https://localhost:5601 with the username and password you set up in the `.env` file.
+By default there is a self signed certificate, but you can change that if you edit the docker compose files. Once inside
+Kibana you will go to Stack Management->Data Views and create a data view for `logs*` with the timestamp `@timestamp`.
+Now you will be able to go to Discover and start searching your logs! Your data is persistent, in the `VOLUME_MOUNT` directory.
+If you would like to remove all data, just `rm -rf VOLUME_MOUNT`, substituting the directory you set into that remove command.
+The next time you start your cluster it will be brand new for more data.
+
## Upgrading zeek2es
Most upgrades should be as simple as copying the newer [zeek2es.py](zeek2es.py) over
diff --git a/docker/.env b/docker/.env
new file mode 100644
index 0000000..3a21869
--- /dev/null
+++ b/docker/.env
@@ -0,0 +1,32 @@
+# Password for the 'elastic' user (at least 6 characters) CHANGEME!!!
+ELASTIC_PASSWORD=elastic
+
+# Password for the 'kibana_system' user (at least 6 characters) CHANGEME!!!
+KIBANA_PASSWORD=elasticANDkibana
+
+# Version of Elastic products
+STACK_VERSION=8.1.3
+
+# Set the cluster name
+CLUSTER_NAME=docker-cluster
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+#LICENSE=trial
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=9200
+#ES_PORT=127.0.0.1:9200
+
+# Port to expose Kibana to the host
+KIBANA_PORT=5601
+#KIBANA_PORT=80
+
+# Increase or decrease based on the available host memory (in bytes)
+MEM_LIMIT=1073741824
+
+# Project namespace (defaults to the current folder name if not set)
+#COMPOSE_PROJECT_NAME=myproject
+
+# Where the data directory resides for volumes CHANGEME!!!
+VOLUME_MOUNT=./
\ No newline at end of file
diff --git a/docker/data/.empty b/docker/data/.empty
new file mode 100644
index 0000000..e69de29
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
new file mode 100644
index 0000000..4886f37
--- /dev/null
+++ b/docker/docker-compose.yml
@@ -0,0 +1,254 @@
+version: "2.2"
+
+services:
+ setup:
+ image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+ volumes:
+ - certs:/usr/share/elasticsearch/config/certs
+ user: "0"
+ command: >
+ bash -c '
+ if [ x${ELASTIC_PASSWORD} == x ]; then
+ echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+ exit 1;
+ elif [ x${KIBANA_PASSWORD} == x ]; then
+ echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+ exit 1;
+ fi;
+ if [ ! -f config/certs/ca.zip ]; then
+ echo "Creating CA";
+ bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+ unzip config/certs/ca.zip -d config/certs;
+ fi;
+ if [ ! -f config/certs/certs.zip ]; then
+ echo "Creating certs";
+ echo -ne \
+ "instances:\n"\
+ " - name: es01\n"\
+ " dns:\n"\
+ " - es01\n"\
+ " - localhost\n"\
+ " ip:\n"\
+ " - 127.0.0.1\n"\
+ " - name: es02\n"\
+ " dns:\n"\
+ " - es02\n"\
+ " - localhost\n"\
+ " ip:\n"\
+ " - 127.0.0.1\n"\
+ " - name: es03\n"\
+ " dns:\n"\
+ " - es03\n"\
+ " - localhost\n"\
+ " ip:\n"\
+ " - 127.0.0.1\n"\
+ " - name: kibana\n"\
+ " dns:\n"\
+ " - kibana\n"\
+ " - localhost\n"\
+ " ip:\n"\
+ " - 127.0.0.1\n"\
+ > config/certs/instances.yml;
+ bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+ unzip config/certs/certs.zip -d config/certs;
+ fi;
+ echo "Setting file permissions"
+ chown -R root:root config/certs;
+ find . -type d -exec chmod 750 \{\} \;;
+ find . -type f -exec chmod 640 \{\} \;;
+ echo "Waiting for Elasticsearch availability";
+ until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+ echo "Setting kibana_system password";
+ until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+ echo "All done!";
+ '
+ healthcheck:
+ test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+ interval: 1s
+ timeout: 5s
+ retries: 120
+
+ es01:
+ depends_on:
+ setup:
+ condition: service_healthy
+ image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+ restart: "unless-stopped"
+ volumes:
+ - certs:/usr/share/elasticsearch/config/certs
+ - ${VOLUME_MOUNT}/data/es01:/usr/share/elasticsearch/data
+ ports:
+ - ${ES_PORT}:9200
+ environment:
+ - node.name=es01
+ - cluster.name=${CLUSTER_NAME}
+ - cluster.initial_master_nodes=es01,es02,es03
+ - discovery.seed_hosts=es02,es03
+ - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+ - bootstrap.memory_lock=true
+ - xpack.security.enabled=true
+ - xpack.security.http.ssl.enabled=true
+ - xpack.security.http.ssl.key=certs/es01/es01.key
+ - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+ - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+ - xpack.security.http.ssl.verification_mode=certificate
+ - xpack.security.transport.ssl.enabled=true
+ - xpack.security.transport.ssl.key=certs/es01/es01.key
+ - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+ - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+ - xpack.security.transport.ssl.verification_mode=certificate
+ - xpack.license.self_generated.type=${LICENSE}
+ mem_limit: ${MEM_LIMIT}
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ healthcheck:
+ test:
+ [
+ "CMD-SHELL",
+ "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+ ]
+ interval: 10s
+ timeout: 10s
+ retries: 120
+
+ es02:
+ depends_on:
+ - es01
+ image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+ restart: "unless-stopped"
+ volumes:
+ - certs:/usr/share/elasticsearch/config/certs
+ - ${VOLUME_MOUNT}/data/es02:/usr/share/elasticsearch/data
+ environment:
+ - node.name=es02
+ - cluster.name=${CLUSTER_NAME}
+ - cluster.initial_master_nodes=es01,es02,es03
+ - discovery.seed_hosts=es01,es03
+ - bootstrap.memory_lock=true
+ - xpack.security.enabled=true
+ - xpack.security.http.ssl.enabled=true
+ - xpack.security.http.ssl.key=certs/es02/es02.key
+ - xpack.security.http.ssl.certificate=certs/es02/es02.crt
+ - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+ - xpack.security.http.ssl.verification_mode=certificate
+ - xpack.security.transport.ssl.enabled=true
+ - xpack.security.transport.ssl.key=certs/es02/es02.key
+ - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
+ - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+ - xpack.security.transport.ssl.verification_mode=certificate
+ - xpack.license.self_generated.type=${LICENSE}
+ mem_limit: ${MEM_LIMIT}
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ healthcheck:
+ test:
+ [
+ "CMD-SHELL",
+ "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+ ]
+ interval: 10s
+ timeout: 10s
+ retries: 120
+
+ es03:
+ depends_on:
+ - es02
+ image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+ restart: "unless-stopped"
+ volumes:
+ - certs:/usr/share/elasticsearch/config/certs
+ - ${VOLUME_MOUNT}/data/es03:/usr/share/elasticsearch/data
+ environment:
+ - node.name=es03
+ - cluster.name=${CLUSTER_NAME}
+ - cluster.initial_master_nodes=es01,es02,es03
+ - discovery.seed_hosts=es01,es02
+ - bootstrap.memory_lock=true
+ - xpack.security.enabled=true
+ - xpack.security.http.ssl.enabled=true
+ - xpack.security.http.ssl.key=certs/es03/es03.key
+ - xpack.security.http.ssl.certificate=certs/es03/es03.crt
+ - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+ - xpack.security.http.ssl.verification_mode=certificate
+ - xpack.security.transport.ssl.enabled=true
+ - xpack.security.transport.ssl.key=certs/es03/es03.key
+ - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
+ - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+ - xpack.security.transport.ssl.verification_mode=certificate
+ - xpack.license.self_generated.type=${LICENSE}
+ mem_limit: ${MEM_LIMIT}
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ healthcheck:
+ test:
+ [
+ "CMD-SHELL",
+ "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+ ]
+ interval: 10s
+ timeout: 10s
+ retries: 120
+
+ kibana:
+ depends_on:
+ es01:
+ condition: service_healthy
+ es02:
+ condition: service_healthy
+ es03:
+ condition: service_healthy
+ image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+ restart: "unless-stopped"
+ volumes:
+ - certs:/usr/share/kibana/config/certs
+ - ${VOLUME_MOUNT}/data/kibana:/usr/share/kibana/data
+ ports:
+ - ${KIBANA_PORT}:5601
+ environment:
+ - SERVERNAME=kibana
+ - ELASTICSEARCH_HOSTS=https://es01:9200
+ - ELASTICSEARCH_USERNAME=kibana_system
+ - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+ - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+ - SERVER_SSL_ENABLED=true
+ - SERVER_SSL_KEY=/usr/share/kibana/config/certs/kibana/kibana.key
+ - SERVER_SSL_CERTIFICATE=/usr/share/kibana/config/certs/kibana/kibana.crt
+ - SERVER_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+# - SERVER_SSL_PASSWORD=${KIBANA_CERT_PASSWORD}
+ mem_limit: ${MEM_LIMIT}
+ healthcheck:
+ test:
+ [
+ "CMD-SHELL",
+ "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
+ ]
+ interval: 10s
+ timeout: 10s
+ retries: 120
+
+ zeek2es:
+ build:
+ context: ./zeek2es
+ dockerfile: Dockerfile
+ restart: "unless-stopped"
+ depends_on:
+ es01:
+ condition: service_healthy
+ es02:
+ condition: service_healthy
+ es03:
+ condition: service_healthy
+ volumes:
+ - ${VOLUME_MOUNT}/data/logs:/logs
+ entrypoint: /entrypoint.sh
+ tty: true
+
+volumes:
+ certs:
+ driver: local
\ No newline at end of file
diff --git a/docker/zeek2es/Dockerfile b/docker/zeek2es/Dockerfile
new file mode 100644
index 0000000..b1efe07
--- /dev/null
+++ b/docker/zeek2es/Dockerfile
@@ -0,0 +1,35 @@
+FROM ubuntu:focal
+
+RUN apt-get -q update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ curl \
+ fswatch \
+ geoipupdate \
+ git \
+ iproute2 \
+ jq \
+ less \
+ net-tools \
+ parallel \
+ python3 \
+ python3-dev \
+ python3-pip \
+ python3-setuptools \
+ python3-wheel \
+ swig \
+ tcpdump \
+ tcpreplay \
+ termshark \
+ tshark \
+ vim \
+ wget && \
+ pip3 install --no-cache-dir pre-commit requests
+
+# Clean up files
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf ~/.cache/pip
+
+# Install zeek2es
+RUN cd / && git clone https://github.com/corelight/zeek2es.git
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod 755 /entrypoint.sh
\ No newline at end of file
diff --git a/docker/zeek2es/entrypoint.sh b/docker/zeek2es/entrypoint.sh
new file mode 100644
index 0000000..7308887
--- /dev/null
+++ b/docker/zeek2es/entrypoint.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+fswatch -m poll_monitor --event Created -r /logs | parallel -j 3 python3 /zeek2es/zeek2es.py {} --compress -g -l 5000 -d 25 -u https://es01:9200 --user elastic --passwd elastic :::: -
\ No newline at end of file