Bugfix/fixing testsuite and enhanced documentation for testing

.env

@@ -0,0 +1,41 @@
+# Project namespace (defaults to the current folder name if not set)
+#COMPOSE_PROJECT_NAME=myproject
+
+
+# Password for the 'elastic' user (at least 6 characters)
+ELASTIC_PASSWORD=changeme
+
+
+# Password for the 'kibana_system' user (at least 6 characters)
+KIBANA_PASSWORD=changeme
+
+
+# Version of Elastic products
+STACK_VERSION=8.7.1
+
+
+# Set the cluster name
+CLUSTER_NAME=docker-cluster
+
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+#LICENSE=trial
+
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=9200
+
+
+# Port to expose Kibana to the host
+KIBANA_PORT=5601
+
+
+# Increase or decrease based on the available host memory (in bytes)
+ES_MEM_LIMIT=1073741824
+KB_MEM_LIMIT=1073741824
+LS_MEM_LIMIT=1073741824
+
+
+# SAMPLE Predefined Key only to be used in POC environments
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2

.gitignore

@@ -102,7 +102,6 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
 .venv
 env/
 venv/

README.md

@@ -173,27 +173,46 @@ class MyAPIEvaluator(Evaluator):
 
 ## Testing
 
+### Proconditions
+
+1. Install [gdal](https://gdal.org/download.html#binaries) on your system. This package and some site-packages will use it to calculate geospatial data.
+2. Provide an elasticsearch instance. You can simply use the provided docker compose setup.
+
+
+### Python dependencies
+
 For testing, several requirements must be satisfied. These can be installed, via pip:
 
 ```bash
 pip install -r requirements-dev.txt
 pip install -r requirements-test.txt
 ```
 
+### Docker
+
+Start the elasticsearch instance:
+
+```bash
+docker compose up
+
+```
+
+### Excecuting tests
+
+
 The functionality can be tested using `pytest`.
 
 ```bash
 python -m pytest
 ```
 
-### Docker
 
 To execute tests in Docker:
 
+```bash
+docker compose up --build test
 ```
-docker build -t pygeofilter/test -f Dockerfile-3.9 .
-docker run --rm pygeofilter/test
-```
+
 
 ## Backends
 

Dockerfile-3.9 → bullseye-3.9.Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-buster
+FROM python:3.9-bullseye
 
 LABEL description="Test executor"
 
@@ -8,6 +8,7 @@ RUN apt-get update --fix-missing \
   binutils \
   libproj-dev \
   gdal-bin \
+  libgdal-dev \
   libsqlite3-mod-spatialite \
   spatialite-bin \
   && rm -rf /var/lib/apt/lists/*
@@ -19,13 +20,12 @@ COPY requirements-test.txt .
 COPY requirements-dev.txt .
 RUN pip install -r requirements-test.txt
 RUN pip install -r requirements-dev.txt
+RUN pip install pygdal=="`gdal-config --version`.*"
 
 COPY pygeofilter pygeofilter
 COPY tests tests
 COPY README.md .
 COPY setup.py .
 RUN pip install -e .
 
-RUN chmod +x tests/execute-tests.sh 
-
-CMD ["tests/execute-tests.sh"]
+CMD ["python", "-m", "pytest"]

docker-compose.yml

@@ -0,0 +1,135 @@
+version: '3.8'
+services:
+
+  test:
+    build: 
+      context: .
+      dockerfile: bullseye-3.9.Dockerfile 
+    tty: true # To support colorized log output.
+    networks:
+      - default
+    depends_on:
+      - es01
+    environment:
+      - eshostname=es01
+
+  es01:
+    depends_on:
+     setup:
+       condition: service_healthy
+
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
+    ports:
+      - ${ES_PORT}:9200
+    env_file:
+      - .env
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true 
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${ES_MEM_LIMIT}
+    volumes:
+     - certs:/usr/share/elasticsearch/config/certs
+     - esdata01:/usr/share/elasticsearch/data
+
+    healthcheck:
+     test:
+       [
+         "CMD-SHELL",
+         "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+       ]
+     interval: 10s
+     timeout: 10s
+     retries: 120
+
+
+  setup:
+   image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
+   volumes:
+     - certs:/usr/share/elasticsearch/config/certs
+   user: "0"
+   env_file:
+      - .env
+   command: >
+     bash -c '
+       if [ x${ELASTIC_PASSWORD} == x ]; then
+         echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+         exit 1;
+       elif [ x${KIBANA_PASSWORD} == x ]; then
+         echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+         exit 1;
+       fi;
+       if [ ! -f config/certs/ca.zip ]; then
+         echo "Creating CA";
+         bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+         unzip config/certs/ca.zip -d config/certs;
+       fi;
+       if [ ! -f config/certs/certs.zip ]; then
+         echo "Creating certs";
+         echo -ne \
+         "instances:\n"\
+         "  - name: es01\n"\
+         "    dns:\n"\
+         "      - es01\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
+         "  - name: kibana\n"\
+         "    dns:\n"\
+         "      - kibana\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
+         > config/certs/instances.yml;
+         bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+         unzip config/certs/certs.zip -d config/certs;
+       fi;
+       echo "Setting file permissions"
+       chown -R root:root config/certs;
+       find . -type d -exec chmod 750 \{\} \;;
+       find . -type f -exec chmod 640 \{\} \;;
+       echo "Waiting for Elasticsearch availability";
+       until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+       echo "Setting kibana_system password";
+       until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+       echo "All done!";
+     '
+   healthcheck:
+     test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+     interval: 1s
+     timeout: 5s
+     retries: 120
+
+
+volumes:
+ certs:
+   driver: local
+ esdata01:
+   driver: local
+ kibanadata:
+   driver: local
+ metricbeatdata01:
+   driver: local
+ filebeatdata01:
+   driver: local
+ logstashdata01:
+   driver: local
+
+
+networks:
+ default:
+   name: elastic
+   external: false