diff --git a/.env b/.env new file mode 100644 index 0000000..5013ac6 --- /dev/null +++ b/.env @@ -0,0 +1,41 @@ +# Project namespace (defaults to the current folder name if not set) +#COMPOSE_PROJECT_NAME=myproject + + +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD=changeme + + +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD=changeme + + +# Version of Elastic products +STACK_VERSION=8.7.1 + + +# Set the cluster name +CLUSTER_NAME=docker-cluster + + +# Set to 'basic' or 'trial' to automatically start the 30-day trial +LICENSE=basic +#LICENSE=trial + + +# Port to expose Elasticsearch HTTP API to the host +ES_PORT=9200 + + +# Port to expose Kibana to the host +KIBANA_PORT=5601 + + +# Increase or decrease based on the available host memory (in bytes) +ES_MEM_LIMIT=1073741824 +KB_MEM_LIMIT=1073741824 +LS_MEM_LIMIT=1073741824 + + +# SAMPLE Predefined Key only to be used in POC environments +ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 diff --git a/.gitignore b/.gitignore index 0964046..607cf71 100644 --- a/.gitignore +++ b/.gitignore @@ -102,7 +102,6 @@ celerybeat.pid *.sage.py # Environments -.env .venv env/ venv/ diff --git a/README.md b/README.md index ad770cc..d8a36d8 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,14 @@ class MyAPIEvaluator(Evaluator): ## Testing +### Proconditions + +1. Install [gdal](https://gdal.org/download.html#binaries) on your system. This package and some site-packages will use it to calculate geospatial data. +2. Provide an elasticsearch instance. You can simply use the provided docker compose setup. + + +### Python dependencies + For testing, several requirements must be satisfied. These can be installed, via pip: ```bash @@ -180,20 +188,31 @@ pip install -r requirements-dev.txt pip install -r requirements-test.txt ``` +### Docker + +Start the elasticsearch instance: + +```bash +docker compose up + +``` + +### Excecuting tests + + The functionality can be tested using `pytest`. ```bash python -m pytest ``` -### Docker To execute tests in Docker: +```bash +docker compose up --build test ``` -docker build -t pygeofilter/test -f Dockerfile-3.9 . -docker run --rm pygeofilter/test -``` + ## Backends diff --git a/Dockerfile-3.9 b/bullseye-3.9.Dockerfile similarity index 81% rename from Dockerfile-3.9 rename to bullseye-3.9.Dockerfile index 94c019a..9b8cd40 100644 --- a/Dockerfile-3.9 +++ b/bullseye-3.9.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-buster +FROM python:3.9-bullseye LABEL description="Test executor" @@ -8,6 +8,7 @@ RUN apt-get update --fix-missing \ binutils \ libproj-dev \ gdal-bin \ + libgdal-dev \ libsqlite3-mod-spatialite \ spatialite-bin \ && rm -rf /var/lib/apt/lists/* @@ -19,6 +20,7 @@ COPY requirements-test.txt . COPY requirements-dev.txt . RUN pip install -r requirements-test.txt RUN pip install -r requirements-dev.txt +RUN pip install pygdal=="`gdal-config --version`.*" COPY pygeofilter pygeofilter COPY tests tests @@ -26,6 +28,4 @@ COPY README.md . COPY setup.py . RUN pip install -e . -RUN chmod +x tests/execute-tests.sh - -CMD ["tests/execute-tests.sh"] +CMD ["python", "-m", "pytest"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..682637c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,135 @@ +version: '3.8' +services: + + test: + build: + context: . + dockerfile: bullseye-3.9.Dockerfile + tty: true # To support colorized log output. + networks: + - default + depends_on: + - es01 + environment: + - eshostname=es01 + + es01: + depends_on: + setup: + condition: service_healthy + + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 + ports: + - ${ES_PORT}:9200 + env_file: + - .env + environment: + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${ES_MEM_LIMIT} + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + env_file: + - .env + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + +volumes: + certs: + driver: local + esdata01: + driver: local + kibanadata: + driver: local + metricbeatdata01: + driver: local + filebeatdata01: + driver: local + logstashdata01: + driver: local + + +networks: + default: + name: elastic + external: false \ No newline at end of file diff --git a/tests/backends/elasticsearch/test_evaluate.py b/tests/backends/elasticsearch/test_evaluate.py index e3e8904..bdec193 100644 --- a/tests/backends/elasticsearch/test_evaluate.py +++ b/tests/backends/elasticsearch/test_evaluate.py @@ -1,5 +1,5 @@ # pylint: disable=W0621,C0114,C0115,C0116 - +import os import pytest from elasticsearch_dsl import ( Date, @@ -53,8 +53,12 @@ class Index: @pytest.fixture(autouse=True, scope="session") def connection(): + hostname = os.environ.get("eshostname", "localhost") connections.create_connection( - hosts=["http://localhost:9200"], + hosts=[f"https://{hostname}:9200"], + ca_certs=False, + verify_certs=False, + basic_auth=("elastic", "changeme") ) @@ -107,9 +111,7 @@ def data(index): def filter_(ast_): query = to_filter(ast_, version="8.2") - print(query) result = Record.search().query(query).execute() - print([r.identifier for r in result]) return result @@ -135,35 +137,35 @@ def test_comparison(data): def test_combination(data): result = filter_(parse("int_attribute = 5 AND float_attribute < 6.0")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("int_attribute = 6 OR float_attribute < 6.0")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier def test_between(data): result = filter_(parse("float_attribute BETWEEN -1 AND 1")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("int_attribute NOT BETWEEN 4 AND 6")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier def test_like(data): result = filter_(parse("str_attribute LIKE 'this is a test'")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("str_attribute LIKE 'this is % test'")) assert len(result) == 2 result = filter_(parse("str_attribute NOT LIKE '% another test'")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("str_attribute NOT LIKE 'this is . test'")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier result = filter_(parse("str_attribute ILIKE 'THIS IS . TEST'")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("str_attribute ILIKE 'THIS IS % TEST'")) assert len(result) == 2 @@ -171,18 +173,18 @@ def test_like(data): def test_in(data): result = filter_(parse("int_attribute IN ( 1, 2, 3, 4, 5 )")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("int_attribute NOT IN ( 1, 2, 3, 4, 5 )")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier def test_null(data): result = filter_(parse("maybe_str_attribute IS NULL")) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_(parse("maybe_str_attribute IS NOT NULL")) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier def test_has_attr(): @@ -203,17 +205,17 @@ def test_temporal(data): ], ) ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_( parse("datetime_attribute BEFORE 2000-01-01T00:00:05.00Z"), ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier result = filter_( parse("datetime_attribute AFTER 2000-01-01T00:00:05.00Z"), ) - assert len(result) == 1 and result[0].identifier is data[1].identifier + assert len(result) == 1 and result[0].identifier == data[1].identifier # def test_array(): @@ -258,14 +260,14 @@ def test_spatial(data): result = filter_( parse("INTERSECTS(geometry, ENVELOPE (0.0 1.0 0.0 1.0))"), ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier # TODO: test more spatial queries result = filter_( parse("BBOX(center, 2, 2, 3, 3)"), ) - assert len(result) == 1 and result[0].identifier is data[0].identifier + assert len(result) == 1 and result[0].identifier == data[0].identifier # def test_arithmetic(): diff --git a/tests/backends/sqlalchemy/test_evaluate.py b/tests/backends/sqlalchemy/test_evaluate.py index 705f3af..b12e019 100644 --- a/tests/backends/sqlalchemy/test_evaluate.py +++ b/tests/backends/sqlalchemy/test_evaluate.py @@ -31,7 +31,6 @@ class Record(Base): geometry_type="MULTIPOLYGON", srid=4326, spatial_index=False, - management=True, ) ) float_attribute = Column(Float)