From b6d115663076476ef5ee795a5bff2e87e1a3c68f Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Fri, 11 Oct 2024 13:17:01 -0700 Subject: [PATCH] Squashed commit of the following: commit 10539ed8cd4e38fb00af771fca608c76e005de7d Author: ajnisbet Date: Fri Oct 11 13:15:54 2024 -0700 Upgrade version commit d1fc1a733a097b2fbd5dd39336c29a15fb8f0840 Author: ajnisbet Date: Fri Oct 11 13:11:43 2024 -0700 Working watchdog commit 9d510718cdeabf43bdfe258e0852a7d56793ec91 Author: Arne Setzer <25772747+arnesetzer@users.noreply.github.com> Date: Fri Oct 11 20:58:43 2024 +0200 Add support for hot reloading the config.yaml * add support for hot reloading the config.yaml * watchdogs also watches on example-config.yaml * watchdog remove os.system stuff * watchdog add requirments.{txt,in} --------- Co-authored-by: Andrew Nisbet commit fc407a246cf6946cd859a163a3afaa21b7018885 Author: ajnisbet Date: Fri Oct 11 11:56:00 2024 -0700 Upgrade dependencies commit 5ec7c25df4430e81895d39823da8f4a586002b93 Author: ajnisbet Date: Fri Oct 11 11:35:31 2024 -0700 Docker tweaks commit 87c974267a81b092f6dc9c78c84c8084a730896a Merge: cc7f997 e93ff94 Author: ajnisbet Date: Fri Oct 11 11:35:00 2024 -0700 Merge branch 'master' into dev commit cc7f9979344827374dbf034ded9e0c7fc64ab1db Author: ajnisbet Date: Tue Feb 20 11:46:21 2024 -0800 Tidy cors --- Makefile | 10 +++- VERSION | 2 +- docker/Dockerfile | 10 ++-- docker/config_watcher.py | 110 +++++++++++++++++++++++++++++++++++++++ docker/supervisord.conf | 19 +++++++ docker/warm_cache.py | 4 +- docs/changelog.md | 4 ++ docs/server.md | 2 + example-config.yaml | 2 +- opentopodata/api.py | 7 ++- opentopodata/backend.py | 8 ++- requirements.in | 5 +- requirements.txt | 62 +++++++++++----------- 13 files changed, 199 insertions(+), 46 deletions(-) create mode 100644 docker/config_watcher.py diff --git a/Makefile b/Makefile index 12db43f..650ccf6 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,12 @@ build: build-m1: docker build --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile . +rebuild: + docker build --no-cache --tag opentopodata:$(VERSION) --file docker/Dockerfile . + +rebuild-m1: + docker build --no-cache --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile . + run: docker run --rm -it --volume "$(shell pwd)/data:/app/data:ro" -p 5000:5000 opentopodata:$(VERSION) @@ -23,10 +29,10 @@ run-local: FLASK_APP=opentopodata/api.py FLASK_DEBUG=1 flask run --port 5000 black: - black --target-version py39 tests opentopodata + black --target-version py311 tests opentopodata docker black-check: - docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py39 tests opentopodata + docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py311 tests opentopodata update-requirements: build # pip-compile gets confused if there's already a requirements.txt file, and diff --git a/VERSION b/VERSION index abb1658..ed21137 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.9.0 \ No newline at end of file +1.10.0 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d6d1b8..b722280 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # Container for packages that need to be built from source but have massive dev dependencies. -FROM python:3.11.8-slim-bookworm as builder +FROM python:3.11.10-slim-bookworm as builder RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -7,14 +7,16 @@ RUN set -e && \ python3.11-dev RUN pip config set global.disable-pip-version-check true && \ - pip wheel --wheel-dir=/root/wheels uwsgi==2.0.24 && \ - pip wheel --wheel-dir=/root/wheels regex==2023.12.25 + pip wheel --wheel-dir=/root/wheels uwsgi==2.0.27 && \ + pip wheel --wheel-dir=/root/wheels regex==2024.9.11 # The actual container. -FROM python:3.11.8-slim-bookworm +FROM python:3.11.10-slim-bookworm RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ + inotify-tools \ + nano \ nginx \ memcached \ supervisor && \ diff --git a/docker/config_watcher.py b/docker/config_watcher.py new file mode 100644 index 0000000..c01a451 --- /dev/null +++ b/docker/config_watcher.py @@ -0,0 +1,110 @@ +import logging +import time +from pathlib import Path +import subprocess +import sys + +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + + +# Paths. +CONFIG_DIR = Path("/app/") +CONFIG_PATH = Path("/app/config.yaml") +EXAMPLE_CONFIG_PATH = Path("/app/example-config.yaml") + +# Debouncing: once the config has been reloaded, any queued unprocessed events should be discarded. +LAST_INVOCATION_TIME = time.time() + + +# Logger setup. +logger = logging.getLogger("configwatcher") +LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" +formatter = logging.Formatter(LOG_FORMAT) +logger.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.INFO) +handler.setFormatter(formatter) +logger.addHandler(handler) + + +def run_cmd(cmd, shell=False): + r = subprocess.run(cmd, shell=shell, capture_output=True) + is_error = r.returncode != 0 + stdout = r.stdout.decode("utf-8") + if is_error: + logger.error(f"Error running command, returncode: {r.returncode}") + logger.error("cmd:") + logger.error(" ".join(cmd)) + if r.stdout: + logger.error("stdout:") + logger.error(stdout) + if r.stderr: + logger.error("stderr:") + logger.error(r.stderr.decode("utf-8")) + raise ValueError + return stdout + + +def reload_config(): + global LAST_INVOCATION_TIME + LAST_INVOCATION_TIME = time.time() + logger.info("Restarting OTD due to config change.") + run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "stop", "uwsgi"]) + run_cmd( + ["supervisorctl", "-c", "/app/docker/supervisord.conf", "restart", "memcached"] + ) + run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "uwsgi"]) + run_cmd( + ["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "warm_cache"] + ) + LAST_INVOCATION_TIME = time.time() + logger.info("Restarted OTD due to config change.") + + +class Handler(FileSystemEventHandler): + + def on_any_event(self, event): + watch_paths_str = [ + EXAMPLE_CONFIG_PATH.as_posix(), + CONFIG_PATH.as_posix(), + ] + + # Filter unwanted events. + if event.event_type not in {"modified", "created"}: + logger.debug(f"Dropping event with type {event.event_type=}") + return + if event.is_directory: + logger.debug(f"Dropping dir event") + return + if event.src_path not in watch_paths_str: + logger.debug(f"Dropping event with path {event.src_path=}") + return + if not Path(event.src_path).exists(): + logger.debug(f"Dropping event for nonexistent path {event.src_path=}") + return + + # Debouncing. + mtime = Path(event.src_path).lstat().st_mtime + if mtime < LAST_INVOCATION_TIME: + msg = f"Dropping event for file that hasn't been modified since the last run. {event.src_path=}" + logger.debug(msg) + return + + logger.debug(f"Dispatching event on {event.src_path=}") + reload_config() + + +if __name__ == "__main__": + + event_handler = Handler() + observer = Observer() + observer.schedule(event_handler, CONFIG_DIR, recursive=False) + observer.start() + + try: + while True: + time.sleep(1) + finally: + observer.stop() + observer.join() diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 7ff47ca..57f6577 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -2,6 +2,19 @@ nodaemon=true user=root + +# Supervisorctl config/ +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + + +# OTD services. + [program:uwsgi] command=/usr/local/bin/uwsgi --ini /app/docker/uwsgi.ini --processes %(ENV_N_UWSGI_THREADS)s stdout_logfile=/dev/stdout @@ -33,3 +46,9 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 autorestart=false +[program:watch_config] +command=python /app/docker/config_watcher.py +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 \ No newline at end of file diff --git a/docker/warm_cache.py b/docker/warm_cache.py index c5bd92d..1644313 100644 --- a/docker/warm_cache.py +++ b/docker/warm_cache.py @@ -34,5 +34,7 @@ break else: - logging.error("Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working.") + logging.error( + "Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working." + ) sys.exit(1) diff --git a/docs/changelog.md b/docs/changelog.md index a8ccdce..4d0eef1 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ This is a list of changes to Open Topo Data between each release. +## Version 1.10.0 (11 Oct 2024) +* Minior dependency upgrades +* Auto-reloading of config files without restarting docker ([#100](https://github.com/ajnisbet/opentopodata/pull/100) thanks [@arnesetzer](https://github.com/arnesetzer)!) + ## Version 1.9.0 (19 Feb 2024) * Dependency upgrades, including python to 3.11 and rasterio to 1.3.9 diff --git a/docs/server.md b/docs/server.md index b77239f..d2f6151 100644 --- a/docs/server.md +++ b/docs/server.md @@ -97,6 +97,8 @@ opentopodata which would expose `localhost:5000/v1/etopo1` and `localhost:5000/v1/srtm90m`. +Mofifying the config file triggers a restart of OpenTopoData (which will reload the new config). + ### Config spec diff --git a/example-config.yaml b/example-config.yaml index 6e5014a..6c356e5 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -1,5 +1,5 @@ # An example of a config.yaml file showing all possible options. If no -# config.yaml file exists, opentopodata will load example-config.yaml instead. +# config.yaml file exists, opentopodata will load example-config.yaml instead.. # 400 error will be thrown above this limit. diff --git a/opentopodata/api.py b/opentopodata/api.py index ea8d675..992a62d 100644 --- a/opentopodata/api.py +++ b/opentopodata/api.py @@ -85,10 +85,9 @@ def apply_cors(response): of the access_control_allow_origin config option. """ try: - if _load_config()["access_control_allow_origin"]: - response.headers["access-control-allow-origin"] = _load_config()[ - "access_control_allow_origin" - ] + cors_value = _load_config()["access_control_allow_origin"] + if cors_value: + response.headers["access-control-allow-origin"] = cors_value except config.ConfigError: # If the config isn't loading, allow the request to complete without # CORS so user can see error message. diff --git a/opentopodata/backend.py b/opentopodata/backend.py index 5280b32..94702a2 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -105,7 +105,13 @@ def _get_elevation_from_path(lats, lons, path, interpolation): oob_indices = _validate_points_lie_within_raster( xs, ys, lats, lons, f.bounds, f.res ) - rows, cols = tuple(f.index(xs, ys, op=_noop)) + print(f"{xs=}") + print(f"{ys=}") + tmp = f.index(xs.tolist(), ys.tolist(), op=_noop) + print(f"{tmp=}") + rows, cols = tuple(tmp) + + # rows, cols = tuple(f.index(xs, ys, op=_noop)) # Different versions of rasterio may or may not collapse single # f.index() lookups into scalars. We want to always have an diff --git a/requirements.in b/requirements.in index 637a868..914d5a7 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ black -Flask>=2.2.2 # Some flask 2.0 deprecations got real. +Flask>=2.2.2 # Flask 2.0 deprecations were enforced. flask-caching geographiclib numpy @@ -11,5 +11,6 @@ pytest pytest-cov pytest-timeout PyYAML -rasterio>=1.3.8 # Avoid memory leak https://github.com/ajnisbet/opentopodata/issues/68 +rasterio>=1.3.8,<1.4.0 # 1.3.8+ avoids memory leak https://github.com/ajnisbet/opentopodata/issues/68; 1.4.0 introduces some bugs in rowcol/xy (as of 2024-10-11). requests +watchdog diff --git a/requirements.txt b/requirements.txt index b900d0a..6cb80ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,22 +6,22 @@ # affine==2.4.0 # via rasterio -attrs==23.2.0 +attrs==24.2.0 # via rasterio -black==24.2.0 +black==24.10.0 # via -r requirements.in -blinker==1.7.0 +blinker==1.8.2 # via flask -build==1.0.3 +build==1.2.2.post1 # via pip-tools cachelib==0.9.0 # via flask-caching -certifi==2024.2.2 +certifi==2024.8.30 # via # pyproj # rasterio # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -35,82 +35,84 @@ click-plugins==1.1.1 # via rasterio cligj==0.7.2 # via rasterio -coverage[toml]==7.4.1 +coverage[toml]==7.6.2 # via pytest-cov -flask==3.0.2 +flask==3.0.3 # via # -r requirements.in # flask-caching -flask-caching==2.1.0 +flask-caching==2.3.0 # via -r requirements.in geographiclib==2.0 # via -r requirements.in -idna==3.6 +idna==3.10 # via requests iniconfig==2.0.0 # via pytest -itsdangerous==2.1.2 +itsdangerous==2.2.0 # via flask -jinja2==3.1.3 +jinja2==3.1.4 # via flask -markupsafe==2.1.5 +markupsafe==3.0.1 # via # jinja2 # werkzeug mypy-extensions==1.0.0 # via black -numpy==1.26.4 +numpy==2.1.2 # via # -r requirements.in # rasterio # snuggs -packaging==23.2 +packaging==24.1 # via # black # build # pytest pathspec==0.12.1 # via black -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements.in -platformdirs==4.2.0 +platformdirs==4.3.6 # via black -pluggy==1.4.0 +pluggy==1.5.0 # via pytest polyline==2.0.2 # via -r requirements.in pylibmc==1.6.3 # via -r requirements.in -pyparsing==3.1.1 +pyparsing==3.1.4 # via snuggs -pyproj==3.6.1 +pyproj==3.7.0 # via -r requirements.in -pyproject-hooks==1.0.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.0.1 +pytest==8.3.3 # via # -r requirements.in # pytest-cov # pytest-timeout -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements.in -pytest-timeout==2.2.0 +pytest-timeout==2.3.1 # via -r requirements.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via -r requirements.in -rasterio==1.3.9 +rasterio==1.3.11 # via -r requirements.in -requests==2.31.0 +requests==2.32.3 # via -r requirements.in snuggs==1.4.7 # via rasterio -urllib3==2.2.1 +urllib3==2.2.3 # via requests -werkzeug==3.0.1 +watchdog==5.0.3 + # via -r requirements.in +werkzeug==3.0.4 # via flask -wheel==0.42.0 +wheel==0.44.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: