Squashed commit of the following:

commit 10539ed
Author: ajnisbet <ajnisbet@gmail.com>
Date:   Fri Oct 11 13:15:54 2024 -0700

    Upgrade version

commit d1fc1a7
Author: ajnisbet <ajnisbet@gmail.com>
Date:   Fri Oct 11 13:11:43 2024 -0700

    Working watchdog

commit 9d51071
Author: Arne Setzer <25772747+arnesetzer@users.noreply.github.com>
Date:   Fri Oct 11 20:58:43 2024 +0200

    Add support for hot reloading the config.yaml

    * add support for hot reloading the config.yaml

    * watchdogs also watches on example-config.yaml

    * watchdog remove os.system stuff

    * watchdog add requirments.{txt,in}

    ---------

    Co-authored-by: Andrew Nisbet <ajnisbet@users.noreply.github.com>

commit fc407a2
Author: ajnisbet <ajnisbet@gmail.com>
Date:   Fri Oct 11 11:56:00 2024 -0700

    Upgrade dependencies

commit 5ec7c25
Author: ajnisbet <ajnisbet@gmail.com>
Date:   Fri Oct 11 11:35:31 2024 -0700

    Docker tweaks

commit 87c9742
Merge: cc7f997 e93ff94
Author: ajnisbet <ajnisbet@gmail.com>
Date:   Fri Oct 11 11:35:00 2024 -0700

    Merge branch 'master' into dev

commit cc7f997
Author: ajnisbet <ajnisbet@gmail.com>
Date:   Tue Feb 20 11:46:21 2024 -0800

    Tidy cors

Makefile

@@ -7,6 +7,12 @@ build:
 build-m1:
 	docker build --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile .
 
+rebuild:
+	docker build --no-cache --tag opentopodata:$(VERSION) --file docker/Dockerfile .
+
+rebuild-m1:
+	docker build --no-cache --tag opentopodata:$(VERSION) --file docker/apple-silicon.Dockerfile .
+
 run:
 	docker run --rm -it --volume "$(shell pwd)/data:/app/data:ro" -p 5000:5000 opentopodata:$(VERSION) 
 
@@ -23,10 +29,10 @@ run-local:
 	FLASK_APP=opentopodata/api.py FLASK_DEBUG=1 flask run --port 5000
 
 black:
-	black --target-version py39 tests opentopodata
+	black --target-version py311 tests opentopodata docker
 
 black-check:
-	docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py39 tests opentopodata
+	docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py311 tests opentopodata
 
 update-requirements: build
 	# pip-compile gets confused if there's already a requirements.txt file, and

VERSION

@@ -1 +1 @@
-1.9.0
+1.10.0

docker/Dockerfile

@@ -1,20 +1,22 @@
 # Container for packages that need to be built from source but have massive dev dependencies.
-FROM python:3.11.8-slim-bookworm as builder
+FROM python:3.11.10-slim-bookworm as builder
 RUN set -e && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
         gcc \
         python3.11-dev
 
 RUN pip config set global.disable-pip-version-check true && \
-    pip wheel --wheel-dir=/root/wheels uwsgi==2.0.24 && \
-    pip wheel --wheel-dir=/root/wheels regex==2023.12.25
+    pip wheel --wheel-dir=/root/wheels uwsgi==2.0.27 && \
+    pip wheel --wheel-dir=/root/wheels regex==2024.9.11 
 
 # The actual container.
-FROM python:3.11.8-slim-bookworm
+FROM python:3.11.10-slim-bookworm
 RUN set -e && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
+        inotify-tools \
+        nano \
         nginx \
         memcached \
         supervisor && \

docker/config_watcher.py

@@ -0,0 +1,110 @@
+import logging
+import time
+from pathlib import Path
+import subprocess
+import sys
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+
+# Paths.
+CONFIG_DIR = Path("/app/")
+CONFIG_PATH = Path("/app/config.yaml")
+EXAMPLE_CONFIG_PATH = Path("/app/example-config.yaml")
+
+# Debouncing: once the config has been reloaded, any queued unprocessed events should be discarded.
+LAST_INVOCATION_TIME = time.time()
+
+
+# Logger setup.
+logger = logging.getLogger("configwatcher")
+LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
+formatter = logging.Formatter(LOG_FORMAT)
+logger.setLevel(logging.INFO)
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+
+def run_cmd(cmd, shell=False):
+    r = subprocess.run(cmd, shell=shell, capture_output=True)
+    is_error = r.returncode != 0
+    stdout = r.stdout.decode("utf-8")
+    if is_error:
+        logger.error(f"Error running command, returncode: {r.returncode}")
+        logger.error("cmd:")
+        logger.error(" ".join(cmd))
+        if r.stdout:
+            logger.error("stdout:")
+            logger.error(stdout)
+        if r.stderr:
+            logger.error("stderr:")
+            logger.error(r.stderr.decode("utf-8"))
+        raise ValueError
+    return stdout
+
+
+def reload_config():
+    global LAST_INVOCATION_TIME
+    LAST_INVOCATION_TIME = time.time()
+    logger.info("Restarting OTD due to config change.")
+    run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "stop", "uwsgi"])
+    run_cmd(
+        ["supervisorctl", "-c", "/app/docker/supervisord.conf", "restart", "memcached"]
+    )
+    run_cmd(["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "uwsgi"])
+    run_cmd(
+        ["supervisorctl", "-c", "/app/docker/supervisord.conf", "start", "warm_cache"]
+    )
+    LAST_INVOCATION_TIME = time.time()
+    logger.info("Restarted OTD due to config change.")
+
+
+class Handler(FileSystemEventHandler):
+
+    def on_any_event(self, event):
+        watch_paths_str = [
+            EXAMPLE_CONFIG_PATH.as_posix(),
+            CONFIG_PATH.as_posix(),
+        ]
+
+        # Filter unwanted events.
+        if event.event_type not in {"modified", "created"}:
+            logger.debug(f"Dropping event with type {event.event_type=}")
+            return
+        if event.is_directory:
+            logger.debug(f"Dropping dir event")
+            return
+        if event.src_path not in watch_paths_str:
+            logger.debug(f"Dropping event with path {event.src_path=}")
+            return
+        if not Path(event.src_path).exists():
+            logger.debug(f"Dropping event for nonexistent path {event.src_path=}")
+            return
+
+        # Debouncing.
+        mtime = Path(event.src_path).lstat().st_mtime
+        if mtime < LAST_INVOCATION_TIME:
+            msg = f"Dropping event for file that hasn't been modified since the last run. {event.src_path=}"
+            logger.debug(msg)
+            return
+
+        logger.debug(f"Dispatching event on {event.src_path=}")
+        reload_config()
+
+
+if __name__ == "__main__":
+
+    event_handler = Handler()
+    observer = Observer()
+    observer.schedule(event_handler, CONFIG_DIR, recursive=False)
+    observer.start()
+
+    try:
+        while True:
+            time.sleep(1)
+    finally:
+        observer.stop()
+        observer.join()

docker/supervisord.conf

@@ -2,6 +2,19 @@
 nodaemon=true
 user=root
 
+
+# Supervisorctl config/
+[unix_http_server]
+file=/var/run/supervisor.sock
+chmod=0700
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock
+
+
+# OTD services.
+
 [program:uwsgi]
 command=/usr/local/bin/uwsgi --ini /app/docker/uwsgi.ini --processes %(ENV_N_UWSGI_THREADS)s
 stdout_logfile=/dev/stdout
@@ -33,3 +46,9 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 autorestart=false
 
+[program:watch_config]
+command=python /app/docker/config_watcher.py
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docker/warm_cache.py

@@ -34,5 +34,7 @@
         break
 
     else:
-        logging.error("Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working.")
+        logging.error(
+            "Timeout while trying to pre-populate the cache. This probably means that Open Topo Data isn't working."
+        )
         sys.exit(1)

docs/changelog.md

@@ -2,6 +2,10 @@
 
 This is a list of changes to Open Topo Data between each release.
 
+## Version 1.10.0 (11 Oct 2024)
+* Minior dependency upgrades
+* Auto-reloading of config files without restarting docker ([#100](https://github.com/ajnisbet/opentopodata/pull/100) thanks [@arnesetzer](https://github.com/arnesetzer)!)
+
 
 ## Version 1.9.0 (19 Feb 2024)
 * Dependency upgrades, including python to 3.11 and rasterio to 1.3.9

docs/server.md

@@ -97,6 +97,8 @@ opentopodata
 
 which would expose `localhost:5000/v1/etopo1` and `localhost:5000/v1/srtm90m`.
 
+Mofifying the config file triggers a restart of OpenTopoData (which will reload the new config).
+
 
 ### Config spec
 

example-config.yaml

@@ -1,5 +1,5 @@
 # An example of a config.yaml file showing all possible options. If no
-# config.yaml file exists, opentopodata will load example-config.yaml instead.
+# config.yaml file exists, opentopodata will load example-config.yaml instead..
 
 
 # 400 error will be thrown above this limit.

opentopodata/api.py

@@ -85,10 +85,9 @@ def apply_cors(response):
     of the access_control_allow_origin config option.
     """
     try:
-        if _load_config()["access_control_allow_origin"]:
-            response.headers["access-control-allow-origin"] = _load_config()[
-                "access_control_allow_origin"
-            ]
+        cors_value = _load_config()["access_control_allow_origin"]
+        if cors_value:
+            response.headers["access-control-allow-origin"] = cors_value
     except config.ConfigError:
         # If the config isn't loading, allow the request to complete without
         # CORS so user can see error message.

opentopodata/backend.py

@@ -105,7 +105,13 @@ def _get_elevation_from_path(lats, lons, path, interpolation):
             oob_indices = _validate_points_lie_within_raster(
                 xs, ys, lats, lons, f.bounds, f.res
             )
-            rows, cols = tuple(f.index(xs, ys, op=_noop))
+            print(f"{xs=}")
+            print(f"{ys=}")
+            tmp = f.index(xs.tolist(), ys.tolist(), op=_noop)
+            print(f"{tmp=}")
+            rows, cols = tuple(tmp)
+
+            # rows, cols = tuple(f.index(xs, ys, op=_noop))
 
             # Different versions of rasterio may or may not collapse single
             # f.index() lookups into scalars. We want to always have an

requirements.in

@@ -1,5 +1,5 @@
 black
-Flask>=2.2.2  # Some flask 2.0 deprecations got real.
+Flask>=2.2.2  # Flask 2.0 deprecations were enforced.
 flask-caching
 geographiclib
 numpy
@@ -11,5 +11,6 @@ pytest
 pytest-cov
 pytest-timeout
 PyYAML
-rasterio>=1.3.8  # Avoid memory leak https://github.com/ajnisbet/opentopodata/issues/68
+rasterio>=1.3.8,<1.4.0  # 1.3.8+ avoids memory leak https://github.com/ajnisbet/opentopodata/issues/68; 1.4.0 introduces some bugs in rowcol/xy (as of 2024-10-11).
 requests
+watchdog