Add support for hot reloading the config.yaml

* add support for hot reloading the config.yaml

* watchdogs also watches on example-config.yaml

* watchdog remove os.system stuff

* watchdog add requirments.{txt,in}

---------

Co-authored-by: Andrew Nisbet <ajnisbet@users.noreply.github.com>

docker/supervisord.conf

@@ -33,3 +33,10 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 autorestart=false
 
+[program:watchdog]
+command=python /app/docker/watcher.py
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+

docker/watcher.py

@@ -0,0 +1,48 @@
+import logging
+import time
+import warm_cache
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+from pymemcache.client.base import Client
+
+
+#Connect to memcached socket. Since this is hardcoded in the supervisored.conf it shouldn't change that much
+memcacheClient = Client("/tmp/memcached.sock")
+
+
+def reload_config():
+    logging.error("config.yaml changes. Restarting uswgi and memcached to make the changes")
+    #Restart uwsgi
+    #TODO
+    #Flush memcache using pymemcached
+    memcacheClient.flush_all()
+    warm_cache
+
+class Handler(FileSystemEventHandler):
+
+    def on_created(self, event):
+       reload_config()
+    def on_modified(self, event):
+        reload_config()
+
+
+
+
+
+CONFIG_PATH = "/app/config.yaml"
+EXAMPLE_CONFIG_PATH = "/app/example-config.yaml"
+
+if __name__ == "__main__":
+    event_handler = Handler()
+    observer = Observer()
+    observer.schedule(event_handler, CONFIG_PATH)
+    observer.schedule(event_handler, EXAMPLE_CONFIG_PATH)
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    finally:
+        observer.stop()
+        observer.join()

requirements.in

@@ -13,3 +13,5 @@ pytest-timeout
 PyYAML
 rasterio>=1.3.8,<1.4.0  # 1.3.8+ avoids memory leak https://github.com/ajnisbet/opentopodata/issues/68; 1.4.0 introduces some bugs in rowcol/xy (as of 2024-10-11).
 requests
+watchdog # Monitor changes on the config.yaml
+pymemcache # Handle cache flushing without messing

requirements.txt

@@ -81,10 +81,14 @@ polyline==2.0.2
     # via -r requirements.in
 pylibmc==1.6.3
     # via -r requirements.in
+
+pymemcache==4.0.0
+    # via -r requirements.in
 pyparsing==3.1.4
-    # via snuggs
+    # via rasterio
 pyproj==3.7.0
     # via -r requirements.in
+
 pyproject-hooks==1.2.0
     # via
     #   build
@@ -100,14 +104,15 @@ pytest-timeout==2.3.1
     # via -r requirements.in
 pyyaml==6.0.2
     # via -r requirements.in
-rasterio==1.3.11
+rasterio==1.4.1
     # via -r requirements.in
 requests==2.32.3
     # via -r requirements.in
-snuggs==1.4.7
-    # via rasterio
 urllib3==2.2.3
     # via requests
+watchdog==5.0.3
+    # via -r requirements.in
+
 werkzeug==3.0.4
     # via flask
 wheel==0.44.0