From ceaac0281f9f67a47e4cc1265dfaa09e9408a738 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Sun, 2 Apr 2023 13:18:51 -0600 Subject: [PATCH 01/20] Document need to restart memcache when not using docker Fixes https://github.com/ajnisbet/opentopodata/issues/72 --- docs/notes/running-without-docker.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/notes/running-without-docker.md b/docs/notes/running-without-docker.md index c305beb..bc904ad 100644 --- a/docs/notes/running-without-docker.md +++ b/docs/notes/running-without-docker.md @@ -17,13 +17,13 @@ git clone https://github.com/ajnisbet/opentopodata.git cd opentopodata ``` -Install system dependencies +Install system dependencies (if you're not using Debian 10, install whatever python3.X-dev matches your installed python)L ```bash apt install gcc python3.7-dev python3-pip ``` -Debian 10 comes with an old version of pip, it needs to be updated: +Debian 10 comes with an old version of pip, it needs to be updated so we can install wheels: ```bash pip3 install --upgrade pip @@ -38,7 +38,7 @@ cat requirements.txt | grep pyproj and install that pinned version ```bash -pip3 install pyproj==3.0.0.post1 +pip3 install pyproj==3.4.1 ``` then the remaining python packages can be installed: @@ -133,4 +133,9 @@ Then manage Open Topo Data with systemctl daemon-reload systemctl enable opentopodata.service systemctl start opentopodata.service -``` \ No newline at end of file +``` + +!!! warning "Warning" + Opentopodata caches `config.yaml` in two places: memcache and uwsgi. + + If you update the config file (to eg add a new dataset) you'll need to restart memcached **first**, then opentopodata. \ No newline at end of file From 26c84faccc1614b628f718d2cf06657d6f92e220 Mon Sep 17 00:00:00 2001 From: Benjamin Meier Date: Thu, 6 Apr 2023 17:46:22 +0200 Subject: [PATCH 02/20] Use mmap to cache file for subsequent reads. (#74) * Upgrade dependencies * Revert "Upgrade dependencies" This reverts commit 3d15da9e16118857fbb9dfa59e32fecdab709583. * Update README.md * Use mmap to cache file for subsequent reads. --------- Co-authored-by: ajnisbet Co-authored-by: Andrew Nisbet --- opentopodata/backend.py | 117 ++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/opentopodata/backend.py b/opentopodata/backend.py index 5280b32..aa2605f 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -1,4 +1,5 @@ import collections +import mmap from rasterio.enums import Resampling import numpy as np @@ -86,63 +87,65 @@ def _get_elevation_from_path(lats, lons, path, interpolation): lats = np.asarray(lats) try: - with rasterio.open(path) as f: - if f.crs is None: - msg = "Dataset has no coordinate reference system." - msg += f" Check the file '{path}' is a geo raster." - msg += " Otherwise you'll have to add the crs manually with a tool like gdaltranslate." - raise InputError(msg) - - try: - if f.crs.is_epsg_code: - xs, ys = utils.reproject_latlons(lats, lons, epsg=f.crs.to_epsg()) - else: - xs, ys = utils.reproject_latlons(lats, lons, wkt=f.crs.to_wkt()) - except ValueError: - raise InputError("Unable to transform latlons to dataset projection.") - - # Check bounds. - oob_indices = _validate_points_lie_within_raster( - xs, ys, lats, lons, f.bounds, f.res - ) - rows, cols = tuple(f.index(xs, ys, op=_noop)) - - # Different versions of rasterio may or may not collapse single - # f.index() lookups into scalars. We want to always have an - # array. - rows = np.atleast_1d(rows) - cols = np.atleast_1d(cols) - - # Offset by 0.5 to convert from center coords (provided by - # f.index) to ul coords (expected by f.read). - rows = rows - 0.5 - cols = cols - 0.5 - - # Because of floating point precision, indices may slightly exceed - # array bounds. Because we've checked the locations are within the - # file bounds, it's safe to clip to the array shape. - rows = rows.clip(0, f.height - 1) - cols = cols.clip(0, f.width - 1) - - # Read the locations, using a 1x1 window. The `masked` kwarg makes - # rasterio replace NODATA values with np.nan. The `boundless` kwarg - # forces the windowed elevation to be a 1x1 array, even when it all - # values are NODATA. - for i, (row, col) in enumerate(zip(rows, cols)): - if i in oob_indices: - z_all.append(None) - continue - window = rasterio.windows.Window(col, row, 1, 1) - z_array = f.read( - indexes=1, - window=window, - resampling=interpolation, - out_dtype=float, - boundless=True, - masked=True, - ) - z = np.ma.filled(z_array, np.nan)[0][0] - z_all.append(z) + with open(path, 'rb') as bf: + with mmap.mmap( bf.fileno(), length=0, access=mmap.ACCESS_READ ) as mmap_obj: + with rasterio.open(mmap_obj) as f: + if f.crs is None: + msg = "Dataset has no coordinate reference system." + msg += f" Check the file '{path}' is a geo raster." + msg += " Otherwise you'll have to add the crs manually with a tool like gdaltranslate." + raise InputError(msg) + + try: + if f.crs.is_epsg_code: + xs, ys = utils.reproject_latlons(lats, lons, epsg=f.crs.to_epsg()) + else: + xs, ys = utils.reproject_latlons(lats, lons, wkt=f.crs.to_wkt()) + except ValueError: + raise InputError("Unable to transform latlons to dataset projection.") + + # Check bounds. + oob_indices = _validate_points_lie_within_raster( + xs, ys, lats, lons, f.bounds, f.res + ) + rows, cols = tuple(f.index(xs, ys, op=_noop)) + + # Different versions of rasterio may or may not collapse single + # f.index() lookups into scalars. We want to always have an + # array. + rows = np.atleast_1d(rows) + cols = np.atleast_1d(cols) + + # Offset by 0.5 to convert from center coords (provided by + # f.index) to ul coords (expected by f.read). + rows = rows - 0.5 + cols = cols - 0.5 + + # Because of floating point precision, indices may slightly exceed + # array bounds. Because we've checked the locations are within the + # file bounds, it's safe to clip to the array shape. + rows = rows.clip(0, f.height - 1) + cols = cols.clip(0, f.width - 1) + + # Read the locations, using a 1x1 window. The `masked` kwarg makes + # rasterio replace NODATA values with np.nan. The `boundless` kwarg + # forces the windowed elevation to be a 1x1 array, even when it all + # values are NODATA. + for i, (row, col) in enumerate(zip(rows, cols)): + if i in oob_indices: + z_all.append(None) + continue + window = rasterio.windows.Window(col, row, 1, 1) + z_array = f.read( + indexes=1, + window=window, + resampling=interpolation, + out_dtype=float, + boundless=True, + masked=True, + ) + z = np.ma.filled(z_array, np.nan)[0][0] + z_all.append(z) # Depending on the file format, when rasterio finds an invalid projection # of file, it might load it with a None crs, or it might throw an error. From c02174997e2a4b24804a89c2eb99e79009c002fa Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Thu, 6 Apr 2023 08:55:09 -0700 Subject: [PATCH 03/20] Black --- opentopodata/backend.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/opentopodata/backend.py b/opentopodata/backend.py index aa2605f..afce035 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -87,8 +87,8 @@ def _get_elevation_from_path(lats, lons, path, interpolation): lats = np.asarray(lats) try: - with open(path, 'rb') as bf: - with mmap.mmap( bf.fileno(), length=0, access=mmap.ACCESS_READ ) as mmap_obj: + with open(path, "rb") as bf: + with mmap.mmap(bf.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: with rasterio.open(mmap_obj) as f: if f.crs is None: msg = "Dataset has no coordinate reference system." @@ -98,11 +98,17 @@ def _get_elevation_from_path(lats, lons, path, interpolation): try: if f.crs.is_epsg_code: - xs, ys = utils.reproject_latlons(lats, lons, epsg=f.crs.to_epsg()) + xs, ys = utils.reproject_latlons( + lats, lons, epsg=f.crs.to_epsg() + ) else: - xs, ys = utils.reproject_latlons(lats, lons, wkt=f.crs.to_wkt()) + xs, ys = utils.reproject_latlons( + lats, lons, wkt=f.crs.to_wkt() + ) except ValueError: - raise InputError("Unable to transform latlons to dataset projection.") + raise InputError( + "Unable to transform latlons to dataset projection." + ) # Check bounds. oob_indices = _validate_points_lie_within_raster( From 830321094b20156dd3fafba6138d195043cb57dd Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Fri, 18 Aug 2023 15:00:58 -0700 Subject: [PATCH 04/20] Add add mmap (but some gdal stuff is broken now...) --- Makefile | 2 +- docs/changelog.md | 2 + example-config.yaml | 6 ++ opentopodata/backend.py | 128 ++++++++++++++++++++-------------------- opentopodata/config.py | 2 + requirements.in | 2 +- requirements.txt | 62 +++++++++---------- tests/test_backend.py | 10 ++++ 8 files changed, 119 insertions(+), 95 deletions(-) diff --git a/Makefile b/Makefile index 0b9b1a6..b8a715f 100644 --- a/Makefile +++ b/Makefile @@ -32,5 +32,5 @@ update-requirements: build # pip-compile gets confused if there's already a requirements.txt file, and # it can't be deleted without breaking the docker mount. So instead do the # compiling in /tmp. Should run test suite afterwards. - docker run --rm -v $(shell pwd)/requirements.txt:/app/requirements.txt -w /tmp opentopodata:$(VERSION) /bin/bash -c "cp /app/requirements.in .; pip-compile requirements.in; cp requirements.txt /app/requirements.txt" + docker run --rm -v $(shell pwd)/requirements.txt:/app/requirements.txt -w /tmp opentopodata:$(VERSION) /bin/bash -c "cp /app/requirements.in .; pip-compile requirements.in --resolver backtracking; cp requirements.txt /app/requirements.txt" diff --git a/docs/changelog.md b/docs/changelog.md index 44902a7..b0d8cf9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,8 @@ This is a list of changes to Open Topo Data between each release. +## Version 1.8.4 (18 Aug 2023) + ## Version 1.8.3 (7 Feb 2023) * Fix memory leak ([#68](https://github.com/ajnisbet/opentopodata/issues/68)) diff --git a/example-config.yaml b/example-config.yaml index 4df114c..4262c25 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -5,10 +5,16 @@ # 400 error will be thrown above this limit. max_locations_per_request: 100 + # CORS header. Should be null for no CORS, '*' for all domains, or a url with # protocol, domain, and port ('https://api.example.com/'). Default is null. access_control_allow_origin: "*" + +# Use mmap to cache files for faster repeated reads on slow networked filesystems. +# See https://github.com/ajnisbet/opentopodata/pull/74 +read_with_mmap: false + datasets: # A small testing dataset is included in the repo. diff --git a/opentopodata/backend.py b/opentopodata/backend.py index afce035..04b2782 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -70,7 +70,7 @@ def _validate_points_lie_within_raster(xs, ys, lats, lons, bounds, res): return sorted(oob_indices) -def _get_elevation_from_path(lats, lons, path, interpolation): +def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): """Read values at locations in a raster. Args: @@ -87,71 +87,70 @@ def _get_elevation_from_path(lats, lons, path, interpolation): lats = np.asarray(lats) try: - with open(path, "rb") as bf: - with mmap.mmap(bf.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: - with rasterio.open(mmap_obj) as f: - if f.crs is None: - msg = "Dataset has no coordinate reference system." - msg += f" Check the file '{path}' is a geo raster." - msg += " Otherwise you'll have to add the crs manually with a tool like gdaltranslate." - raise InputError(msg) - - try: - if f.crs.is_epsg_code: - xs, ys = utils.reproject_latlons( - lats, lons, epsg=f.crs.to_epsg() - ) - else: - xs, ys = utils.reproject_latlons( - lats, lons, wkt=f.crs.to_wkt() - ) - except ValueError: - raise InputError( - "Unable to transform latlons to dataset projection." + with open(path, "rb") as dataset: + if use_mmap: + dataset = mmap.mmap(dataset.fileno()) + with rasterio.open(dataset) as f: + if f.crs is None: + msg = "Dataset has no coordinate reference system." + msg += f" Check the file '{path}' is a geo raster." + msg += " Otherwise you'll have to add the crs manually with a tool like gdaltranslate." + raise InputError(msg) + + try: + if f.crs.is_epsg_code: + xs, ys = utils.reproject_latlons( + lats, lons, epsg=f.crs.to_epsg() ) + else: + xs, ys = utils.reproject_latlons(lats, lons, wkt=f.crs.to_wkt()) + except ValueError: + raise InputError( + "Unable to transform latlons to dataset projection." + ) - # Check bounds. - oob_indices = _validate_points_lie_within_raster( - xs, ys, lats, lons, f.bounds, f.res + # Check bounds. + oob_indices = _validate_points_lie_within_raster( + xs, ys, lats, lons, f.bounds, f.res + ) + rows, cols = tuple(f.index(xs, ys, op=_noop)) + + # Different versions of rasterio may or may not collapse single + # f.index() lookups into scalars. We want to always have an + # array. + rows = np.atleast_1d(rows) + cols = np.atleast_1d(cols) + + # Offset by 0.5 to convert from center coords (provided by + # f.index) to ul coords (expected by f.read). + rows = rows - 0.5 + cols = cols - 0.5 + + # Because of floating point precision, indices may slightly exceed + # array bounds. Because we've checked the locations are within the + # file bounds, it's safe to clip to the array shape. + rows = rows.clip(0, f.height - 1) + cols = cols.clip(0, f.width - 1) + + # Read the locations, using a 1x1 window. The `masked` kwarg makes + # rasterio replace NODATA values with np.nan. The `boundless` kwarg + # forces the windowed elevation to be a 1x1 array, even when it all + # values are NODATA. + for i, (row, col) in enumerate(zip(rows, cols)): + if i in oob_indices: + z_all.append(None) + continue + window = rasterio.windows.Window(col, row, 1, 1) + z_array = f.read( + indexes=1, + window=window, + resampling=interpolation, + out_dtype=float, + boundless=True, + masked=True, ) - rows, cols = tuple(f.index(xs, ys, op=_noop)) - - # Different versions of rasterio may or may not collapse single - # f.index() lookups into scalars. We want to always have an - # array. - rows = np.atleast_1d(rows) - cols = np.atleast_1d(cols) - - # Offset by 0.5 to convert from center coords (provided by - # f.index) to ul coords (expected by f.read). - rows = rows - 0.5 - cols = cols - 0.5 - - # Because of floating point precision, indices may slightly exceed - # array bounds. Because we've checked the locations are within the - # file bounds, it's safe to clip to the array shape. - rows = rows.clip(0, f.height - 1) - cols = cols.clip(0, f.width - 1) - - # Read the locations, using a 1x1 window. The `masked` kwarg makes - # rasterio replace NODATA values with np.nan. The `boundless` kwarg - # forces the windowed elevation to be a 1x1 array, even when it all - # values are NODATA. - for i, (row, col) in enumerate(zip(rows, cols)): - if i in oob_indices: - z_all.append(None) - continue - window = rasterio.windows.Window(col, row, 1, 1) - z_array = f.read( - indexes=1, - window=window, - resampling=interpolation, - out_dtype=float, - boundless=True, - masked=True, - ) - z = np.ma.filled(z_array, np.nan)[0][0] - z_all.append(z) + z = np.ma.filled(z_array, np.nan)[0][0] + z_all.append(z) # Depending on the file format, when rasterio finds an invalid projection # of file, it might load it with a None crs, or it might throw an error. @@ -162,6 +161,9 @@ def _get_elevation_from_path(lats, lons, path, interpolation): msg += " and that the file is not corrupt." raise InputError(msg) raise e + finally: + if isinstance(dataset, mmap.mmap): + dataset.close() return z_all diff --git a/opentopodata/config.py b/opentopodata/config.py index 9ccf99d..ac94fad 100644 --- a/opentopodata/config.py +++ b/opentopodata/config.py @@ -21,6 +21,7 @@ "dataset.filename_tile_size": 1, "dataset.filename_epsg": utils.WGS84_LATLON_EPSG, "access_control_allow_origin": None, + "read_with_mmap": False, } @@ -147,6 +148,7 @@ def load_config(): config["access_control_allow_origin"] = config.get( "access_control_allow_origin", DEFAULTS["access_control_allow_origin"] ) + config["read_with_mmap"] = config.get("read_with_mmap", DEFAULTS["read_with_mmap"]) # Validate CORS. Must have protocol, domain, and optionally port. _validate_cors(config["access_control_allow_origin"]) diff --git a/requirements.in b/requirements.in index c0312fc..8cf5e93 100644 --- a/requirements.in +++ b/requirements.in @@ -10,5 +10,5 @@ pyproj pytest pytest-cov PyYAML -rasterio==1.2.10 +rasterio<1.3.0 requests diff --git a/requirements.txt b/requirements.txt index 7b614ae..153eedd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,24 +6,24 @@ # affine==2.4.0 # via rasterio -attrs==22.2.0 - # via - # pytest - # rasterio -black==23.1.0 +attrs==23.1.0 + # via rasterio +black==23.7.0 # via -r requirements.in +blinker==1.6.2 + # via flask build==0.10.0 # via pip-tools cachelib==0.9.0 # via flask-caching -certifi==2022.12.7 +certifi==2023.7.22 # via # pyproj # rasterio # requests -charset-normalizer==3.0.1 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.7 # via # black # click-plugins @@ -35,11 +35,11 @@ click-plugins==1.1.1 # via rasterio cligj==0.7.2 # via rasterio -coverage[toml]==7.1.0 +coverage[toml]==7.3.0 # via pytest-cov -exceptiongroup==1.1.0 +exceptiongroup==1.1.3 # via pytest -flask==2.2.2 +flask==2.3.2 # via # -r requirements.in # flask-caching @@ -49,7 +49,7 @@ geographiclib==2.0 # via -r requirements.in idna==3.4 # via requests -importlib-metadata==6.0.0 +importlib-metadata==6.8.0 # via flask iniconfig==2.0.0 # via pytest @@ -57,51 +57,51 @@ itsdangerous==2.1.2 # via flask jinja2==3.1.2 # via flask -markupsafe==2.1.2 +markupsafe==2.1.3 # via # jinja2 # werkzeug mypy-extensions==1.0.0 # via black -numpy==1.24.2 +numpy==1.25.2 # via # -r requirements.in # rasterio # snuggs -packaging==23.0 +packaging==23.1 # via # black # build # pytest -pathspec==0.11.0 +pathspec==0.11.2 # via black -pip-tools==6.12.2 +pip-tools==7.3.0 # via -r requirements.in -platformdirs==3.0.0 +platformdirs==3.10.0 # via black -pluggy==1.0.0 +pluggy==1.2.0 # via pytest polyline==2.0.0 # via -r requirements.in pylibmc==1.6.3 # via -r requirements.in -pyparsing==3.0.9 +pyparsing==3.1.1 # via snuggs -pyproj==3.4.1 +pyproj==3.6.0 # via -r requirements.in pyproject-hooks==1.0.0 # via build -pytest==7.2.1 +pytest==7.4.0 # via # -r requirements.in # pytest-cov -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements.in -pyyaml==6.0 +pyyaml==6.0.1 # via -r requirements.in rasterio==1.2.10 # via -r requirements.in -requests==2.28.2 +requests==2.31.0 # via -r requirements.in snuggs==1.4.7 # via rasterio @@ -110,16 +110,18 @@ tomli==2.0.1 # black # build # coverage + # pip-tools + # pyproject-hooks # pytest -typing-extensions==4.4.0 +typing-extensions==4.7.1 # via black -urllib3==1.26.14 +urllib3==2.0.4 # via requests -werkzeug==2.2.2 +werkzeug==2.3.7 # via flask -wheel==0.38.4 +wheel==0.41.1 # via pip-tools -zipp==3.12.1 +zipp==3.16.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/tests/test_backend.py b/tests/test_backend.py index b6e39d1..db6129a 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -154,6 +154,16 @@ def test_bilinear_interpolation(self): 0.4, 0.3, self.geotiff_z[:2, :2] ) + def test_mmap(self): + lats = [89.6] + lons = [-179.7] + z = backend._get_elevation_from_path( + lats, lons, ETOPO1_GEOTIFF_PATH, "bilinear", use_mmap=True + ) + assert pytest.approx(z[0]) == self._interp_bilinear( + 0.4, 0.3, self.geotiff_z[:2, :2] + ) + def test_none_outside_dataset(self): lats = [0, 0, -90.1, 90.1] lons = [-180.1, 180.1, 0, 0] From f5f9cb8cfff92524eefc08a68739a70b4aa5e392 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Tue, 22 Aug 2023 10:58:32 -0700 Subject: [PATCH 05/20] Last try at implementing mmap --- opentopodata/backend.py | 129 ++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/opentopodata/backend.py b/opentopodata/backend.py index 04b2782..8a5ace5 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -86,71 +86,72 @@ def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): lons = np.asarray(lons) lats = np.asarray(lats) + f_mmap = None + dataset = None + try: - with open(path, "rb") as dataset: - if use_mmap: - dataset = mmap.mmap(dataset.fileno()) - with rasterio.open(dataset) as f: - if f.crs is None: - msg = "Dataset has no coordinate reference system." - msg += f" Check the file '{path}' is a geo raster." - msg += " Otherwise you'll have to add the crs manually with a tool like gdaltranslate." - raise InputError(msg) - - try: - if f.crs.is_epsg_code: - xs, ys = utils.reproject_latlons( - lats, lons, epsg=f.crs.to_epsg() - ) - else: - xs, ys = utils.reproject_latlons(lats, lons, wkt=f.crs.to_wkt()) - except ValueError: - raise InputError( - "Unable to transform latlons to dataset projection." - ) - - # Check bounds. - oob_indices = _validate_points_lie_within_raster( - xs, ys, lats, lons, f.bounds, f.res + if use_mmap: + f_mmap = open(path, "rb") + dataset = mmap.mmap(f_mmap.fileno(), length=0, access=mmap.ACCESS_READ) + else: + dataset = path + with rasterio.open(dataset) as f: + if f.crs is None: + msg = "Dataset has no coordinate reference system." + msg += f" Check the file '{path}' is a geo raster." + msg += " Otherwise you'll have to add the crs manually with a tool like gdaltranslate." + raise InputError(msg) + + try: + if f.crs.is_epsg_code: + xs, ys = utils.reproject_latlons(lats, lons, epsg=f.crs.to_epsg()) + else: + xs, ys = utils.reproject_latlons(lats, lons, wkt=f.crs.to_wkt()) + except ValueError: + raise InputError("Unable to transform latlons to dataset projection.") + + # Check bounds. + oob_indices = _validate_points_lie_within_raster( + xs, ys, lats, lons, f.bounds, f.res + ) + rows, cols = tuple(f.index(xs, ys, op=_noop)) + + # Different versions of rasterio may or may not collapse single + # f.index() lookups into scalars. We want to always have an + # array. + rows = np.atleast_1d(rows) + cols = np.atleast_1d(cols) + + # Offset by 0.5 to convert from center coords (provided by + # f.index) to ul coords (expected by f.read). + rows = rows - 0.5 + cols = cols - 0.5 + + # Because of floating point precision, indices may slightly exceed + # array bounds. Because we've checked the locations are within the + # file bounds, it's safe to clip to the array shape. + rows = rows.clip(0, f.height - 1) + cols = cols.clip(0, f.width - 1) + + # Read the locations, using a 1x1 window. The `masked` kwarg makes + # rasterio replace NODATA values with np.nan. The `boundless` kwarg + # forces the windowed elevation to be a 1x1 array, even when it all + # values are NODATA. + for i, (row, col) in enumerate(zip(rows, cols)): + if i in oob_indices: + z_all.append(None) + continue + window = rasterio.windows.Window(col, row, 1, 1) + z_array = f.read( + indexes=1, + window=window, + resampling=interpolation, + out_dtype=float, + boundless=True, + masked=True, ) - rows, cols = tuple(f.index(xs, ys, op=_noop)) - - # Different versions of rasterio may or may not collapse single - # f.index() lookups into scalars. We want to always have an - # array. - rows = np.atleast_1d(rows) - cols = np.atleast_1d(cols) - - # Offset by 0.5 to convert from center coords (provided by - # f.index) to ul coords (expected by f.read). - rows = rows - 0.5 - cols = cols - 0.5 - - # Because of floating point precision, indices may slightly exceed - # array bounds. Because we've checked the locations are within the - # file bounds, it's safe to clip to the array shape. - rows = rows.clip(0, f.height - 1) - cols = cols.clip(0, f.width - 1) - - # Read the locations, using a 1x1 window. The `masked` kwarg makes - # rasterio replace NODATA values with np.nan. The `boundless` kwarg - # forces the windowed elevation to be a 1x1 array, even when it all - # values are NODATA. - for i, (row, col) in enumerate(zip(rows, cols)): - if i in oob_indices: - z_all.append(None) - continue - window = rasterio.windows.Window(col, row, 1, 1) - z_array = f.read( - indexes=1, - window=window, - resampling=interpolation, - out_dtype=float, - boundless=True, - masked=True, - ) - z = np.ma.filled(z_array, np.nan)[0][0] - z_all.append(z) + z = np.ma.filled(z_array, np.nan)[0][0] + z_all.append(z) # Depending on the file format, when rasterio finds an invalid projection # of file, it might load it with a None crs, or it might throw an error. @@ -164,6 +165,8 @@ def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): finally: if isinstance(dataset, mmap.mmap): dataset.close() + if f_mmap: + f_mmap.close() return z_all From 37bb1bfb5d19d2d37ebcd5de998ee2b3a235d101 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Tue, 22 Aug 2023 11:05:35 -0700 Subject: [PATCH 06/20] Dependency upgrade --- Makefile | 6 +++--- docker/apple-silicon.Dockerfile | 5 +++-- example-config.yaml | 4 ---- opentopodata/backend.py | 16 +--------------- opentopodata/config.py | 2 -- requirements.in | 1 + requirements.txt | 7 +++++-- 7 files changed, 13 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index b8a715f..12db43f 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ daemon: docker run --rm -itd --volume "$(shell pwd)/data:/app/data:ro" -p 5000:5000 opentopodata:$(VERSION) test: build black-check - docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html + docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) python -m pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html --timeout=10 test-m1: build-m1 black-check - docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html + docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) python -m pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html --timeout=10 run-local: FLASK_APP=opentopodata/api.py FLASK_DEBUG=1 flask run --port 5000 @@ -26,7 +26,7 @@ black: black --target-version py39 tests opentopodata black-check: - docker run --rm opentopodata:$(VERSION) black --check --target-version py39 tests opentopodata + docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py39 tests opentopodata update-requirements: build # pip-compile gets confused if there's already a requirements.txt file, and diff --git a/docker/apple-silicon.Dockerfile b/docker/apple-silicon.Dockerfile index 9c21ac3..9f81059 100644 --- a/docker/apple-silicon.Dockerfile +++ b/docker/apple-silicon.Dockerfile @@ -5,7 +5,8 @@ # It works just the same as the main image, but is much larger and slower to # build. -FROM osgeo/gdal:ubuntu-full-3.5.2 +FROM ghcr.io/osgeo/gdal:ubuntu-full-3.6.4 +RUN python --version RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -16,7 +17,7 @@ RUN set -e && \ g++ \ supervisor \ libmemcached-dev \ - python3.8-dev && \ + python3.10-dev && \ rm -rf /var/lib/apt/lists/* COPY requirements.txt /app/requirements.txt diff --git a/example-config.yaml b/example-config.yaml index 4262c25..6e5014a 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -11,10 +11,6 @@ max_locations_per_request: 100 access_control_allow_origin: "*" -# Use mmap to cache files for faster repeated reads on slow networked filesystems. -# See https://github.com/ajnisbet/opentopodata/pull/74 -read_with_mmap: false - datasets: # A small testing dataset is included in the repo. diff --git a/opentopodata/backend.py b/opentopodata/backend.py index 8a5ace5..d5a6c8f 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -1,5 +1,4 @@ import collections -import mmap from rasterio.enums import Resampling import numpy as np @@ -86,16 +85,8 @@ def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): lons = np.asarray(lons) lats = np.asarray(lats) - f_mmap = None - dataset = None - try: - if use_mmap: - f_mmap = open(path, "rb") - dataset = mmap.mmap(f_mmap.fileno(), length=0, access=mmap.ACCESS_READ) - else: - dataset = path - with rasterio.open(dataset) as f: + with rasterio.open(path) as f: if f.crs is None: msg = "Dataset has no coordinate reference system." msg += f" Check the file '{path}' is a geo raster." @@ -162,11 +153,6 @@ def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): msg += " and that the file is not corrupt." raise InputError(msg) raise e - finally: - if isinstance(dataset, mmap.mmap): - dataset.close() - if f_mmap: - f_mmap.close() return z_all diff --git a/opentopodata/config.py b/opentopodata/config.py index ac94fad..9ccf99d 100644 --- a/opentopodata/config.py +++ b/opentopodata/config.py @@ -21,7 +21,6 @@ "dataset.filename_tile_size": 1, "dataset.filename_epsg": utils.WGS84_LATLON_EPSG, "access_control_allow_origin": None, - "read_with_mmap": False, } @@ -148,7 +147,6 @@ def load_config(): config["access_control_allow_origin"] = config.get( "access_control_allow_origin", DEFAULTS["access_control_allow_origin"] ) - config["read_with_mmap"] = config.get("read_with_mmap", DEFAULTS["read_with_mmap"]) # Validate CORS. Must have protocol, domain, and optionally port. _validate_cors(config["access_control_allow_origin"]) diff --git a/requirements.in b/requirements.in index 8cf5e93..4e3052b 100644 --- a/requirements.in +++ b/requirements.in @@ -9,6 +9,7 @@ pylibmc pyproj pytest pytest-cov +pytest-timeout PyYAML rasterio<1.3.0 requests diff --git a/requirements.txt b/requirements.txt index 153eedd..19192ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ coverage[toml]==7.3.0 # via pytest-cov exceptiongroup==1.1.3 # via pytest -flask==2.3.2 +flask==2.3.3 # via # -r requirements.in # flask-caching @@ -95,8 +95,11 @@ pytest==7.4.0 # via # -r requirements.in # pytest-cov + # pytest-timeout pytest-cov==4.1.0 # via -r requirements.in +pytest-timeout==2.1.0 + # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in rasterio==1.2.10 @@ -119,7 +122,7 @@ urllib3==2.0.4 # via requests werkzeug==2.3.7 # via flask -wheel==0.41.1 +wheel==0.41.2 # via pip-tools zipp==3.16.2 # via importlib-metadata From 299a72b595870e4c2323b7b5d1c1f6a2fe843277 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Tue, 22 Aug 2023 11:12:25 -0700 Subject: [PATCH 07/20] Fix docker image --- docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 28ac740..764eca8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # Container for packages that need to be built from source but have massive dev dependencies. -FROM python:3.9.16-slim-bullseye as builder +FROM python:3.9.17-slim-bullseye as builder RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -7,11 +7,11 @@ RUN set -e && \ python3.9-dev RUN pip config set global.disable-pip-version-check true && \ - pip wheel --wheel-dir=/root/wheels uwsgi==2.0.21 && \ - pip wheel --wheel-dir=/root/wheels regex==2022.10.31 + pip wheel --wheel-dir=/root/wheels uwsgi==2.0.22 && \ + pip wheel --wheel-dir=/root/wheels regex==2023.8.8 # The actual container. -FROM python:3.9.16-slim-bullseye +FROM python:3.9.17-slim-bullseye RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ From 3d5ef96d4cf304556bc7a814bd548beb5d38dcbb Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Tue, 22 Aug 2023 11:17:35 -0700 Subject: [PATCH 08/20] Upgrade rasterio to 1.3 --- opentopodata/backend.py | 2 +- requirements.in | 2 +- requirements.txt | 2 +- tests/test_backend.py | 10 ---------- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/opentopodata/backend.py b/opentopodata/backend.py index d5a6c8f..5280b32 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -69,7 +69,7 @@ def _validate_points_lie_within_raster(xs, ys, lats, lons, bounds, res): return sorted(oob_indices) -def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): +def _get_elevation_from_path(lats, lons, path, interpolation): """Read values at locations in a raster. Args: diff --git a/requirements.in b/requirements.in index 4e3052b..0d344ef 100644 --- a/requirements.in +++ b/requirements.in @@ -11,5 +11,5 @@ pytest pytest-cov pytest-timeout PyYAML -rasterio<1.3.0 +rasterio==1.3.8 requests diff --git a/requirements.txt b/requirements.txt index 19192ee..422f820 100644 --- a/requirements.txt +++ b/requirements.txt @@ -102,7 +102,7 @@ pytest-timeout==2.1.0 # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in -rasterio==1.2.10 +rasterio==1.3.8 # via -r requirements.in requests==2.31.0 # via -r requirements.in diff --git a/tests/test_backend.py b/tests/test_backend.py index db6129a..b6e39d1 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -154,16 +154,6 @@ def test_bilinear_interpolation(self): 0.4, 0.3, self.geotiff_z[:2, :2] ) - def test_mmap(self): - lats = [89.6] - lons = [-179.7] - z = backend._get_elevation_from_path( - lats, lons, ETOPO1_GEOTIFF_PATH, "bilinear", use_mmap=True - ) - assert pytest.approx(z[0]) == self._interp_bilinear( - 0.4, 0.3, self.geotiff_z[:2, :2] - ) - def test_none_outside_dataset(self): lats = [0, 0, -90.1, 90.1] lons = [-180.1, 180.1, 0, 0] From 2534c2683dca252aa31ffb2f417d3097357e1d8e Mon Sep 17 00:00:00 2001 From: Arne Setzer Date: Thu, 9 Nov 2023 23:47:53 +0100 Subject: [PATCH 09/20] Add a note for parallelisation (#85) Co-authored-by: arnsetzer <25772747+arnesetzer@users.noreply.github.com> --- docs/notes/performance-optimisation.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/notes/performance-optimisation.md b/docs/notes/performance-optimisation.md index 537915f..30efce9 100644 --- a/docs/notes/performance-optimisation.md +++ b/docs/notes/performance-optimisation.md @@ -19,6 +19,14 @@ Batch request are faster (per point queried) than single-point requests, and lar Batch queries are fastest if the points are located next to each other. Sorting the locations you are querying before batching will improve performance. Ideally sort by some block-level attribute like postal code or state/county/region, or by something like `round(lat, 1), round(lon, 1)` depending on your tile size. +If the requests are very large and the server has several CPU cores, try splitting the request and sending it simultaneously. The optimum for the number of requests is slightly higher than the amount of CPU cores used by OpenTopodata. + +Example: With 4 CPU cores in use, a maximum of 5-6 requests should run simultaneously. + +The number of CPU cores used is displayed when OpenTopodata is started. Alternatively, it can also be determined with the following command: +```bash +docker logs elevation-service1 2>&1 | grep "CPU cores" +``` ## Dataset format From d7fb8197fdf31dde5c456a23c70a42ab65a04915 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 14:30:29 -0800 Subject: [PATCH 10/20] Docs tweak --- docs/notes/performance-optimisation.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/notes/performance-optimisation.md b/docs/notes/performance-optimisation.md index 30efce9..5c045de 100644 --- a/docs/notes/performance-optimisation.md +++ b/docs/notes/performance-optimisation.md @@ -19,13 +19,9 @@ Batch request are faster (per point queried) than single-point requests, and lar Batch queries are fastest if the points are located next to each other. Sorting the locations you are querying before batching will improve performance. Ideally sort by some block-level attribute like postal code or state/county/region, or by something like `round(lat, 1), round(lon, 1)` depending on your tile size. -If the requests are very large and the server has several CPU cores, try splitting the request and sending it simultaneously. The optimum for the number of requests is slightly higher than the amount of CPU cores used by OpenTopodata. - -Example: With 4 CPU cores in use, a maximum of 5-6 requests should run simultaneously. - -The number of CPU cores used is displayed when OpenTopodata is started. Alternatively, it can also be determined with the following command: +If the requests are very large and the server has several CPU cores, try splitting the request and sending it simultaneously. The optimum for the number of requests is slightly higher than the amount of CPU cores used by Open Topo Data. The number of CPU cores used is displayed when OpenTopodata is started. If you missed the log message, you can find iw with the following command: ```bash -docker logs elevation-service1 2>&1 | grep "CPU cores" +docker logs {NAME_OF_CONTAINER} 2>&1 | grep "CPU cores" ``` From cb7a69d4ba1b1772b3da60a625f33c50479d4ad8 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 14:36:21 -0800 Subject: [PATCH 11/20] Upgrade dependencies --- docker/Dockerfile | 2 +- requirements.txt | 66 +++++++++++++++++++++++++---------------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 764eca8..3eff239 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,7 @@ RUN set -e && \ RUN pip config set global.disable-pip-version-check true && \ pip wheel --wheel-dir=/root/wheels uwsgi==2.0.22 && \ - pip wheel --wheel-dir=/root/wheels regex==2023.8.8 + pip wheel --wheel-dir=/root/wheels regex==2023.12.25 # The actual container. FROM python:3.9.17-slim-bullseye diff --git a/requirements.txt b/requirements.txt index 422f820..e991303 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,22 +6,22 @@ # affine==2.4.0 # via rasterio -attrs==23.1.0 +attrs==23.2.0 # via rasterio -black==23.7.0 +black==24.2.0 # via -r requirements.in -blinker==1.6.2 +blinker==1.7.0 # via flask -build==0.10.0 +build==1.0.3 # via pip-tools cachelib==0.9.0 # via flask-caching -certifi==2023.7.22 +certifi==2024.2.2 # via # pyproj # rasterio # requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -35,70 +35,74 @@ click-plugins==1.1.1 # via rasterio cligj==0.7.2 # via rasterio -coverage[toml]==7.3.0 +coverage[toml]==7.4.1 # via pytest-cov -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via pytest -flask==2.3.3 +flask==3.0.2 # via # -r requirements.in # flask-caching -flask-caching==2.0.2 +flask-caching==2.1.0 # via -r requirements.in geographiclib==2.0 # via -r requirements.in -idna==3.4 +idna==3.6 # via requests -importlib-metadata==6.8.0 - # via flask +importlib-metadata==7.0.1 + # via + # build + # flask iniconfig==2.0.0 # via pytest itsdangerous==2.1.2 # via flask -jinja2==3.1.2 +jinja2==3.1.3 # via flask -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # werkzeug mypy-extensions==1.0.0 # via black -numpy==1.25.2 +numpy==1.26.4 # via # -r requirements.in # rasterio # snuggs -packaging==23.1 +packaging==23.2 # via # black # build # pytest -pathspec==0.11.2 +pathspec==0.12.1 # via black -pip-tools==7.3.0 +pip-tools==7.4.0 # via -r requirements.in -platformdirs==3.10.0 +platformdirs==4.2.0 # via black -pluggy==1.2.0 +pluggy==1.4.0 # via pytest -polyline==2.0.0 +polyline==2.0.2 # via -r requirements.in pylibmc==1.6.3 # via -r requirements.in pyparsing==3.1.1 # via snuggs -pyproj==3.6.0 +pyproj==3.6.1 # via -r requirements.in pyproject-hooks==1.0.0 - # via build -pytest==7.4.0 + # via + # build + # pip-tools +pytest==8.0.1 # via # -r requirements.in # pytest-cov # pytest-timeout pytest-cov==4.1.0 # via -r requirements.in -pytest-timeout==2.1.0 +pytest-timeout==2.2.0 # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in @@ -116,15 +120,15 @@ tomli==2.0.1 # pip-tools # pyproject-hooks # pytest -typing-extensions==4.7.1 +typing-extensions==4.9.0 # via black -urllib3==2.0.4 +urllib3==2.2.1 # via requests -werkzeug==2.3.7 +werkzeug==3.0.1 # via flask -wheel==0.41.2 +wheel==0.42.0 # via pip-tools -zipp==3.16.2 +zipp==3.17.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From b903632bf3daffd69404dd26e5181e2fb93b706a Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 15:01:04 -0800 Subject: [PATCH 12/20] Support preflight requests --- docs/changelog.md | 5 ++++- opentopodata/api.py | 34 +++++++++++++++++++++++----------- tests/test_api.py | 9 +++++++++ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b0d8cf9..ae5ac32 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,7 +3,10 @@ This is a list of changes to Open Topo Data between each release. -## Version 1.8.4 (18 Aug 2023) +## Version 1.8.4 (19 Feb 2024) +* Dependency upgrades +* Fix handling of preflight requests ([#93](https://github.com/ajnisbet/opentopodata/issues/93)) + ## Version 1.8.3 (7 Feb 2023) diff --git a/opentopodata/api.py b/opentopodata/api.py index a7e1f5e..7f85f8c 100644 --- a/opentopodata/api.py +++ b/opentopodata/api.py @@ -1,7 +1,7 @@ import logging import os -from flask import Flask, jsonify, request +from flask import Flask, jsonify, request, Response from flask_caching import Cache import polyline @@ -64,6 +64,18 @@ def _load_config_memcache(): return config.load_config() +@app.before_request +def handle_preflight(): + # If before_request returns a non-none value, the regular view isn't run. + # after_request() does still run though, so the CORS header and OTD version + # will be set correctly there. + if request.method == "OPTIONS": + response = Response(status=204) + response.headers["access-control-allow-methods"] = "GET,POST,OPTIONS,HEAD" + response.headers["access-control-allow-headers"] = "content-type,x-api-key" + return response + + @app.after_request def apply_cors(response): """Set CORs header. @@ -84,6 +96,16 @@ def apply_cors(response): return response +@app.after_request +def add_version(response): + if "version" not in _SIMPLE_CACHE: + with open(VERSION_PATH) as f: + version = f.read().strip() + _SIMPLE_CACHE["version"] = version + response.headers["x-opentopodata-version"] = _SIMPLE_CACHE["version"] + return response + + class ClientError(ValueError): """Invalid input data. @@ -543,13 +565,3 @@ def get_elevation(dataset_name): app.logger.error(e) msg = "Unhandled server error, see server logs for details." return jsonify({"status": "SERVER_ERROR", "error": msg}), 500 - - -@app.after_request -def add_version(response): - if "version" not in _SIMPLE_CACHE: - with open(VERSION_PATH) as f: - version = f.read().strip() - _SIMPLE_CACHE["version"] = version - response.headers["x-opentopodata-version"] = _SIMPLE_CACHE["version"] - return response diff --git a/tests/test_api.py b/tests/test_api.py index d363504..9a2d90f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -41,6 +41,15 @@ def test_no_cors(self, patch_config): response = test_api.get(url) assert response.headers.get("access-control-allow-origin") is None + def test_options(self): + test_api = api.app.test_client() + url = "/" + response = test_api.options(url) + assert response.status_code == 204 + assert "x-opentopodata-version" in response.headers + assert "access-control-allow-methods" in response.headers + assert response.headers.get("access-control-allow-origin") == "*" + class TestFindRequestAgument: def test_no_argument(self, patch_config): From 0b64919b353c86a20d7934a15d063a49d1d557ae Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 15:04:54 -0800 Subject: [PATCH 13/20] Remove now-unneeded OPTIONs methods Fixes https://github.com/ajnisbet/opentopodata/issues/93 --- opentopodata/api.py | 10 +++++----- tests/test_api.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/opentopodata/api.py b/opentopodata/api.py index 7f85f8c..2e0df6a 100644 --- a/opentopodata/api.py +++ b/opentopodata/api.py @@ -458,8 +458,8 @@ def _get_datasets(name): return datasets -@app.route("/", methods=["GET", "POST", "OPTIONS", "HEAD"]) -@app.route("/v1/", methods=["GET", "POST", "OPTIONS", "HEAD"]) +@app.route("/", methods=["GET", "POST", "HEAD"]) +@app.route("/v1/", methods=["GET", "POST", "HEAD"]) def get_help_message(): msg = "No dataset name provided." msg += " Try a url like '/v1/test-dataset?locations=-10,120' to get started," @@ -467,7 +467,7 @@ def get_help_message(): return jsonify({"status": "INVALID_REQUEST", "error": msg}), 404 -@app.route("/health", methods=["GET", "OPTIONS", "HEAD"]) +@app.route("/health", methods=["GET", "HEAD"]) def get_health_status(): """Status endpoint for e.g., uptime check or load balancing.""" try: @@ -480,7 +480,7 @@ def get_health_status(): return jsonify(data), 500 -@app.route("/datasets", methods=["GET", "OPTIONS", "HEAD"]) +@app.route("/datasets", methods=["GET", "HEAD"]) def get_datasets_info(): """List of datasets on the server.""" try: @@ -501,7 +501,7 @@ def get_datasets_info(): return jsonify(data), 500 -@app.route("/v1/", methods=["GET", "POST", "OPTIONS", "HEAD"]) +@app.route("/v1/", methods=["GET", "POST", "HEAD"]) def get_elevation(dataset_name): """Calculate the elevation for the given locations. diff --git a/tests/test_api.py b/tests/test_api.py index 9a2d90f..ca7f2e3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,7 +1,6 @@ import math import pytest -from flask_caching import Cache import rasterio from unittest.mock import patch import numpy as np @@ -9,7 +8,6 @@ from opentopodata import api from opentopodata import backend -from opentopodata import config GEOTIFF_PATH = "tests/data/datasets/test-etopo1-resampled-1deg/ETOPO1_Ice_g_geotiff.resampled-1deg.tif" From de7798fe6e095436e2eb1f0f9d78ef20614a85d8 Mon Sep 17 00:00:00 2001 From: Arne Setzer Date: Tue, 20 Feb 2024 00:15:00 +0100 Subject: [PATCH 14/20] Feature: Output as geojson Feature Collection (#86) * Add support for geojson output * Fix failed tests * Change to correct geojson syntax * formatting * documentation geojson * Fix lon lat order --------- Co-authored-by: arnsetzer <25772747+arnesetzer@users.noreply.github.com> Co-authored-by: Andrew Nisbet --- docs/api.md | 58 +++++++++++++++++++++++++++++++++++++++++++++ opentopodata/api.py | 35 ++++++++++++++++++++------- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/docs/api.md b/docs/api.md index 5a68fcd..4b09a93 100644 --- a/docs/api.md +++ b/docs/api.md @@ -24,12 +24,15 @@ Latitudes and longitudes should be in `EPSG:4326` (also known as WGS-84 format), * The default option `null` makes NODATA indistinguishable from a location outside the dataset bounds. * `NaN` (not a number) values aren't valid in json and will break some clients. The `nan` option was default before version 1.4 and is provided only for backwards compatibility. * When querying multiple datasets, this NODATA replacement only applies to the last dataset in the stack. +* `geojson`: If set to `True` the response will be in geojson according to RFC7946. ### Response +#### Default + A json object, compatible with the Google Maps Elevation API. * `status`: Will be `OK` for a successful request, `INVALID_REQUEST` for an input (4xx) error, and `SERVER_ERROR` for anything else (5xx). Required. @@ -46,8 +49,21 @@ Some notes about the elevation value: * If the request location isn't covered by any raster in the dataset, Open Topo Data will return `null`. * Unless the `nodata_value` parameter is set, a `null` elevation could either mean the location is outside the dataset bounds, or a NODATA within the raster bounds. +#### Geojson +A json object, fulfilling the RFC 7946 or an error message +* `type`: FeatureCollection +* `features`: An array of provided points + * `type`: Feature + * `geometry` + * `type`: Point + * `coordinates`: Longitude, latitude and Elevation + * `properties` + * `dataset`: Name of the dataset +In case of an error: +* `status`: Will be `OK` for a successful request, `INVALID_REQUEST` for an input (4xx) error, and `SERVER_ERROR` for anything else (5xx). Required. +* `error`: Description of what went wrong, when `status` isn't `OK`. ### Example `GET` api.opentopodata.org/v1/srtm90m?locations=-43.5,172.5|27.6,1.98&interpolation=cubic @@ -79,6 +95,48 @@ Some notes about the elevation value: } ``` +#### With geojson + +`GET` api.opentopodata.org/v1/srtm90m?locations=-43.5,172.5|27.6,1.98&interpolation=cubic&geojson=True + + + + +```json +{ + "features": [ + { + "geometry": { + "coordinates": [ + 172.5, + -43.5, + 45 + ], + "type": "Point" + }, + "properties": { + "dataset": "srtm90m" + }, + "type": "Feature" + }, + { + "geometry": { + "coordinates": [ + 1.98, + 27.6, + 402 + ], + "type": "Point" + }, + "properties": { + "dataset": "srtm90m" + }, + "type": "Feature" + } + ], + "type": "FeatureCollection" +} +``` --- diff --git a/opentopodata/api.py b/opentopodata/api.py index 2e0df6a..b44f855 100644 --- a/opentopodata/api.py +++ b/opentopodata/api.py @@ -19,6 +19,7 @@ LON_MIN = -180 LON_MAX = 180 VERSION_PATH = "VERSION" +DEFAULT_GEOJSON_VALUE = False # Memcache is used to store the latlon -> filename lookups, which can take a @@ -524,6 +525,10 @@ def get_elevation(dataset_name): _find_request_argument(request, "locations"), _load_config()["max_locations_per_request"], ) + try: + geojson = _find_request_argument(request, "geojson") + except: + geojson = DEFAULT_GEOJSON_VALUE # Check if need to do sampling. n_samples = _parse_n_samples( @@ -541,15 +546,27 @@ def get_elevation(dataset_name): # Build response. results = [] - for z, dataset_name, lat, lon in zip(elevations, dataset_names, lats, lons): - results.append( - { - "elevation": z, - "dataset": dataset_name, - "location": {"lat": lat, "lng": lon}, - } - ) - data = {"status": "OK", "results": results} + # Return the results in geojson format. Default false to keep API consistancy + if geojson: + for z, dataset_name, lat, lon in zip(elevations, dataset_names, lats, lons): + results.append( + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [lon, lat, z]}, + "properties": {"dataset": dataset_name}, + }, + ) + data = {"type": "FeatureCollection", "features": results} + else: + for z, dataset_name, lat, lon in zip(elevations, dataset_names, lats, lons): + results.append( + { + "elevation": z, + "dataset": dataset_name, + "location": {"lat": lat, "lng": lon}, + } + ) + data = {"status": "OK", "results": results} return jsonify(data) except (ClientError, backend.InputError) as e: From 9c8a80dd3a627aedb8884f2095478be79f8c7e35 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 15:34:56 -0800 Subject: [PATCH 15/20] Rework geojson --- docs/api.md | 34 +++++++++++++--------------------- docs/changelog.md | 3 ++- opentopodata/api.py | 23 ++++++++++++++++------- tests/test_api.py | 10 ++++++++++ 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/docs/api.md b/docs/api.md index 4b09a93..241fca8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -24,15 +24,13 @@ Latitudes and longitudes should be in `EPSG:4326` (also known as WGS-84 format), * The default option `null` makes NODATA indistinguishable from a location outside the dataset bounds. * `NaN` (not a number) values aren't valid in json and will break some clients. The `nan` option was default before version 1.4 and is provided only for backwards compatibility. * When querying multiple datasets, this NODATA replacement only applies to the last dataset in the stack. -* `geojson`: If set to `True` the response will be in geojson according to RFC7946. +* `format`: Either `json` or `geojson`. Default: `json`. ### Response -#### Default - A json object, compatible with the Google Maps Elevation API. * `status`: Will be `OK` for a successful request, `INVALID_REQUEST` for an input (4xx) error, and `SERVER_ERROR` for anything else (5xx). Required. @@ -49,28 +47,12 @@ Some notes about the elevation value: * If the request location isn't covered by any raster in the dataset, Open Topo Data will return `null`. * Unless the `nodata_value` parameter is set, a `null` elevation could either mean the location is outside the dataset bounds, or a NODATA within the raster bounds. -#### Geojson - -A json object, fulfilling the RFC 7946 or an error message -* `type`: FeatureCollection -* `features`: An array of provided points - * `type`: Feature - * `geometry` - * `type`: Point - * `coordinates`: Longitude, latitude and Elevation - * `properties` - * `dataset`: Name of the dataset -In case of an error: -* `status`: Will be `OK` for a successful request, `INVALID_REQUEST` for an input (4xx) error, and `SERVER_ERROR` for anything else (5xx). Required. -* `error`: Description of what went wrong, when `status` isn't `OK`. ### Example `GET` api.opentopodata.org/v1/srtm90m?locations=-43.5,172.5|27.6,1.98&interpolation=cubic - - ```json { "results": [ @@ -95,9 +77,16 @@ In case of an error: } ``` -#### With geojson -`GET` api.opentopodata.org/v1/srtm90m?locations=-43.5,172.5|27.6,1.98&interpolation=cubic&geojson=True +### GeoJSON response + +If `format=geojson` is passed, you get a `FeatureCollection` of `Point` geometries instead. Each feature has its elevation as the `z` coordinate, and a `dataset` property specifying the source (corresponding to `results[].dataset` in the regular json response): + + +### GeoJSON example + + +`GET` api.opentopodata.org/v1/srtm90m?locations=-43.5,172.5|27.6,1.98&interpolation=cubic&format=geojson @@ -137,6 +126,9 @@ In case of an error: "type": "FeatureCollection" } ``` + + + --- diff --git a/docs/changelog.md b/docs/changelog.md index ae5ac32..1e12260 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,9 +3,10 @@ This is a list of changes to Open Topo Data between each release. -## Version 1.8.4 (19 Feb 2024) +## Version 1.9.0 (19 Feb 2024) * Dependency upgrades * Fix handling of preflight requests ([#93](https://github.com/ajnisbet/opentopodata/issues/93)) +* Add support for geojson responses ([#86](https://github.com/ajnisbet/opentopodata/pull/86)) ## Version 1.8.3 (7 Feb 2023) diff --git a/opentopodata/api.py b/opentopodata/api.py index b44f855..ea8d675 100644 --- a/opentopodata/api.py +++ b/opentopodata/api.py @@ -19,7 +19,7 @@ LON_MIN = -180 LON_MAX = 180 VERSION_PATH = "VERSION" -DEFAULT_GEOJSON_VALUE = False +DEFAULT_FORMAT_VALUE = "json" # Memcache is used to store the latlon -> filename lookups, which can take a @@ -151,6 +151,16 @@ def _find_request_argument(request, arg): raise ClientError("Invalid JSON.") +def _parse_format(format): + if not format: + format = DEFAULT_FORMAT_VALUE + + if format not in {"json", "geojson"}: + raise ClientError("Format must be 'json' or 'geojson'.") + + return format + + def _parse_interpolation(method): """Check the interpolation method is supported. @@ -525,10 +535,7 @@ def get_elevation(dataset_name): _find_request_argument(request, "locations"), _load_config()["max_locations_per_request"], ) - try: - geojson = _find_request_argument(request, "geojson") - except: - geojson = DEFAULT_GEOJSON_VALUE + format = _parse_format(_find_request_argument(request, "format")) # Check if need to do sampling. n_samples = _parse_n_samples( @@ -546,8 +553,9 @@ def get_elevation(dataset_name): # Build response. results = [] - # Return the results in geojson format. Default false to keep API consistancy - if geojson: + + # Convert to json or geojson format. + if format == "geojson": for z, dataset_name, lat, lon in zip(elevations, dataset_names, lats, lons): results.append( { @@ -557,6 +565,7 @@ def get_elevation(dataset_name): }, ) data = {"type": "FeatureCollection", "features": results} + else: for z, dataset_name, lat, lon in zip(elevations, dataset_names, lats, lons): results.append( diff --git a/tests/test_api.py b/tests/test_api.py index ca7f2e3..eb98f5a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -289,6 +289,16 @@ def test_repeated_locations(self, patch_config): assert len(rjson["results"]) == 2 assert rjson["results"][0] == rjson["results"][1] + def test_repeated_locations_geojson(self, patch_config): + url = ( + "/v1/etopo1deg?locations=1.5,0.1|1.5,0.1&interpolation=cubic&format=geojson" + ) + response = self.test_api.get(url) + rjson = response.json + assert response.status_code == 200 + assert len(rjson["features"]) == 2 + assert rjson["features"][0] == rjson["features"][1] + def test_polyline_latlon_equivalence(self, patch_config): url_latlon = "/v1/etopo1deg?locations=-90,180|1.5,0.1" url_polyline = "/v1/etopo1deg?locations=~bidP_gsia@_bnmP~u_ia@" From 6188f76f51a7fe26fb46078c6cedf5018a524686 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 15:36:07 -0800 Subject: [PATCH 16/20] Credit Fixes https://github.com/ajnisbet/opentopodata/pull/86 --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 1e12260..51c3b78 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,7 +6,7 @@ This is a list of changes to Open Topo Data between each release. ## Version 1.9.0 (19 Feb 2024) * Dependency upgrades * Fix handling of preflight requests ([#93](https://github.com/ajnisbet/opentopodata/issues/93)) -* Add support for geojson responses ([#86](https://github.com/ajnisbet/opentopodata/pull/86)) +* Add support for geojson responses ([#86](https://github.com/ajnisbet/opentopodata/pull/86), thanks [@arnesetzer](https://github.com/arnesetzer)!) ## Version 1.8.3 (7 Feb 2023) From 36643eb329e35c154c934a0fa8a091fa46642f7e Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 15:44:28 -0800 Subject: [PATCH 17/20] Document new EUDEM download --- docs/datasets/eudem.md | 122 +++-------------------------------------- 1 file changed, 8 insertions(+), 114 deletions(-) diff --git a/docs/datasets/eudem.md b/docs/datasets/eudem.md index 25be1c8..48c60d4 100644 --- a/docs/datasets/eudem.md +++ b/docs/datasets/eudem.md @@ -30,45 +30,25 @@ The advantage of the `NODATA` oceans is that you cane use EU-DEM without clippin ## Adding EU-DEM to Open Topo Data +As of Jan 2024, EU-DEM is no longer available to download via copernicus.eu. -Make a new folder for the dataset: +I have uploaded my version of the dataset at [https://files.gpxz.io/eudem_buffered.zip](https://files.gpxz.io/eudem_buffered.zip), see [EUDEM download](https://www.gpxz.io/blog/eudem) for more details. -```bash -mkdir ./data/eudem -``` - -Download the dataset from [Copernicus](https://land.copernicus.eu/imagery-in-situ/eu-dem/eu-dem-v1.1?tab=download). There are 27 files. Unzip them and move all the `.TIF` files into the data folder (you don't need the `.aux.xml`, `.ovr`, or `.TFw` files). - -Your data folder should now contain only 27 TIF files: +Download and unzip the folder into: ```bash -ls ./data/eudem - -# eu_dem_v11_E00N20.TIF -# eu_dem_v11_E10N00.TIF -# eu_dem_v11_E10N10.TIF -# ... -``` - - -If you have [gdal](https://gdal.org) installed, the easiest thing to do here is build a [VRT](https://gdal.org/drivers/raster/vrt.html) - a single raster file that links to the 27 tiles and which Open Topo Data can treat as a single-file dataset. - -```bash -mkdir ./data/eudem-vrt -cd ./data/eudem-vrt -gdalbuildvrt -tr 25 25 -tap -te 0 0 8000000 6000000 eudem.vrt ../eudem/*.TIF -cd ../../ +mkdir ./data/eudem ``` - -The `tr`, `tap`, and `te` options in the above command ensure that slices from the VRT will use the exact values and grid of the source rasters. - +There are 27 files. Then create a `config.yaml` file: ```yaml datasets: - name: eudem25m - path: data/eudem-vrt/ + path: data/eudem + filename_epsg: 3035 + filename_tile_size: 1000000 ``` Finally, rebuild to enable the new dataset at [localhost:5000/v1/eudem25m?locations=51.575,-3.220](http://localhost:5000/v1/eudem25m?locations=51.575,-3.220). @@ -82,92 +62,6 @@ make build && make run If you don't have gdal installed, you can use the tiles directly. There are instructions for this [here](https://github.com/ajnisbet/opentopodata/blob/f012ec136bebcd97e1dc05645e91a6d2487127dc/docs/datasets/eudem.md#adding-eu-dem-to-open-topo-data), but because the EU-DEM tiles don't come with an overlap you will get a `null` elevation at locations within 0.5 pixels of tile edges. -### Buffering tiles (optional) - -The tiles provided by EU-DEM don't overlap and cover slightly less than a 1000km square. This means you'll get a `null` result for coordinates along the tile edges. - -The `.vrt` approach above solves the overlap issue, but for improved performance you can leave the tiles separate and add a buffer to each one. This is the code I used on the public API to do this: - - -```python -import os -from glob import glob -import subprocess - -import rasterio - - -# Prepare paths. -input_pattern = 'data/eudem/*.TIF' -input_paths = sorted(glob(input_pattern)) -assert input_paths -vrt_path = 'data/eudem-vrt/eudem.vrt' -output_dir = 'data/eudem-buffered/' -os.makedirs(output_dir, exist_ok=True) - - - -# EU-DEM specific options. -tile_size = 1_000_000 -buffer_size = 50 - -for input_path in input_paths: - - # Get tile bounds. - with rasterio.open(input_path) as f: - bottom = int(f.bounds.bottom) - left = int(f.bounds.left) - - # For EU-DEM only: round this partial tile down to the nearest tile_size. - if left == 943750: - left = 0 - - # New tile name in SRTM format. - output_name = 'N' + str(bottom).zfill(7) + 'E' + str(left).zfill(7) + '.TIF' - output_path = os.path.join(output_dir, output_name) - - # New bounds. - xmin = left - buffer_size - xmax = left + tile_size + buffer_size - ymin = bottom - buffer_size - ymax = bottom + tile_size + buffer_size - - # EU-DEM tiles don't cover negative locations. - xmin = max(0, xmin) - ymin = max(0, ymin) - - # Do the transformation. - cmd = [ - 'gdal_translate', - '-a_srs', 'EPSG:3035', # EU-DEM crs. - '-co', 'NUM_THREADS=ALL_CPUS', - '-co', 'COMPRESS=DEFLATE', - '-co', 'BIGTIFF=YES', - '--config', 'GDAL_CACHEMAX','512', - '-projwin', str(xmin), str(ymax), str(xmax), str(ymin), - vrt_path, output_path, - ] - r = subprocess.run(cmd) - r.check_returncode() -``` - -These new files can be used in Open Topo Data with the following `config.yaml` file - - -```yaml -datasets: -- name: eudem25m - path: data/eudem-buffered/ - filename_epsg: 3035 - filename_tile_size: 1000000 -``` - -and rebuilding: - -```bash -make build && make run -``` - ## Public API From fd6c694c7455d8cc2d9d0da67baa45a700762e35 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 15:47:09 -0800 Subject: [PATCH 18/20] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fe4e75f..abb1658 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.3 \ No newline at end of file +1.9.0 \ No newline at end of file From 7e029738da365f83a9af54a4b56a887bbd76e2db Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 16:23:33 -0800 Subject: [PATCH 19/20] Upgrade dependencies --- docker/Dockerfile | 8 ++++---- docs/changelog.md | 6 ++++-- docs/notes/kubernetes.md | 7 ++++++- requirements.in | 4 ++-- requirements.txt | 22 ++-------------------- 5 files changed, 18 insertions(+), 29 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3eff239..30c0847 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,17 +1,17 @@ # Container for packages that need to be built from source but have massive dev dependencies. -FROM python:3.9.17-slim-bullseye as builder +FROM python:3.11.8-slim-bookworm as builder RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ gcc \ - python3.9-dev + python3.12-dev RUN pip config set global.disable-pip-version-check true && \ - pip wheel --wheel-dir=/root/wheels uwsgi==2.0.22 && \ + pip wheel --wheel-dir=/root/wheels uwsgi==2.0.24 && \ pip wheel --wheel-dir=/root/wheels regex==2023.12.25 # The actual container. -FROM python:3.9.17-slim-bullseye +FROM python:3.11.8-slim-bookworm RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/docs/changelog.md b/docs/changelog.md index 51c3b78..a8ccdce 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,9 +4,11 @@ This is a list of changes to Open Topo Data between each release. ## Version 1.9.0 (19 Feb 2024) -* Dependency upgrades -* Fix handling of preflight requests ([#93](https://github.com/ajnisbet/opentopodata/issues/93)) +* Dependency upgrades, including python to 3.11 and rasterio to 1.3.9 * Add support for geojson responses ([#86](https://github.com/ajnisbet/opentopodata/pull/86), thanks [@arnesetzer](https://github.com/arnesetzer)!) +* Fix handling of preflight requests ([#93](https://github.com/ajnisbet/opentopodata/issues/93), thanks [@MaRaSu](https://github.com/MaRaSu)!) +* Fix error message bug ([#70](https://github.com/ajnisbet/opentopodata/pull/70), thanks [@khendrickx](https://github.com/khendrickx)!) + ## Version 1.8.3 (7 Feb 2023) diff --git a/docs/notes/kubernetes.md b/docs/notes/kubernetes.md index b853438..9ea7fee 100644 --- a/docs/notes/kubernetes.md +++ b/docs/notes/kubernetes.md @@ -82,4 +82,9 @@ spec: - containerPort: 5000 restartPolicy: Always -``` \ No newline at end of file +``` + + +--- + +Thanks to [@khintz](https://github.com/khintz) for contributing this documentation in [#57](https://github.com/ajnisbet/opentopodata/pull/57)! \ No newline at end of file diff --git a/requirements.in b/requirements.in index 0d344ef..637a868 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ black -Flask>=2.2.2 +Flask>=2.2.2 # Some flask 2.0 deprecations got real. flask-caching geographiclib numpy @@ -11,5 +11,5 @@ pytest pytest-cov pytest-timeout PyYAML -rasterio==1.3.8 +rasterio>=1.3.8 # Avoid memory leak https://github.com/ajnisbet/opentopodata/issues/68 requests diff --git a/requirements.txt b/requirements.txt index e991303..b900d0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements.in @@ -37,8 +37,6 @@ cligj==0.7.2 # via rasterio coverage[toml]==7.4.1 # via pytest-cov -exceptiongroup==1.2.0 - # via pytest flask==3.0.2 # via # -r requirements.in @@ -49,10 +47,6 @@ geographiclib==2.0 # via -r requirements.in idna==3.6 # via requests -importlib-metadata==7.0.1 - # via - # build - # flask iniconfig==2.0.0 # via pytest itsdangerous==2.1.2 @@ -106,30 +100,18 @@ pytest-timeout==2.2.0 # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in -rasterio==1.3.8 +rasterio==1.3.9 # via -r requirements.in requests==2.31.0 # via -r requirements.in snuggs==1.4.7 # via rasterio -tomli==2.0.1 - # via - # black - # build - # coverage - # pip-tools - # pyproject-hooks - # pytest -typing-extensions==4.9.0 - # via black urllib3==2.2.1 # via requests werkzeug==3.0.1 # via flask wheel==0.42.0 # via pip-tools -zipp==3.17.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip From 0d111ac927c944fdb1a6d9b49117863f888ed6bf Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Mon, 19 Feb 2024 16:25:44 -0800 Subject: [PATCH 20/20] Fix py312 downgrade --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 30c0847..7d6d1b8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ gcc \ - python3.12-dev + python3.11-dev RUN pip config set global.disable-pip-version-check true && \ pip wheel --wheel-dir=/root/wheels uwsgi==2.0.24 && \