From 4572734737ba253851e4b4966e9f3af30b2ef306 Mon Sep 17 00:00:00 2001 From: bunburya Date: Mon, 6 Feb 2023 21:36:52 +0000 Subject: [PATCH] fixing handling of all-water tile regions --- bother_utils/srtm.py | 20 +++++++++++++++++--- test/gen_examples.sh | 4 +++- test/test_bother.py | 28 ++++++++++++++++++---------- test/test_coords.txt | 3 +++ 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/bother_utils/srtm.py b/bother_utils/srtm.py index 27b4d4f..eaca2c8 100644 --- a/bother_utils/srtm.py +++ b/bother_utils/srtm.py @@ -58,6 +58,13 @@ def get_tif_fpath(x: int, y: int, cache_dir: str) -> str: def download_zip(url: str, save_path: str, chunk_size: int = 1024): r = requests.get(url, stream=True) + if not r.ok: + if r.status_code == 404: + # 404 can be returned if we try fetch a tile that corresponds to + # a part of the world with no land, which is okay. + return None + else: + r.raise_for_status() r.raise_for_status() total_size_in_bytes = int(r.headers.get('content-length', 0)) @@ -129,8 +136,12 @@ def fetch_all_zips(fnames: Dict[Tuple[int, int], str], cache_dir: str) -> Dict[T print(f'Downloading {fname} from {url}.') fpath = download_zip(url, fpath) if fpath is None: - print(f'Got 404 when downloading {fname}; there is probably no data available.') - continue + print(f'WARNING: Received HTTP 404 response when attempting to download {fname}.') + print('This could mean that (1) this tile corresponds to a region of the earth ' + 'where there is no land; (2) this tile is outside the coverage area of the ' + 'SRTM data; or (3) there is some issue with the website (which may be ' + 'temporary).') + print('We will assume the cause is (1) and will proceed accordingly.') else: print(f'{fname} was cached in {cache_dir}.') fpaths[xy] = fpath @@ -162,7 +173,10 @@ def create_tif_file(left: float, bottom: float, right: float, top: float, to_fil zip_fnames[(x, y)] = ZIP_FNAME.format(x=x, y=y) zip_fpaths = fetch_all_zips(zip_fnames, cache_dir) unzip_all(zip_fpaths.values(), cache_dir) - srcs = [rasterio.open(get_tif_fpath(x, y, cache_dir), 'r', nodata=nodata) for x, y in xy] + srcs = [] + for x, y in xy: + if zip_fpaths[(x, y)] is not None: + srcs.append(rasterio.open(get_tif_fpath(x, y, cache_dir), 'r', nodata=nodata)) print(f'Creating TIF file from following files: {[s.name for s in srcs]}.') #print(f'Heights are: {[s.height for s in srcs]}.') #print(f'Widths are: {[s.width for s in srcs]}.') diff --git a/test/gen_examples.sh b/test/gen_examples.sh index 5c65579..93f2f50 100755 --- a/test/gen_examples.sh +++ b/test/gen_examples.sh @@ -13,6 +13,7 @@ pipenv run ../bother -ot examples/titicaca.tif -b -16.970092 -70.246331 -15.0323 pipenv run ../bother -ot examples/constance.tif -b 47.321556 8.830936 47.980224 9.860904 examples/constance.png pipenv run ../bother -ot examples/alps.tif -b 46.408240 9.555657 46.975534 10.378602 examples/alps.png pipenv run ../bother -ot examples/germany_nw.tif -b 53.373069 7.387423 53.814826 8.234738 examples/germany_nw.png +#pipenv run ../bother -ot examples/vanc_oregon.tif -b 41.82 -129.946 51.468 -116.125 examples/vanc_oregon.png # Mallorca: Crop image but don't do any other processing (including no reprojection) pipenv run ../bother -it examples/mallorca.tif --epsg 4326 --crop 1574x787 c -si 2048x1024 examples/mallorca_no_processing_resized.png @@ -32,4 +33,5 @@ pipenv run ../bother --epsg 4326 -l -b -16.970092 -70.246331 -15.032316 -68.3347 # Germany: Raise undersea pixels and also raise low pixels. pipenv run ../bother -rl 0 -ru 1 -b 53.373069 7.387423 53.814826 8.234738 examples/germany_nw_undersea_raised.png - +# Vancouver & Oregon: Crop and scale +#pipenv run ../bother -it examples/vanc_oregon.tif --crop 11578x11578 c -si 4000x4000 examples/vanc_oregon_scaled.png diff --git a/test/test_bother.py b/test/test_bother.py index 553510c..edfc682 100755 --- a/test/test_bother.py +++ b/test/test_bother.py @@ -25,9 +25,12 @@ 'constance': (47.321556, 8.830936, 47.980224, 9.860904), 'titicaca': (-16.970092, -70.246331, -15.032316, -68.334710), 'alps': (46.408240, 9.555657, 46.975534, 10.378602), - 'germany_nw': (53.373069, 7.387423, 53.814826, 8.234738) + 'germany_nw': (53.373069, 7.387423, 53.814826, 8.234738), } +# We don't test this with the rest, because the raw image is too +# large to compare (PIL crashes) +vanc_oregon = (41.82, -129.946, 51.468, -116.125) # No options provided MALLORCA_TIF = os.path.join(EXAMPLES_DIR, 'mallorca.tif') @@ -39,14 +42,12 @@ class BotherTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - print('SETTING UP CLASS') if not os.path.exists(TEST_DIR): os.mkdir(TEST_DIR) print(os.listdir()) @classmethod def tearDownClass(cls): - print('TEARING DOWN CLASS') shutil.rmtree(TEST_DIR) def _assert_images_equal(self, im1, im2): @@ -60,8 +61,8 @@ def _assert_image_files_equal(self, fpath1, fpath2): with Image.open(fpath2) as im2: self._assert_images_equal(im1, im2) - def test_1_srtm_tif(self): - """Test the basic downloading of TIF files.""" + def test_01_srtm_tif(self): + """Test the basic creation of TIF files.""" for eg in test_coords: bottom, left, top, right = test_coords[eg] @@ -70,7 +71,7 @@ def test_1_srtm_tif(self): create_tif_file(left, bottom, right, top, os.path.abspath(tif_file)) self._assert_image_files_equal(tif_file, eg_tif_file) - def test_2_resize(self): + def test_02_resize(self): """Test basic cropping and scaling.""" with open(os.path.join(TEST_DIR, 'mallorca.tif'), 'rb') as f: @@ -86,7 +87,7 @@ def test_2_resize(self): with Image.open(os.path.join(EXAMPLES_DIR, 'mallorca_no_processing_resized.png')) as im2: self._assert_images_equal(im1, im2) - def test_3_downsample_lakes_raise_low_flat(self): + def test_03_downsample_lakes_raise_low_flat(self): """Test data resampling, lake detection, raising low pixels and flattening.""" with open(os.path.join(EXAMPLES_DIR, 'ireland.tif'), 'rb') as f: @@ -110,7 +111,7 @@ def test_3_downsample_lakes_raise_low_flat(self): with Image.open(os.path.join(EXAMPLES_DIR, 'ireland_resampled_lakes_low_raised_flat.png')) as im2: self._assert_images_equal(im1, im2) - def test_4_lakes_all_above_sea_epsg_upsample(self): + def test_04_lakes_all_above_sea_epsg_upsample(self): """Test a map where everything is above sea level (including one map where there are lakes). Also tests reprojection and upsampling of data. """ @@ -145,7 +146,7 @@ def test_4_lakes_all_above_sea_epsg_upsample(self): with Image.open(os.path.join(EXAMPLES_DIR, 'alps.png')) as im2: self._assert_images_equal(im1, im2) - def test_5_nested_cache_dir(self): + def test_05_nested_cache_dir(self): """Test cache directory creation when multiple levels of the cache path do not exist.""" bottom, left, top, right = test_coords["mallorca"] @@ -155,7 +156,7 @@ def test_5_nested_cache_dir(self): create_tif_file(left, bottom, right, top, os.path.abspath(tif_file), cache_dir=cache_dir) self._assert_image_files_equal(tif_file, eg_tif_file) - def test_6_raise_undersea(self): + def test_06_raise_undersea(self): """Test raising undersea pixels.""" bottom, left, top, right = test_coords['germany_nw'] @@ -168,5 +169,12 @@ def test_6_raise_undersea(self): with Image.open(os.path.join(EXAMPLES_DIR, 'germany_nw_undersea_raised.png')) as im2: self._assert_images_equal(im1, im2) + def test_07_water_tile(self): + """Test coverage of tile areas with no land.""" + + bottom, left, top, right = vanc_oregon + tif_file = os.path.join(TEST_DIR, 'vanc_oregon.tif') + create_tif_file(left, bottom, right, top, os.path.abspath(tif_file)) + if __name__ == '__main__': unittest.main() diff --git a/test/test_coords.txt b/test/test_coords.txt index 2782ac2..86a13d1 100644 --- a/test/test_coords.txt +++ b/test/test_coords.txt @@ -18,3 +18,6 @@ # Northwest coast, Germany 53.373069 7.387423 53.814826 8.234738 + +# Vancouver Island + Oregon (includes a lot of ocean) +41.82 -129.946 51.468 -116.125