Merge pull request #10 from bunburya/handle_empty_tiles

fixing handling of all-water tile regions

bother_utils/srtm.py

@@ -58,6 +58,13 @@ def get_tif_fpath(x: int, y: int, cache_dir: str) -> str:
 
 def download_zip(url: str, save_path: str, chunk_size: int = 1024):
     r = requests.get(url, stream=True)
+    if not r.ok:
+        if r.status_code == 404:
+            # 404 can be returned if we try fetch a tile that corresponds to
+            # a part of the world with no land, which is okay.
+            return None
+        else:
+            r.raise_for_status()
     r.raise_for_status()
     total_size_in_bytes = int(r.headers.get('content-length', 0))
 
@@ -129,8 +136,12 @@ def fetch_all_zips(fnames: Dict[Tuple[int, int], str], cache_dir: str) -> Dict[T
             print(f'Downloading {fname} from {url}.')
             fpath = download_zip(url, fpath)
             if fpath is None:
-                print(f'Got 404 when downloading {fname}; there is probably no data available.')
-                continue
+                print(f'WARNING: Received HTTP 404 response when attempting to download {fname}.')
+                print('This could mean that (1) this tile corresponds to a region of the earth '
+                      'where there is no land; (2) this tile is outside the coverage area of the '
+                      'SRTM data; or (3) there is some issue with the website (which may be '
+                      'temporary).')
+                print('We will assume the cause is (1) and will proceed accordingly.')
         else:
             print(f'{fname} was cached in {cache_dir}.')
         fpaths[xy] = fpath
@@ -162,7 +173,10 @@ def create_tif_file(left: float, bottom: float, right: float, top: float, to_fil
         zip_fnames[(x, y)] = ZIP_FNAME.format(x=x, y=y)
     zip_fpaths = fetch_all_zips(zip_fnames, cache_dir)
     unzip_all(zip_fpaths.values(), cache_dir)
-    srcs = [rasterio.open(get_tif_fpath(x, y, cache_dir), 'r', nodata=nodata) for x, y in xy]
+    srcs = []
+    for x, y in xy:
+        if zip_fpaths[(x, y)] is not None:
+            srcs.append(rasterio.open(get_tif_fpath(x, y, cache_dir), 'r', nodata=nodata))
     print(f'Creating TIF file from following files: {[s.name for s in srcs]}.')
     #print(f'Heights are: {[s.height for s in srcs]}.')
     #print(f'Widths are: {[s.width for s in srcs]}.')

test/gen_examples.sh

@@ -13,6 +13,7 @@ pipenv run ../bother -ot examples/titicaca.tif -b -16.970092 -70.246331 -15.0323
 pipenv run ../bother -ot examples/constance.tif -b 47.321556 8.830936 47.980224 9.860904 examples/constance.png
 pipenv run ../bother -ot examples/alps.tif -b 46.408240 9.555657 46.975534 10.378602 examples/alps.png
 pipenv run ../bother -ot examples/germany_nw.tif -b 53.373069 7.387423 53.814826 8.234738 examples/germany_nw.png
+#pipenv run ../bother -ot examples/vanc_oregon.tif -b 41.82 -129.946 51.468 -116.125 examples/vanc_oregon.png
 
 # Mallorca: Crop image but don't do any other processing (including no reprojection)
 pipenv run ../bother -it examples/mallorca.tif --epsg 4326 --crop 1574x787 c -si 2048x1024 examples/mallorca_no_processing_resized.png
@@ -32,4 +33,5 @@ pipenv run ../bother --epsg 4326 -l -b -16.970092 -70.246331 -15.032316 -68.3347
 # Germany: Raise undersea pixels and also raise low pixels.
 pipenv run ../bother -rl 0 -ru 1 -b 53.373069 7.387423 53.814826 8.234738 examples/germany_nw_undersea_raised.png
 
-
+# Vancouver & Oregon: Crop and scale
+#pipenv run ../bother -it examples/vanc_oregon.tif --crop 11578x11578 c -si 4000x4000 examples/vanc_oregon_scaled.png

test/test_bother.py

@@ -25,9 +25,12 @@
     'constance': (47.321556, 8.830936, 47.980224, 9.860904),
     'titicaca': (-16.970092, -70.246331, -15.032316, -68.334710),
     'alps': (46.408240, 9.555657, 46.975534, 10.378602),
-    'germany_nw': (53.373069, 7.387423, 53.814826, 8.234738)
+    'germany_nw': (53.373069, 7.387423, 53.814826, 8.234738),
 }
 
+# We don't test this with the rest, because the raw image is too
+# large to compare (PIL crashes)
+vanc_oregon = (41.82, -129.946, 51.468, -116.125)
 
 # No options provided
 MALLORCA_TIF = os.path.join(EXAMPLES_DIR, 'mallorca.tif')
@@ -39,14 +42,12 @@ class BotherTestCase(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        print('SETTING UP CLASS')
         if not os.path.exists(TEST_DIR):
             os.mkdir(TEST_DIR)
         print(os.listdir())
 
     @classmethod
     def tearDownClass(cls):
-        print('TEARING DOWN CLASS')
         shutil.rmtree(TEST_DIR)
 
     def _assert_images_equal(self, im1, im2):
@@ -60,8 +61,8 @@ def _assert_image_files_equal(self, fpath1, fpath2):
             with Image.open(fpath2) as im2:
                 self._assert_images_equal(im1, im2)
 
-    def test_1_srtm_tif(self):
-        """Test the basic downloading of TIF files."""
+    def test_01_srtm_tif(self):
+        """Test the basic creation of TIF files."""
 
         for eg in test_coords:
             bottom, left, top, right = test_coords[eg]
@@ -70,7 +71,7 @@ def test_1_srtm_tif(self):
             create_tif_file(left, bottom, right, top, os.path.abspath(tif_file))
             self._assert_image_files_equal(tif_file, eg_tif_file)
 
-    def test_2_resize(self):
+    def test_02_resize(self):
         """Test basic cropping and scaling."""
 
         with open(os.path.join(TEST_DIR, 'mallorca.tif'), 'rb') as f:
@@ -86,7 +87,7 @@ def test_2_resize(self):
         with Image.open(os.path.join(EXAMPLES_DIR, 'mallorca_no_processing_resized.png')) as im2:
             self._assert_images_equal(im1, im2)
 
-    def test_3_downsample_lakes_raise_low_flat(self):
+    def test_03_downsample_lakes_raise_low_flat(self):
         """Test data resampling, lake detection, raising low pixels and flattening."""
 
         with open(os.path.join(EXAMPLES_DIR, 'ireland.tif'), 'rb') as f:
@@ -110,7 +111,7 @@ def test_3_downsample_lakes_raise_low_flat(self):
         with Image.open(os.path.join(EXAMPLES_DIR, 'ireland_resampled_lakes_low_raised_flat.png')) as im2:
             self._assert_images_equal(im1, im2)
 
-    def test_4_lakes_all_above_sea_epsg_upsample(self):
+    def test_04_lakes_all_above_sea_epsg_upsample(self):
         """Test a map where everything is above sea level (including one map where there are lakes).
         Also tests reprojection and upsampling of data.
         """
@@ -145,7 +146,7 @@ def test_4_lakes_all_above_sea_epsg_upsample(self):
         with Image.open(os.path.join(EXAMPLES_DIR, 'alps.png')) as im2:
             self._assert_images_equal(im1, im2)
 
-    def test_5_nested_cache_dir(self):
+    def test_05_nested_cache_dir(self):
         """Test cache directory creation when multiple levels of the cache path do not exist."""
 
         bottom, left, top, right = test_coords["mallorca"]
@@ -155,7 +156,7 @@ def test_5_nested_cache_dir(self):
         create_tif_file(left, bottom, right, top, os.path.abspath(tif_file), cache_dir=cache_dir)
         self._assert_image_files_equal(tif_file, eg_tif_file)
 
-    def test_6_raise_undersea(self):
+    def test_06_raise_undersea(self):
         """Test raising undersea pixels."""
 
         bottom, left, top, right = test_coords['germany_nw']
@@ -168,5 +169,12 @@ def test_6_raise_undersea(self):
         with Image.open(os.path.join(EXAMPLES_DIR, 'germany_nw_undersea_raised.png')) as im2:
             self._assert_images_equal(im1, im2)
 
+    def test_07_water_tile(self):
+        """Test coverage of tile areas with no land."""
+
+        bottom, left, top, right = vanc_oregon
+        tif_file = os.path.join(TEST_DIR, 'vanc_oregon.tif')
+        create_tif_file(left, bottom, right, top, os.path.abspath(tif_file))
+
 if __name__ == '__main__':
     unittest.main()

test/test_coords.txt

@@ -18,3 +18,6 @@
 
 # Northwest coast, Germany
 53.373069 7.387423 53.814826 8.234738
+
+# Vancouver Island + Oregon (includes a lot of ocean)
+41.82 -129.946 51.468 -116.125