Skip to content

Commit

Permalink
Merge pull request #1763 from girder/gdal-vector-formats
Browse files Browse the repository at this point in the history
Allow gdal to read and rasterize vector formats
  • Loading branch information
manthey authored Jan 8, 2025
2 parents f0e2624 + f95a0de commit 2e0f0b4
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 6 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

## 1.30.6

### Features

- Allow gdal to read and rasterize vector formats ([#1763](../../pull/1763))

### Improvements

- Harden the geojson annotation parser ([#1743](../../pull/1743))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@


# Default to ignoring files with no extension and some specific extensions.
config.ConfigValues['source_bioformats_ignored_names'] = r'(^[^.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi|nd2|ome|nc|json|isyntax|mrxs|zip|zarr(\.db|\.zip)))$' # noqa
config.ConfigValues['source_bioformats_ignored_names'] = r'(^[^.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi|nd2|ome|nc|json|geojson|isyntax|mrxs|zip|zarr(\.db|\.zip)))$' # noqa


def _monitor_thread():
Expand Down
56 changes: 52 additions & 4 deletions sources/gdal/large_image_source_gdal/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
gdal_array = None
gdalconst = None
osr = None
ogr = None
pyproj = None


Expand Down Expand Up @@ -89,7 +90,9 @@ class GDALFileTileSource(GDALBaseFileTileSource, metaclass=LruCacheMetaclass):
cacheName = 'tilesource'
name = 'gdal'

def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs):
VECTOR_IMAGE_SIZE = 256 * 1024

def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): # noqa
"""
Initialize the tile class. See the base class for other available
parameters.
Expand All @@ -115,8 +118,10 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs):
self._bounds = {}
self._largeImagePath = self._getLargeImagePath()
try:
self.dataset = gdal.Open(self._largeImagePath, gdalconst.GA_ReadOnly)
except (RuntimeError, UnicodeDecodeError):
self.dataset = gdal.OpenEx(self._largeImagePath, gdalconst.GA_ReadOnly)
if not self.dataset.RasterCount and self.dataset.GetLayer() is not None:
self.dataset = self._openVectorSource(self.dataset)
except (RuntimeError, UnicodeDecodeError, OverflowError):
if not os.path.isfile(self._largeImagePath):
raise TileSourceFileNotFoundError(self._largeImagePath) from None
msg = 'File cannot be opened via GDAL'
Expand Down Expand Up @@ -162,6 +167,49 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs):
self._getTileLock = threading.Lock()
self._setDefaultStyle()

def _openVectorSource(self, vec):
# This is mainly to speed up back-to-back canRead and open calls
if hasattr(self.__class__, '_openVectorLock') and hasattr(self.__class__, '_lastVectorDS'):
with self.__class__._openVectorLock:
if self.__class__._lastVectorDS[0] == self._largeImagePath:
return self.__class__._lastVectorDS[1]
layer = vec.GetLayer()
x_min, x_max, y_min, y_max = layer.GetExtent()
try:
proj = layer.GetSpatialRef().ExportToWkt()
if (layer.GetSpatialRef().ExportToProj4() == '+proj=longlat +datum=WGS84 +no_defs' and
(max(abs(x_min), abs(x_max)) > 180 or max(abs(y_min), abs(y_max)) > 90)):
proj = None
except Exception:
proj = None
# Define raster parameters
pixel_size = max(x_max - x_min, y_max - y_min) / self.VECTOR_IMAGE_SIZE
if not pixel_size:
msg = 'Cannot determine dimensions'
raise RuntimeError(msg)
if not proj and pixel_size < 1:
pixel_size = 1
x_res = int((x_max - x_min) / pixel_size)
y_res = int((y_max - y_min) / pixel_size)

# Create an in-memory raster dataset
with tempfile.NamedTemporaryFile(suffix='.tif', delete=True) as tmpfile:
drv = gdal.GetDriverByName('Gtiff')
ds = drv.Create(
tmpfile.name, x_res, y_res, 1, gdal.GDT_Byte,
options=['COMPRESS=DEFLATE', 'PREDICTOR=2', 'TILED=YES',
'SPARSE_OK=TRUE', 'BIGTIFF=IF_SAFER'])
# Set raster geotransform and projection
ds.SetGeoTransform((x_min, pixel_size, 0, y_min, 0, pixel_size))
if proj:
ds.SetProjection(proj)
gdal.RasterizeLayer(ds, [1], layer, burn_values=[255])
if not hasattr(self.__class__, '_openVectorLock'):
self.__class__._openVectorLock = threading.RLock()
with self.__class__._openVectorLock:
self.__class__._lastVectorDS = (self._largeImagePath, ds)
return ds

def _getDriver(self):
"""
Get the GDAL driver used to read this dataset.
Expand Down Expand Up @@ -1000,7 +1048,7 @@ def addKnownExtensions(cls):
cls.mimeTypes = cls.mimeTypes.copy()
for idx in range(gdal.GetDriverCount()):
drv = gdal.GetDriver(idx)
if drv.GetMetadataItem(gdal.DCAP_RASTER):
if drv.GetMetadataItem(gdal.DCAP_RASTER) or drv.GetMetadataItem(gdal.DCAP_VECTOR):
drvexts = drv.GetMetadataItem(gdal.DMD_EXTENSIONS)
if drvexts is not None:
for ext in drvexts.split():
Expand Down
3 changes: 2 additions & 1 deletion sources/multi/large_image_source_multi/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,8 @@ def __init__(self, path, **kwargs): # noqa
try:
with builtins.open(self._largeImagePath) as fptr:
start = fptr.read(1024).strip()
if start[:1] not in ('{', '#', '-') and (start[:1] < 'a' or start[:1] > 'z'):
if (start[:1] not in ('{', '#', '-') and
(start[:1] < 'a' or start[:1] > 'z')) or 'FeatureCollection' in start:
msg = 'File cannot be opened via multi-source reader.'
raise TileSourceError(msg)
fptr.seek(0)
Expand Down
1 change: 1 addition & 0 deletions test/test_files/sample.geojson
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8441,18758,0],[8441,18759,0],[8443,18759,0],[8443,18760,0],[8445,18760,0],[8445,18761,0],[8447,18761,0],[8447,18762,0],[8448,18762,0],[8448,18763,0],[8449,18763,0],[8449,18764,0],[8450,18764,0],[8450,18766,0],[8451,18766,0],[8451,18767,0],[8452,18767,0],[8452,18768,0],[8453,18768,0],[8453,18769,0],[8455,18769,0],[8455,18771,0],[8455,18773,0],[8456,18773,0],[8456,18775,0],[8457,18775,0],[8457,18776,0],[8456,18776,0],[8456,18778,0],[8455,18778,0],[8455,18780,0],[8454,18780,0],[8454,18782,0],[8453,18782,0],[8453,18783,0],[8451,18783,0],[8451,18784,0],[8449,18784,0],[8449,18785,0],[8448,18785,0],[8448,18786,0],[8446,18786,0],[8446,18787,0],[8444,18787,0],[8444,18786,0],[8442,18786,0],[8442,18784,0],[8441,18784,0],[8441,18783,0],[8440,18783,0],[8440,18781,0],[8439,18781,0],[8439,18777,0],[8438,18777,0],[8438,18775,0],[8438,18773,0],[8437,18773,0],[8437,18771,0],[8435,18771,0],[8435,18770,0],[8434,18770,0],[8434,18769,0],[8433,18769,0],[8433,18767,0],[8432,18767,0],[8432,18765,0],[8433,18765,0],[8433,18763,0],[8432,18763,0],[8432,18761,0],[8433,18761,0],[8433,18759,0],[8435,18759,0],[8435,18758,0],[8441,18758,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa1b","type":"polyline","annotation":{"attributes":{"cli":"NucleiDetection","params":{"ImageInversionForm":"default","analysis_mag":20,"analysis_roi":[7675,18750,1223,860],"analysis_tile_size":1024,"foreground_threshold":60,"frame":null,"ignore_border_nuclei":false,"inputImageFile":"/mnt/girder_direct_worker/TCGA-02-0010-01Z-00-DX4.07de2e55-a8fe-40ee-9e98-bcb78050b9f7.svs","local_max_search_radius":10,"max_radius":20,"min_fgnd_frac":0.25,"min_nucleus_area":80,"min_radius":6,"nuclei_annotation_format":"boundary","num_threads_per_worker":1,"num_workers":-1,"outputNucleiAnnotationFile":"/mnt/girder_worker/522dea4885fa48d98b46dcff958823d7/Detects Nuclei-outputNucleiAnnotationFile.anot","reference_mu_lab":[8.63234435,-0.11501964,0.03868433],"reference_std_lab":[0.57506023,0.10403329,0.01364062],"remove_overlapping_nuclei_segmentation":true,"scheduler":"","stain_1":"hematoxylin","stain_1_vector":[-1,-1,-1],"stain_2":"eosin","stain_2_vector":[-1,-1,-1],"stain_3":"null","stain_3_vector":[-1,-1,-1],"style":null,"tile_overlap_value":-1},"src_mu_lab":[8.937002081446192,-0.05466739468053563,0.05831938659685288],"src_sigma_lab":[0.4090189135708649,0.07980666511846066,0.03601404918773042],"version":"1.3.9.dev2"},"name":"Detects Nuclei-outputNucleiAnnotationFile-nuclei-boundary"}}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8618,18777,0],[8620,18777,0],[8620,18778,0],[8621,18778,0],[8621,18780,0],[8622,18780,0],[8622,18782,0],[8623,18782,0],[8623,18783,0],[8622,18783,0],[8622,18786,0],[8623,18786,0],[8623,18788,0],[8622,18788,0],[8622,18796,0],[8621,18796,0],[8621,18799,0],[8620,18799,0],[8620,18801,0],[8619,18801,0],[8619,18806,0],[8620,18806,0],[8620,18808,0],[8620,18810,0],[8618,18810,0],[8618,18811,0],[8616,18811,0],[8616,18812,0],[8615,18812,0],[8615,18811,0],[8613,18811,0],[8613,18810,0],[8612,18810,0],[8612,18808,0],[8612,18797,0],[8613,18797,0],[8613,18792,0],[8612,18792,0],[8612,18790,0],[8610,18790,0],[8610,18788,0],[8609,18788,0],[8609,18787,0],[8610,18787,0],[8610,18785,0],[8612,18785,0],[8612,18784,0],[8614,18784,0],[8614,18782,0],[8615,18782,0],[8615,18779,0],[8616,18779,0],[8616,18777,0],[8618,18777,0],[8618,18777,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa3c","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8573,18770,0],[8575,18770,0],[8575,18772,0],[8576,18772,0],[8576,18776,0],[8577,18776,0],[8577,18778,0],[8578,18778,0],[8578,18779,0],[8580,18779,0],[8580,18780,0],[8581,18780,0],[8581,18782,0],[8582,18782,0],[8582,18783,0],[8581,18783,0],[8581,18785,0],[8580,18785,0],[8580,18788,0],[8579,18788,0],[8579,18790,0],[8578,18790,0],[8578,18792,0],[8577,18792,0],[8577,18795,0],[8577,18797,0],[8575,18797,0],[8575,18798,0],[8574,18798,0],[8574,18797,0],[8572,18797,0],[8572,18796,0],[8571,18796,0],[8571,18794,0],[8571,18792,0],[8570,18792,0],[8570,18790,0],[8569,18790,0],[8569,18787,0],[8568,18787,0],[8568,18782,0],[8569,18782,0],[8569,18780,0],[8568,18780,0],[8568,18777,0],[8569,18777,0],[8569,18775,0],[8570,18775,0],[8570,18772,0],[8571,18772,0],[8571,18770,0],[8573,18770,0],[8573,18770,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa3a","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8461,18806,0],[8461,18807,0],[8463,18807,0],[8463,18809,0],[8464,18809,0],[8464,18810,0],[8463,18810,0],[8463,18813,0],[8464,18813,0],[8464,18815,0],[8466,18815,0],[8466,18817,0],[8467,18817,0],[8467,18820,0],[8468,18820,0],[8468,18825,0],[8469,18825,0],[8469,18826,0],[8468,18826,0],[8468,18828,0],[8467,18828,0],[8467,18829,0],[8465,18829,0],[8465,18830,0],[8463,18830,0],[8463,18831,0],[8459,18831,0],[8459,18830,0],[8457,18830,0],[8457,18829,0],[8456,18829,0],[8456,18827,0],[8455,18827,0],[8455,18825,0],[8454,18825,0],[8454,18824,0],[8455,18824,0],[8455,18822,0],[8454,18822,0],[8454,18818,0],[8454,18810,0],[8455,18810,0],[8455,18808,0],[8456,18808,0],[8456,18807,0],[8458,18807,0],[8458,18806,0],[8461,18806,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa2a","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8535,18797,0],[8535,18798,0],[8537,18798,0],[8537,18799,0],[8538,18799,0],[8538,18800,0],[8540,18800,0],[8540,18802,0],[8540,18804,0],[8539,18804,0],[8539,18806,0],[8538,18806,0],[8538,18809,0],[8537,18809,0],[8537,18811,0],[8538,18811,0],[8538,18813,0],[8539,18813,0],[8539,18814,0],[8541,18814,0],[8541,18815,0],[8544,18815,0],[8544,18816,0],[8542,18816,0],[8542,18817,0],[8539,18817,0],[8539,18818,0],[8535,18818,0],[8535,18819,0],[8533,18819,0],[8533,18818,0],[8532,18818,0],[8532,18816,0],[8532,18814,0],[8531,18814,0],[8531,18812,0],[8530,18812,0],[8530,18810,0],[8529,18810,0],[8529,18809,0],[8528,18809,0],[8528,18807,0],[8527,18807,0],[8527,18805,0],[8528,18805,0],[8528,18803,0],[8530,18803,0],[8530,18802,0],[8531,18802,0],[8531,18800,0],[8532,18800,0],[8532,18798,0],[8534,18798,0],[8534,18797,0],[8535,18797,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa3e","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8364,18815,0],[8364,18816,0],[8366,18816,0],[8366,18817,0],[8368,18817,0],[8368,18818,0],[8369,18818,0],[8369,18820,0],[8370,18820,0],[8370,18821,0],[8372,18821,0],[8372,18822,0],[8373,18822,0],[8373,18824,0],[8374,18824,0],[8374,18826,0],[8375,18826,0],[8375,18830,0],[8374,18830,0],[8374,18832,0],[8373,18832,0],[8373,18833,0],[8371,18833,0],[8371,18834,0],[8370,18834,0],[8370,18835,0],[8368,18835,0],[8366,18835,0],[8366,18834,0],[8365,18834,0],[8365,18833,0],[8364,18833,0],[8364,18831,0],[8363,18831,0],[8363,18830,0],[8362,18830,0],[8362,18829,0],[8360,18829,0],[8360,18828,0],[8358,18828,0],[8358,18827,0],[8357,18827,0],[8357,18825,0],[8356,18825,0],[8356,18821,0],[8357,18821,0],[8357,18819,0],[8358,18819,0],[8358,18818,0],[8359,18818,0],[8359,18817,0],[8360,18817,0],[8360,18816,0],[8362,18816,0],[8362,18815,0],[8364,18815,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa2c","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8613,18851,0],[8613,18852,0],[8615,18852,0],[8615,18853,0],[8616,18853,0],[8616,18854,0],[8617,18854,0],[8617,18856,0],[8618,18856,0],[8618,18857,0],[8619,18857,0],[8619,18858,0],[8620,18858,0],[8620,18860,0],[8621,18860,0],[8621,18863,0],[8622,18863,0],[8622,18866,0],[8623,18866,0],[8623,18867,0],[8622,18867,0],[8622,18869,0],[8620,18869,0],[8618,18869,0],[8618,18867,0],[8616,18867,0],[8616,18866,0],[8614,18866,0],[8614,18867,0],[8612,18867,0],[8612,18866,0],[8610,18866,0],[8610,18865,0],[8608,18865,0],[8608,18864,0],[8607,18864,0],[8607,18862,0],[8606,18862,0],[8606,18859,0],[8607,18859,0],[8607,18856,0],[8608,18856,0],[8608,18854,0],[8609,18854,0],[8609,18853,0],[8610,18853,0],[8610,18852,0],[8612,18852,0],[8612,18851,0],[8613,18851,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fb1e","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8547,18815,0],[8547,18816,0],[8549,18816,0],[8549,18818,0],[8549,18821,0],[8549,18823,0],[8548,18823,0],[8548,18825,0],[8546,18825,0],[8546,18826,0],[8544,18826,0],[8544,18827,0],[8542,18827,0],[8542,18828,0],[8540,18828,0],[8540,18829,0],[8539,18829,0],[8539,18828,0],[8537,18828,0],[8537,18829,0],[8535,18829,0],[8535,18828,0],[8533,18828,0],[8533,18826,0],[8532,18826,0],[8532,18825,0],[8533,18825,0],[8533,18823,0],[8532,18823,0],[8532,18821,0],[8533,18821,0],[8533,18820,0],[8536,18820,0],[8536,18819,0],[8540,18819,0],[8540,18818,0],[8543,18818,0],[8543,18817,0],[8545,18817,0],[8545,18816,0],[8546,18816,0],[8546,18815,0],[8547,18815,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa41","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8562,18816,0],[8562,18817,0],[8564,18817,0],[8564,18818,0],[8566,18818,0],[8568,18818,0],[8568,18820,0],[8569,18820,0],[8569,18823,0],[8568,18823,0],[8568,18825,0],[8567,18825,0],[8567,18827,0],[8566,18827,0],[8566,18828,0],[8564,18828,0],[8564,18829,0],[8562,18829,0],[8562,18830,0],[8561,18830,0],[8561,18829,0],[8559,18829,0],[8559,18828,0],[8558,18828,0],[8558,18826,0],[8557,18826,0],[8557,18824,0],[8556,18824,0],[8556,18822,0],[8557,18822,0],[8557,18820,0],[8558,18820,0],[8558,18819,0],[8559,18819,0],[8559,18817,0],[8561,18817,0],[8561,18816,0],[8562,18816,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa43","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8385,18750,0],[8385,18754,0],[8384,18754,0],[8384,18756,0],[8382,18756,0],[8382,18757,0],[8380,18757,0],[8380,18756,0],[8378,18756,0],[8378,18757,0],[8377,18757,0],[8377,18756,0],[8375,18756,0],[8375,18754,0],[8374,18754,0],[8374,18751,0],[8375,18751,0],[8375,18750,0],[8385,18750,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa17","type":"polyline"}}]}
7 changes: 7 additions & 0 deletions test/test_source_gdal.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,10 @@ def testGetTiledRegionWithoutResample(self):
assert tileMetadata['bounds']['ymin'] == pytest.approx(2149548, 1)
assert '+proj=aea' in tileMetadata['bounds']['srs']
region.unlink()

def testVectorFormat(self):
testDir = os.path.dirname(os.path.realpath(__file__))
imagePath = os.path.join(testDir, 'test_files', 'sample.geojson')
source = self.basemodule.open(imagePath)
assert source.sizeX == 267
assert source.sizeY == 119

0 comments on commit 2e0f0b4

Please sign in to comment.