diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c31867bf..1c327f6e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 1.30.6 +### Features + +- Allow gdal to read and rasterize vector formats ([#1763](../../pull/1763)) + ### Improvements - Harden the geojson annotation parser ([#1743](../../pull/1743)) diff --git a/sources/bioformats/large_image_source_bioformats/__init__.py b/sources/bioformats/large_image_source_bioformats/__init__.py index 3913cebc0..5679cf0b4 100644 --- a/sources/bioformats/large_image_source_bioformats/__init__.py +++ b/sources/bioformats/large_image_source_bioformats/__init__.py @@ -62,7 +62,7 @@ # Default to ignoring files with no extension and some specific extensions. -config.ConfigValues['source_bioformats_ignored_names'] = r'(^[^.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi|nd2|ome|nc|json|isyntax|mrxs|zip|zarr(\.db|\.zip)))$' # noqa +config.ConfigValues['source_bioformats_ignored_names'] = r'(^[^.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi|nd2|ome|nc|json|geojson|isyntax|mrxs|zip|zarr(\.db|\.zip)))$' # noqa def _monitor_thread(): diff --git a/sources/gdal/large_image_source_gdal/__init__.py b/sources/gdal/large_image_source_gdal/__init__.py index a5880a8d3..326d30a87 100644 --- a/sources/gdal/large_image_source_gdal/__init__.py +++ b/sources/gdal/large_image_source_gdal/__init__.py @@ -48,6 +48,7 @@ gdal_array = None gdalconst = None osr = None +ogr = None pyproj = None @@ -89,7 +90,9 @@ class GDALFileTileSource(GDALBaseFileTileSource, metaclass=LruCacheMetaclass): cacheName = 'tilesource' name = 'gdal' - def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): + VECTOR_IMAGE_SIZE = 256 * 1024 + + def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): # noqa """ Initialize the tile class. See the base class for other available parameters. @@ -115,8 +118,10 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): self._bounds = {} self._largeImagePath = self._getLargeImagePath() try: - self.dataset = gdal.Open(self._largeImagePath, gdalconst.GA_ReadOnly) - except (RuntimeError, UnicodeDecodeError): + self.dataset = gdal.OpenEx(self._largeImagePath, gdalconst.GA_ReadOnly) + if not self.dataset.RasterCount and self.dataset.GetLayer() is not None: + self.dataset = self._openVectorSource(self.dataset) + except (RuntimeError, UnicodeDecodeError, OverflowError): if not os.path.isfile(self._largeImagePath): raise TileSourceFileNotFoundError(self._largeImagePath) from None msg = 'File cannot be opened via GDAL' @@ -162,6 +167,49 @@ def __init__(self, path, projection=None, unitsPerPixel=None, **kwargs): self._getTileLock = threading.Lock() self._setDefaultStyle() + def _openVectorSource(self, vec): + # This is mainly to speed up back-to-back canRead and open calls + if hasattr(self.__class__, '_openVectorLock') and hasattr(self.__class__, '_lastVectorDS'): + with self.__class__._openVectorLock: + if self.__class__._lastVectorDS[0] == self._largeImagePath: + return self.__class__._lastVectorDS[1] + layer = vec.GetLayer() + x_min, x_max, y_min, y_max = layer.GetExtent() + try: + proj = layer.GetSpatialRef().ExportToWkt() + if (layer.GetSpatialRef().ExportToProj4() == '+proj=longlat +datum=WGS84 +no_defs' and + (max(abs(x_min), abs(x_max)) > 180 or max(abs(y_min), abs(y_max)) > 90)): + proj = None + except Exception: + proj = None + # Define raster parameters + pixel_size = max(x_max - x_min, y_max - y_min) / self.VECTOR_IMAGE_SIZE + if not pixel_size: + msg = 'Cannot determine dimensions' + raise RuntimeError(msg) + if not proj and pixel_size < 1: + pixel_size = 1 + x_res = int((x_max - x_min) / pixel_size) + y_res = int((y_max - y_min) / pixel_size) + + # Create an in-memory raster dataset + with tempfile.NamedTemporaryFile(suffix='.tif', delete=True) as tmpfile: + drv = gdal.GetDriverByName('Gtiff') + ds = drv.Create( + tmpfile.name, x_res, y_res, 1, gdal.GDT_Byte, + options=['COMPRESS=DEFLATE', 'PREDICTOR=2', 'TILED=YES', + 'SPARSE_OK=TRUE', 'BIGTIFF=IF_SAFER']) + # Set raster geotransform and projection + ds.SetGeoTransform((x_min, pixel_size, 0, y_min, 0, pixel_size)) + if proj: + ds.SetProjection(proj) + gdal.RasterizeLayer(ds, [1], layer, burn_values=[255]) + if not hasattr(self.__class__, '_openVectorLock'): + self.__class__._openVectorLock = threading.RLock() + with self.__class__._openVectorLock: + self.__class__._lastVectorDS = (self._largeImagePath, ds) + return ds + def _getDriver(self): """ Get the GDAL driver used to read this dataset. @@ -1000,7 +1048,7 @@ def addKnownExtensions(cls): cls.mimeTypes = cls.mimeTypes.copy() for idx in range(gdal.GetDriverCount()): drv = gdal.GetDriver(idx) - if drv.GetMetadataItem(gdal.DCAP_RASTER): + if drv.GetMetadataItem(gdal.DCAP_RASTER) or drv.GetMetadataItem(gdal.DCAP_VECTOR): drvexts = drv.GetMetadataItem(gdal.DMD_EXTENSIONS) if drvexts is not None: for ext in drvexts.split(): diff --git a/sources/multi/large_image_source_multi/__init__.py b/sources/multi/large_image_source_multi/__init__.py index 3b883f3c9..395c75b52 100644 --- a/sources/multi/large_image_source_multi/__init__.py +++ b/sources/multi/large_image_source_multi/__init__.py @@ -457,7 +457,8 @@ def __init__(self, path, **kwargs): # noqa try: with builtins.open(self._largeImagePath) as fptr: start = fptr.read(1024).strip() - if start[:1] not in ('{', '#', '-') and (start[:1] < 'a' or start[:1] > 'z'): + if (start[:1] not in ('{', '#', '-') and + (start[:1] < 'a' or start[:1] > 'z')) or 'FeatureCollection' in start: msg = 'File cannot be opened via multi-source reader.' raise TileSourceError(msg) fptr.seek(0) diff --git a/test/test_files/sample.geojson b/test/test_files/sample.geojson new file mode 100644 index 000000000..0e82b2447 --- /dev/null +++ b/test/test_files/sample.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8441,18758,0],[8441,18759,0],[8443,18759,0],[8443,18760,0],[8445,18760,0],[8445,18761,0],[8447,18761,0],[8447,18762,0],[8448,18762,0],[8448,18763,0],[8449,18763,0],[8449,18764,0],[8450,18764,0],[8450,18766,0],[8451,18766,0],[8451,18767,0],[8452,18767,0],[8452,18768,0],[8453,18768,0],[8453,18769,0],[8455,18769,0],[8455,18771,0],[8455,18773,0],[8456,18773,0],[8456,18775,0],[8457,18775,0],[8457,18776,0],[8456,18776,0],[8456,18778,0],[8455,18778,0],[8455,18780,0],[8454,18780,0],[8454,18782,0],[8453,18782,0],[8453,18783,0],[8451,18783,0],[8451,18784,0],[8449,18784,0],[8449,18785,0],[8448,18785,0],[8448,18786,0],[8446,18786,0],[8446,18787,0],[8444,18787,0],[8444,18786,0],[8442,18786,0],[8442,18784,0],[8441,18784,0],[8441,18783,0],[8440,18783,0],[8440,18781,0],[8439,18781,0],[8439,18777,0],[8438,18777,0],[8438,18775,0],[8438,18773,0],[8437,18773,0],[8437,18771,0],[8435,18771,0],[8435,18770,0],[8434,18770,0],[8434,18769,0],[8433,18769,0],[8433,18767,0],[8432,18767,0],[8432,18765,0],[8433,18765,0],[8433,18763,0],[8432,18763,0],[8432,18761,0],[8433,18761,0],[8433,18759,0],[8435,18759,0],[8435,18758,0],[8441,18758,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa1b","type":"polyline","annotation":{"attributes":{"cli":"NucleiDetection","params":{"ImageInversionForm":"default","analysis_mag":20,"analysis_roi":[7675,18750,1223,860],"analysis_tile_size":1024,"foreground_threshold":60,"frame":null,"ignore_border_nuclei":false,"inputImageFile":"/mnt/girder_direct_worker/TCGA-02-0010-01Z-00-DX4.07de2e55-a8fe-40ee-9e98-bcb78050b9f7.svs","local_max_search_radius":10,"max_radius":20,"min_fgnd_frac":0.25,"min_nucleus_area":80,"min_radius":6,"nuclei_annotation_format":"boundary","num_threads_per_worker":1,"num_workers":-1,"outputNucleiAnnotationFile":"/mnt/girder_worker/522dea4885fa48d98b46dcff958823d7/Detects Nuclei-outputNucleiAnnotationFile.anot","reference_mu_lab":[8.63234435,-0.11501964,0.03868433],"reference_std_lab":[0.57506023,0.10403329,0.01364062],"remove_overlapping_nuclei_segmentation":true,"scheduler":"","stain_1":"hematoxylin","stain_1_vector":[-1,-1,-1],"stain_2":"eosin","stain_2_vector":[-1,-1,-1],"stain_3":"null","stain_3_vector":[-1,-1,-1],"style":null,"tile_overlap_value":-1},"src_mu_lab":[8.937002081446192,-0.05466739468053563,0.05831938659685288],"src_sigma_lab":[0.4090189135708649,0.07980666511846066,0.03601404918773042],"version":"1.3.9.dev2"},"name":"Detects Nuclei-outputNucleiAnnotationFile-nuclei-boundary"}}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8618,18777,0],[8620,18777,0],[8620,18778,0],[8621,18778,0],[8621,18780,0],[8622,18780,0],[8622,18782,0],[8623,18782,0],[8623,18783,0],[8622,18783,0],[8622,18786,0],[8623,18786,0],[8623,18788,0],[8622,18788,0],[8622,18796,0],[8621,18796,0],[8621,18799,0],[8620,18799,0],[8620,18801,0],[8619,18801,0],[8619,18806,0],[8620,18806,0],[8620,18808,0],[8620,18810,0],[8618,18810,0],[8618,18811,0],[8616,18811,0],[8616,18812,0],[8615,18812,0],[8615,18811,0],[8613,18811,0],[8613,18810,0],[8612,18810,0],[8612,18808,0],[8612,18797,0],[8613,18797,0],[8613,18792,0],[8612,18792,0],[8612,18790,0],[8610,18790,0],[8610,18788,0],[8609,18788,0],[8609,18787,0],[8610,18787,0],[8610,18785,0],[8612,18785,0],[8612,18784,0],[8614,18784,0],[8614,18782,0],[8615,18782,0],[8615,18779,0],[8616,18779,0],[8616,18777,0],[8618,18777,0],[8618,18777,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa3c","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8573,18770,0],[8575,18770,0],[8575,18772,0],[8576,18772,0],[8576,18776,0],[8577,18776,0],[8577,18778,0],[8578,18778,0],[8578,18779,0],[8580,18779,0],[8580,18780,0],[8581,18780,0],[8581,18782,0],[8582,18782,0],[8582,18783,0],[8581,18783,0],[8581,18785,0],[8580,18785,0],[8580,18788,0],[8579,18788,0],[8579,18790,0],[8578,18790,0],[8578,18792,0],[8577,18792,0],[8577,18795,0],[8577,18797,0],[8575,18797,0],[8575,18798,0],[8574,18798,0],[8574,18797,0],[8572,18797,0],[8572,18796,0],[8571,18796,0],[8571,18794,0],[8571,18792,0],[8570,18792,0],[8570,18790,0],[8569,18790,0],[8569,18787,0],[8568,18787,0],[8568,18782,0],[8569,18782,0],[8569,18780,0],[8568,18780,0],[8568,18777,0],[8569,18777,0],[8569,18775,0],[8570,18775,0],[8570,18772,0],[8571,18772,0],[8571,18770,0],[8573,18770,0],[8573,18770,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa3a","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8461,18806,0],[8461,18807,0],[8463,18807,0],[8463,18809,0],[8464,18809,0],[8464,18810,0],[8463,18810,0],[8463,18813,0],[8464,18813,0],[8464,18815,0],[8466,18815,0],[8466,18817,0],[8467,18817,0],[8467,18820,0],[8468,18820,0],[8468,18825,0],[8469,18825,0],[8469,18826,0],[8468,18826,0],[8468,18828,0],[8467,18828,0],[8467,18829,0],[8465,18829,0],[8465,18830,0],[8463,18830,0],[8463,18831,0],[8459,18831,0],[8459,18830,0],[8457,18830,0],[8457,18829,0],[8456,18829,0],[8456,18827,0],[8455,18827,0],[8455,18825,0],[8454,18825,0],[8454,18824,0],[8455,18824,0],[8455,18822,0],[8454,18822,0],[8454,18818,0],[8454,18810,0],[8455,18810,0],[8455,18808,0],[8456,18808,0],[8456,18807,0],[8458,18807,0],[8458,18806,0],[8461,18806,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa2a","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8535,18797,0],[8535,18798,0],[8537,18798,0],[8537,18799,0],[8538,18799,0],[8538,18800,0],[8540,18800,0],[8540,18802,0],[8540,18804,0],[8539,18804,0],[8539,18806,0],[8538,18806,0],[8538,18809,0],[8537,18809,0],[8537,18811,0],[8538,18811,0],[8538,18813,0],[8539,18813,0],[8539,18814,0],[8541,18814,0],[8541,18815,0],[8544,18815,0],[8544,18816,0],[8542,18816,0],[8542,18817,0],[8539,18817,0],[8539,18818,0],[8535,18818,0],[8535,18819,0],[8533,18819,0],[8533,18818,0],[8532,18818,0],[8532,18816,0],[8532,18814,0],[8531,18814,0],[8531,18812,0],[8530,18812,0],[8530,18810,0],[8529,18810,0],[8529,18809,0],[8528,18809,0],[8528,18807,0],[8527,18807,0],[8527,18805,0],[8528,18805,0],[8528,18803,0],[8530,18803,0],[8530,18802,0],[8531,18802,0],[8531,18800,0],[8532,18800,0],[8532,18798,0],[8534,18798,0],[8534,18797,0],[8535,18797,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa3e","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8364,18815,0],[8364,18816,0],[8366,18816,0],[8366,18817,0],[8368,18817,0],[8368,18818,0],[8369,18818,0],[8369,18820,0],[8370,18820,0],[8370,18821,0],[8372,18821,0],[8372,18822,0],[8373,18822,0],[8373,18824,0],[8374,18824,0],[8374,18826,0],[8375,18826,0],[8375,18830,0],[8374,18830,0],[8374,18832,0],[8373,18832,0],[8373,18833,0],[8371,18833,0],[8371,18834,0],[8370,18834,0],[8370,18835,0],[8368,18835,0],[8366,18835,0],[8366,18834,0],[8365,18834,0],[8365,18833,0],[8364,18833,0],[8364,18831,0],[8363,18831,0],[8363,18830,0],[8362,18830,0],[8362,18829,0],[8360,18829,0],[8360,18828,0],[8358,18828,0],[8358,18827,0],[8357,18827,0],[8357,18825,0],[8356,18825,0],[8356,18821,0],[8357,18821,0],[8357,18819,0],[8358,18819,0],[8358,18818,0],[8359,18818,0],[8359,18817,0],[8360,18817,0],[8360,18816,0],[8362,18816,0],[8362,18815,0],[8364,18815,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa2c","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8613,18851,0],[8613,18852,0],[8615,18852,0],[8615,18853,0],[8616,18853,0],[8616,18854,0],[8617,18854,0],[8617,18856,0],[8618,18856,0],[8618,18857,0],[8619,18857,0],[8619,18858,0],[8620,18858,0],[8620,18860,0],[8621,18860,0],[8621,18863,0],[8622,18863,0],[8622,18866,0],[8623,18866,0],[8623,18867,0],[8622,18867,0],[8622,18869,0],[8620,18869,0],[8618,18869,0],[8618,18867,0],[8616,18867,0],[8616,18866,0],[8614,18866,0],[8614,18867,0],[8612,18867,0],[8612,18866,0],[8610,18866,0],[8610,18865,0],[8608,18865,0],[8608,18864,0],[8607,18864,0],[8607,18862,0],[8606,18862,0],[8606,18859,0],[8607,18859,0],[8607,18856,0],[8608,18856,0],[8608,18854,0],[8609,18854,0],[8609,18853,0],[8610,18853,0],[8610,18852,0],[8612,18852,0],[8612,18851,0],[8613,18851,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fb1e","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8547,18815,0],[8547,18816,0],[8549,18816,0],[8549,18818,0],[8549,18821,0],[8549,18823,0],[8548,18823,0],[8548,18825,0],[8546,18825,0],[8546,18826,0],[8544,18826,0],[8544,18827,0],[8542,18827,0],[8542,18828,0],[8540,18828,0],[8540,18829,0],[8539,18829,0],[8539,18828,0],[8537,18828,0],[8537,18829,0],[8535,18829,0],[8535,18828,0],[8533,18828,0],[8533,18826,0],[8532,18826,0],[8532,18825,0],[8533,18825,0],[8533,18823,0],[8532,18823,0],[8532,18821,0],[8533,18821,0],[8533,18820,0],[8536,18820,0],[8536,18819,0],[8540,18819,0],[8540,18818,0],[8543,18818,0],[8543,18817,0],[8545,18817,0],[8545,18816,0],[8546,18816,0],[8546,18815,0],[8547,18815,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa41","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8562,18816,0],[8562,18817,0],[8564,18817,0],[8564,18818,0],[8566,18818,0],[8568,18818,0],[8568,18820,0],[8569,18820,0],[8569,18823,0],[8568,18823,0],[8568,18825,0],[8567,18825,0],[8567,18827,0],[8566,18827,0],[8566,18828,0],[8564,18828,0],[8564,18829,0],[8562,18829,0],[8562,18830,0],[8561,18830,0],[8561,18829,0],[8559,18829,0],[8559,18828,0],[8558,18828,0],[8558,18826,0],[8557,18826,0],[8557,18824,0],[8556,18824,0],[8556,18822,0],[8557,18822,0],[8557,18820,0],[8558,18820,0],[8558,18819,0],[8559,18819,0],[8559,18817,0],[8561,18817,0],[8561,18816,0],[8562,18816,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa43","type":"polyline"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[8385,18750,0],[8385,18754,0],[8384,18754,0],[8384,18756,0],[8382,18756,0],[8382,18757,0],[8380,18757,0],[8380,18756,0],[8378,18756,0],[8378,18757,0],[8377,18757,0],[8377,18756,0],[8375,18756,0],[8375,18754,0],[8374,18754,0],[8374,18751,0],[8375,18751,0],[8375,18750,0],[8385,18750,0]]]},"properties":{"fillColor":"rgba(0,0,0,0)","lineColor":"rgb(0,255,0)","id":"667c7bd58ca8074b3194fa17","type":"polyline"}}]} diff --git a/test/test_source_gdal.py b/test/test_source_gdal.py index b606c9672..0e98e57eb 100644 --- a/test/test_source_gdal.py +++ b/test/test_source_gdal.py @@ -142,3 +142,10 @@ def testGetTiledRegionWithoutResample(self): assert tileMetadata['bounds']['ymin'] == pytest.approx(2149548, 1) assert '+proj=aea' in tileMetadata['bounds']['srs'] region.unlink() + + def testVectorFormat(self): + testDir = os.path.dirname(os.path.realpath(__file__)) + imagePath = os.path.join(testDir, 'test_files', 'sample.geojson') + source = self.basemodule.open(imagePath) + assert source.sizeX == 267 + assert source.sizeY == 119