From 43e483fb128fc8695efdcf5df87bcb36549be1f3 Mon Sep 17 00:00:00 2001 From: "milos.colic" Date: Wed, 4 Oct 2023 18:12:46 +0100 Subject: [PATCH] Revise all RST_ expressions. Clean up raster disposing. Clean up duplicated expressions. --- .../mosaic/core/geometry/MosaicGeometry.scala | 26 +++- .../MosaicGeometryCollectionESRI.scala | 2 +- .../operator/retile/BalancedSubdivision.scala | 6 +- .../operator/retile/OverlappingTiles.scala | 48 +++++++ .../operator/retile/RasterTessellate.scala | 28 +++-- .../core/raster/operator/retile/ReTile.scala | 29 ++--- .../mosaic/datasource/gdal/ReTileOnRead.scala | 5 +- .../expressions/raster/RST_BandMetaData.scala | 16 +-- .../expressions/raster/RST_BoundingBox.scala | 73 +++++++++++ .../mosaic/expressions/raster/RST_Clip.scala | 22 +--- .../expressions/raster/RST_GeoReference.scala | 22 ++-- .../expressions/raster/RST_GridTiles.scala | 118 ------------------ .../expressions/raster/RST_Height.scala | 11 +- .../expressions/raster/RST_IsEmpty.scala | 7 +- .../expressions/raster/RST_MemSize.scala | 9 +- .../mosaic/expressions/raster/RST_Merge.scala | 9 +- .../expressions/raster/RST_MergeBands.scala | 9 +- .../expressions/raster/RST_MetaData.scala | 11 +- .../mosaic/expressions/raster/RST_NDVI.scala | 32 ++--- .../expressions/raster/RST_NumBands.scala | 9 +- .../expressions/raster/RST_PixelHeight.scala | 1 + .../expressions/raster/RST_PixelWidth.scala | 1 + .../raster/RST_RasterToWorldCoord.scala | 7 +- .../raster/RST_RasterToWorldCoordX.scala | 4 +- .../raster/RST_RasterToWorldCoordY.scala | 4 +- .../expressions/raster/RST_ReTile.scala | 5 +- .../expressions/raster/RST_Rotation.scala | 7 +- .../mosaic/expressions/raster/RST_SRID.scala | 7 +- .../expressions/raster/RST_ScaleX.scala | 9 +- .../expressions/raster/RST_ScaleY.scala | 7 +- .../mosaic/expressions/raster/RST_SkewX.scala | 7 +- .../mosaic/expressions/raster/RST_SkewY.scala | 7 +- .../expressions/raster/RST_Subdatasets.scala | 17 ++- .../expressions/raster/RST_Subdivide.scala | 5 +- .../expressions/raster/RST_Summary.scala | 9 +- .../expressions/raster/RST_Tessellate.scala | 8 +- .../expressions/raster/RST_TryOpen.scala | 5 +- .../expressions/raster/RST_UpperLeftX.scala | 7 +- .../expressions/raster/RST_UpperLeftY.scala | 7 +- .../mosaic/expressions/raster/RST_Width.scala | 9 +- .../raster/RST_WorldToRasterCoord.scala | 7 +- .../raster/RST_WorldToRasterCoordX.scala | 7 +- .../raster/RST_WorldToRasterCoordY.scala | 7 +- .../base/RasterGeneratorExpression.scala | 10 +- .../mosaic/expressions/raster/package.scala | 4 +- .../labs/mosaic/functions/MosaicContext.scala | 7 -- .../geometry/ST_IsValidBehaviors.scala | 69 ---------- .../expressions/geometry/ST_IsValidTest.scala | 9 +- .../raster/RST_GridTilesBehaviors.scala | 51 -------- .../raster/RST_GridTilesTest.scala | 32 ----- .../raster/RST_MergeAggBehaviors.scala | 2 +- .../raster/RST_MergeBehaviors.scala | 4 +- 52 files changed, 292 insertions(+), 542 deletions(-) create mode 100644 src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/OverlappingTiles.scala create mode 100644 src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBox.scala delete mode 100644 src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTiles.scala delete mode 100644 src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesBehaviors.scala delete mode 100644 src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesTest.scala diff --git a/src/main/scala/com/databricks/labs/mosaic/core/geometry/MosaicGeometry.scala b/src/main/scala/com/databricks/labs/mosaic/core/geometry/MosaicGeometry.scala index 2bf4ad193..8fe0a04df 100644 --- a/src/main/scala/com/databricks/labs/mosaic/core/geometry/MosaicGeometry.scala +++ b/src/main/scala/com/databricks/labs/mosaic/core/geometry/MosaicGeometry.scala @@ -5,7 +5,8 @@ import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI import com.databricks.labs.mosaic.core.geometry.linestring.MosaicLineString import com.databricks.labs.mosaic.core.geometry.point.MosaicPoint import org.gdal.ogr.ogr -import org.gdal.osr._ +import org.gdal.osr.SpatialReference +import org.gdal.osr.osrConstants._ import org.locationtech.proj4j._ import java.util.Locale @@ -65,10 +66,10 @@ trait MosaicGeometry extends GeometryWriter with Serializable { def extent: (Double, Double, Double, Double) = { val env = envelope ( - env.minMaxCoord("X", "MIN"), - env.minMaxCoord("Y", "MIN"), - env.minMaxCoord("X", "MAX"), - env.minMaxCoord("Y", "MAX") + env.minMaxCoord("X", "MIN"), + env.minMaxCoord("Y", "MIN"), + env.minMaxCoord("X", "MAX"), + env.minMaxCoord("Y", "MAX") ) } @@ -149,6 +150,21 @@ trait MosaicGeometry extends GeometryWriter with Serializable { def setSpatialReference(srid: Int): Unit + def getSpatialReferenceOSR: SpatialReference = { + val srID = getSpatialReference + if (srID == 0) { + val wsg84 = new SpatialReference() + wsg84.ImportFromEPSG(4326) + wsg84.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER) + wsg84 + } else { + val geomCRS = new SpatialReference() + geomCRS.ImportFromEPSG(srID) + geomCRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER) + geomCRS + } + } + def hasValidCoords(crsBoundsProvider: CRSBoundsProvider, crsCode: String, which: String): Boolean = { val crsCodeIn = crsCode.split(":") val crsBounds = which.toLowerCase(Locale.ROOT) match { diff --git a/src/main/scala/com/databricks/labs/mosaic/core/geometry/geometrycollection/MosaicGeometryCollectionESRI.scala b/src/main/scala/com/databricks/labs/mosaic/core/geometry/geometrycollection/MosaicGeometryCollectionESRI.scala index dfdcb1bc2..b688962db 100644 --- a/src/main/scala/com/databricks/labs/mosaic/core/geometry/geometrycollection/MosaicGeometryCollectionESRI.scala +++ b/src/main/scala/com/databricks/labs/mosaic/core/geometry/geometrycollection/MosaicGeometryCollectionESRI.scala @@ -167,7 +167,7 @@ object MosaicGeometryCollectionESRI extends GeometryReader { // POINT by convention, MULTIPOINT are always flattened to POINT in the internal representation val coordinates = holesRings.head.head.coords MosaicPointESRI( - new OGCPoint(new Point(coordinates(0), coordinates(1)), spatialReference) + new OGCPoint(new Point(coordinates.head, coordinates(1)), spatialReference) ) } else { MosaicGeometryESRI.fromWKT("POINT EMPTY") diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/BalancedSubdivision.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/BalancedSubdivision.scala index 37e4a310e..c7106deed 100644 --- a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/BalancedSubdivision.scala +++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/BalancedSubdivision.scala @@ -37,14 +37,12 @@ object BalancedSubdivision { def splitRaster( mosaicRaster: MosaicRaster, - sizeInMb: Int, - geometryAPI: GeometryAPI, - rasterAPI: RasterAPI + sizeInMb: Int ): immutable.Seq[MosaicRaster] = { val numSplits = getNumSplits(mosaicRaster, sizeInMb) val (x, y) = mosaicRaster.getDimensions val (tileX, tileY) = getTileSize(x, y, numSplits) - ReTile.reTile(mosaicRaster, tileX, tileY, geometryAPI, rasterAPI) + ReTile.reTile(mosaicRaster, tileX, tileY) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/OverlappingTiles.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/OverlappingTiles.scala new file mode 100644 index 000000000..e4dc44ab8 --- /dev/null +++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/OverlappingTiles.scala @@ -0,0 +1,48 @@ +package com.databricks.labs.mosaic.core.raster.operator.retile + +import com.databricks.labs.mosaic.core.raster.MosaicRaster +import com.databricks.labs.mosaic.core.raster.operator.gdal.GDALTranslate +import com.databricks.labs.mosaic.utils.PathUtils + +import scala.collection.immutable + +object OverlappingTiles { + + def reTile( + raster: MosaicRaster, + tileWidth: Int, + tileHeight: Int, + overlapPercentage: Int + ): immutable.Seq[MosaicRaster] = { + val (xSize, ySize) = raster.getDimensions + + val overlapWidth = Math.ceil(tileWidth * overlapPercentage / 100.0).toInt + val overlapHeight = Math.ceil(tileHeight * overlapPercentage / 100.0).toInt + + val tiles = for (i <- 0 until xSize by (tileWidth - overlapWidth)) yield { + for (j <- 0 until ySize by (tileHeight - overlapHeight)) yield { + val xOff = if (i == 0) i else i - 1 + val yOff = if (j == 0) j else j - 1 + val width = Math.min(tileWidth, xSize - i) + 1 + val height = Math.min(tileHeight, ySize - j) + 1 + + val uuid = java.util.UUID.randomUUID.toString + val rasterPath = PathUtils.createTmpFilePath(uuid, "tif") + + val result = GDALTranslate.executeTranslate( + rasterPath, + isTemp = true, + raster, + command = s"gdal_translate -srcwin $xOff $yOff $width $height" + ) + + result.flushCache() + } + } + + tiles.flatten + + + } + +} diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/RasterTessellate.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/RasterTessellate.scala index 7258985e0..f6da0bd57 100644 --- a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/RasterTessellate.scala +++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/RasterTessellate.scala @@ -14,20 +14,28 @@ object RasterTessellate { val bbox = raster.bbox(geometryAPI, indexSR) val cells = Mosaic.mosaicFill(bbox, resolution, keepCoreGeom = false, indexSystem, geometryAPI) val tmpRaster = RasterProject.project(raster, indexSR) - val result = cells + + val chips = cells .map(cell => { val cellID = cell.cellIdAsLong(indexSystem) - val cellRaster = tmpRaster.getRasterForCell(cellID, indexSystem, geometryAPI) - cellRaster.getRaster.FlushCache() - ( - cellRaster.getBandStats.values.map(_("mean")).sum > 0 && !cellRaster.isEmpty, - MosaicRasterChip(cell.index, cellRaster) - ) + val isValidCell = indexSystem.isValid(cellID) + if (!isValidCell) { + (false, MosaicRasterChip(cell.index, null)) + } else { + val cellRaster = tmpRaster.getRasterForCell(cellID, indexSystem, geometryAPI) + val isValidRaster = cellRaster.getBandStats.values.map(_("mean")).sum > 0 && !cellRaster.isEmpty + ( + isValidRaster, + MosaicRasterChip(cell.index, cellRaster) + ) + } }) - .filter(_._1) - .map(_._2) + + val (result, invalid) = chips.partition(_._1) + invalid.foreach(_._2.raster.destroy()) tmpRaster.destroy() - result + + result.map(_._2) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/ReTile.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/ReTile.scala index b439fe58b..61f770dc9 100644 --- a/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/ReTile.scala +++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/operator/retile/ReTile.scala @@ -1,9 +1,8 @@ package com.databricks.labs.mosaic.core.raster.operator.retile -import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.api.RasterAPI -import com.databricks.labs.mosaic.core.raster.operator.clip.RasterClipByVector +import com.databricks.labs.mosaic.core.raster.operator.gdal.GDALTranslate +import com.databricks.labs.mosaic.utils.PathUtils import scala.collection.immutable @@ -12,26 +11,28 @@ object ReTile { def reTile( raster: MosaicRaster, tileWidth: Int, - tileHeight: Int, - geometryAPI: GeometryAPI, - rasterAPI: RasterAPI + tileHeight: Int ): immutable.Seq[MosaicRaster] = { val (xR, yR) = raster.getDimensions val xTiles = Math.ceil(xR / tileWidth).toInt val yTiles = Math.ceil(yR / tileHeight).toInt val tiles = for (x <- 0 until xTiles; y <- 0 until yTiles) yield { - val xMin = x * tileWidth - val yMin = y * tileHeight + val xMin = if (x == 0) x * tileWidth else x * tileWidth - 1 + val yMin = if (y == 0) y * tileHeight else y * tileHeight - 1 - val bbox = geometryAPI.createBbox(xMin, yMin, xMin + tileWidth, yMin + tileHeight) - .mapXY((x, y) => rasterAPI.toWorldCoord(raster.getGeoTransform, x.toInt, y.toInt)) + val rasterUUID = java.util.UUID.randomUUID.toString + val rasterPath = PathUtils.createTmpFilePath(rasterUUID, "tif") - // buffer bbox by the diagonal size of the raster to ensure we get all the pixels in the tile - val bufferR = raster.pixelDiagSize * 1.01 - val bufferedBBox = bbox.buffer(bufferR) + val result = GDALTranslate.executeTranslate( + rasterPath, + isTemp = true, + raster, + command = s"gdal_translate -srcwin $xMin $yMin ${tileWidth + 1} ${tileHeight + 1}" + ) - RasterClipByVector.clip(raster, bufferedBBox, raster.getRaster.GetSpatialRef(), geometryAPI) + result.flushCache() + result } diff --git a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala index 48658c375..f132ba8b8 100644 --- a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala +++ b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala @@ -1,7 +1,5 @@ package com.databricks.labs.mosaic.datasource.gdal -import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI -import com.databricks.labs.mosaic.core.raster.api.RasterAPI.GDAL import com.databricks.labs.mosaic.core.raster.gdal_raster.{MosaicRasterGDAL, RasterCleaner} import com.databricks.labs.mosaic.core.raster.operator.retile.BalancedSubdivision import com.databricks.labs.mosaic.datasource.Utils @@ -43,11 +41,10 @@ object ReTileOnRead extends ReadStrategy { val localCopy = PathUtils.copyToTmp(status.getPath.toString) val raster = MosaicRasterGDAL.readRaster(localCopy) val uuid = getUUID(status) - val geometryAPI = GeometryAPI.apply(options.getOrElse("geometry_api", "JTS")) val size = status.getLen val numSplits = Math.ceil(size / MB16).toInt - val tiles = BalancedSubdivision.splitRaster(raster, numSplits, geometryAPI, GDAL) + val tiles = BalancedSubdivision.splitRaster(raster, numSplits) val rows = tiles.map(tile => { val trimmedSchema = StructType(requiredSchema.filter(field => field.name != RASTER && field.name != LENGTH)) diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetaData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetaData.scala index a63d6264f..54e578de4 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetaData.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetaData.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.core.raster.{MosaicRaster, MosaicRasterBand} import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterBandExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** @@ -22,7 +21,13 @@ import org.apache.spark.sql.types._ * Additional arguments for the expression (expressionConfigs). */ case class RST_BandMetaData(raster: Expression, band: Expression, expressionConfig: MosaicExpressionConfig) - extends RasterBandExpression[RST_BandMetaData](raster, band, MapType(StringType, StringType), returnsRaster = false, expressionConfig = expressionConfig) + extends RasterBandExpression[RST_BandMetaData]( + raster, + band, + MapType(StringType, StringType), + returnsRaster = false, + expressionConfig = expressionConfig + ) with NullIntolerant with CodegenFallback { @@ -35,10 +40,7 @@ case class RST_BandMetaData(raster: Expression, band: Expression, expressionConf * The band metadata of the band as a map type result. */ override def bandTransform(raster: MosaicRaster, band: MosaicRasterBand): Any = { - val metaData = band.metadata - val result = buildMapString(metaData) - RasterCleaner.dispose(raster) - result + buildMapString(band.metadata) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBox.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBox.scala new file mode 100644 index 000000000..a26a2bc31 --- /dev/null +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBox.scala @@ -0,0 +1,73 @@ +package com.databricks.labs.mosaic.expressions.raster + +import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI +import com.databricks.labs.mosaic.core.raster.MosaicRaster +import com.databricks.labs.mosaic.core.types.model.GeometryTypeEnum +import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} +import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression +import com.databricks.labs.mosaic.functions.MosaicExpressionConfig +import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} +import org.apache.spark.sql.types._ + +/** Returns the world coordinates of the raster (x,y) pixel. */ +case class RST_BoundingBox( + raster: Expression, + expressionConfig: MosaicExpressionConfig +) extends RasterExpression[RST_RasterToWorldCoord](raster, BinaryType, returnsRaster = false, expressionConfig = expressionConfig) + with NullIntolerant + with CodegenFallback { + + /** + * The function to be overridden by the extending class. It is called when + * the expression is evaluated. It provides the raster to the expression. + * It abstracts spark serialization from the caller. + * + * @param raster + * The raster to be used. + * @return + * The result of the expression. + */ + override def rasterTransform(raster: MosaicRaster): Any = { + val gt = raster.getRaster.GetGeoTransform() + val (originX, originY) = rasterAPI.toWorldCoord(gt, 0, 0) + val (endX, endY) = rasterAPI.toWorldCoord(gt, raster.xSize, raster.ySize) + val geometryAPI = GeometryAPI(expressionConfig.getGeometryAPI) + val bboxPolygon = geometryAPI.geometry( + Seq( + Seq(originX, originY), + Seq(originX, endY), + Seq(endX, endY), + Seq(endX, originY), + Seq(originX, originY) + ).map(geometryAPI.fromCoords), + GeometryTypeEnum.POLYGON + ) + bboxPolygon.toWKB + } + +} + +/** Expression info required for the expression registration for spark SQL. */ +object RST_BoundingBox extends WithExpressionInfo { + + override def name: String = "rst_boundingbox" + + override def usage: String = + """ + |_FUNC_(expr1) - Returns the bounding box of the raster. + |""".stripMargin + + override def example: String = + """ + | Examples: + | > SELECT _FUNC_(a, b, c); + | (11.2, 12.3) + | """.stripMargin + + override def builder(expressionConfig: MosaicExpressionConfig): FunctionBuilder = { + GenericExpressionFactory.getBaseBuilder[RST_RasterToWorldCoord](3, expressionConfig) + } + +} diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala index 40fa9f541..4afd75a05 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala @@ -10,8 +10,6 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types.BinaryType -import org.gdal.osr -import org.gdal.osr.SpatialReference /** * Returns a set of new rasters with the specified tile size (tileWidth x @@ -25,7 +23,7 @@ case class RST_Clip( rastersExpr, geometryExpr, BinaryType, - returnsRaster = false, + returnsRaster = true, expressionConfig = expressionConfig ) with NullIntolerant @@ -47,22 +45,8 @@ case class RST_Clip( */ override def rasterTransform(raster: MosaicRaster, arg1: Any): Any = { val geometry = geometryAPI.geometry(arg1, geometryExpr.dataType) - val geomCRS = - if (geometry.getSpatialReference == 0) { - val wsg84 = new osr.SpatialReference() - wsg84.ImportFromEPSG(4326) - wsg84.SetAxisMappingStrategy(osr.osrConstants.OAMS_TRADITIONAL_GIS_ORDER) - wsg84 - } - else { - val geomCRS = new SpatialReference() - geomCRS.ImportFromEPSG(geometry.getSpatialReference) - // debug for this - geomCRS.SetAxisMappingStrategy(osr.osrConstants.OAMS_TRADITIONAL_GIS_ORDER) - geomCRS - } - val result = RasterClipByVector.clip(raster, geometry, geomCRS, geometryAPI) - rasterAPI.writeRasters(Seq(result), expressionConfig.getRasterCheckpoint, BinaryType).head + val geomCRS = geometry.getSpatialReferenceOSR + RasterClipByVector.clip(raster, geometry, geomCRS, geometryAPI) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReference.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReference.scala index b20dc8250..4f77abb7c 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReference.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReference.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the georeference of the raster. */ @@ -19,17 +18,16 @@ case class RST_GeoReference(raster: Expression, expressionConfig: MosaicExpressi /** Returns the georeference of the raster. */ override def rasterTransform(raster: MosaicRaster): Any = { val geoTransform = raster.getRaster.GetGeoTransform() - val geoReference = Map( - "upperLeftX" -> geoTransform(0), - "upperLeftY" -> geoTransform(3), - "scaleX" -> geoTransform(1), - "scaleY" -> geoTransform(5), - "skewX" -> geoTransform(2), - "skewY" -> geoTransform(4) + buildMapDouble( + Map( + "upperLeftX" -> geoTransform(0), + "upperLeftY" -> geoTransform(3), + "scaleX" -> geoTransform(1), + "scaleY" -> geoTransform(5), + "skewX" -> geoTransform(2), + "skewY" -> geoTransform(4) + ) ) - val result = buildMapDouble(geoReference) - RasterCleaner.dispose(raster) - result } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTiles.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTiles.scala deleted file mode 100644 index 7e7d4a459..000000000 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTiles.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.databricks.labs.mosaic.expressions.raster - -import com.databricks.labs.mosaic.core.Mosaic -import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI -import com.databricks.labs.mosaic.core.index.{IndexSystem, IndexSystemFactory} -import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.api.RasterAPI -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner -import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} -import com.databricks.labs.mosaic.functions.MosaicExpressionConfig -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.expressions.{CollectionGenerator, Expression, NullIntolerant} -import org.apache.spark.sql.types._ - -/** - * Returns a set of new rasters with the specified tile size (tileWidth x - * tileHeight). - */ -case class RST_GridTiles( - rasterExpr: Expression, - resolutionExpr: Expression, - expressionConfig: MosaicExpressionConfig -) extends CollectionGenerator - with NullIntolerant - with CodegenFallback { - - /** The index system to be used. */ - val indexSystem: IndexSystem = IndexSystemFactory.getIndexSystem(expressionConfig.getIndexSystem) - val geometryAPI: GeometryAPI = GeometryAPI(expressionConfig.getGeometryAPI) - - /** - * The raster API to be used. Enable the raster so that subclasses dont - * need to worry about this. - */ - protected val rasterAPI: RasterAPI = RasterAPI(expressionConfig.getRasterAPI) - rasterAPI.enable() - - override def position: Boolean = false - - override def inline: Boolean = false - - /** - * Generators expressions require an abstraction for element type. Always - * needs to be wrapped in a StructType. The actually type is that of the - * structs element. - */ - override def elementSchema: StructType = StructType(Array(StructField("raster", rasterExpr.dataType))) - - /** - * Returns a set of new rasters with the specified tile size (tileWidth x - * tileHeight). - */ - def rasterGenerator(raster: MosaicRaster, resolution: Int): Seq[MosaicRaster] = { - val indexCRS = indexSystem.osrSpatialRef - val bbox = raster.bbox(geometryAPI, indexCRS) - - val cells = Mosaic - .mosaicFill(bbox, resolution, keepCoreGeom = false, indexSystem, geometryAPI) - .map(_.indexAsLong(indexSystem)) - - val rasters = cells.map(cellID => raster.getRasterForCell(cellID, indexSystem, geometryAPI)) - RasterCleaner.dispose(raster) - - rasters - } - - override def eval(input: InternalRow): TraversableOnce[InternalRow] = { - val checkpointPath = expressionConfig.getRasterCheckpoint - val resolution = resolutionExpr.eval(input).asInstanceOf[Int] - - val raster = rasterAPI.readRaster(rasterExpr.eval(input), rasterExpr.dataType) - val tiles = rasterGenerator(raster, resolution) - - val result = rasterAPI - .writeRasters(tiles, checkpointPath, rasterExpr.dataType) - .map(row => InternalRow.fromSeq(Seq(row))) - - RasterCleaner.dispose(raster) - tiles.foreach(RasterCleaner.dispose) - - result - } - - override def children: Seq[Expression] = Seq(rasterExpr, resolutionExpr) - - override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = { - copy(rasterExpr = newChildren(0), resolutionExpr = newChildren(1)) - } - -} - -/** Expression info required for the expression registration for spark SQL. */ -object RST_GridTiles extends WithExpressionInfo { - - override def name: String = "rst_gridtiles" - - override def usage: String = - """ - |_FUNC_(expr1) - Returns a set of new rasters with the specified tile size (tileWidth x tileHeight). - |""".stripMargin - - override def example: String = - """ - | Examples: - | > SELECT _FUNC_(a, b); - | /path/to/raster_tile_1.tif - | /path/to/raster_tile_2.tif - | /path/to/raster_tile_3.tif - | ... - | """.stripMargin - - override def builder(expressionConfig: MosaicExpressionConfig): FunctionBuilder = { - GenericExpressionFactory.getBaseBuilder[RST_GridTiles](2, expressionConfig) - } - -} diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Height.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Height.scala index d140c6ce7..a064a3afb 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Height.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Height.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the width of the raster. */ @@ -17,15 +16,10 @@ case class RST_Height(raster: Expression, expressionConfig: MosaicExpressionConf with CodegenFallback { /** Returns the width of the raster. */ - override def rasterTransform(raster: MosaicRaster): Any = { - val result = raster.ySize - RasterCleaner.dispose(raster) - result - } + override def rasterTransform(raster: MosaicRaster): Any = raster.ySize } - /** Expression info required for the expression registration for spark SQL. */ object RST_Height extends WithExpressionInfo { @@ -45,4 +39,3 @@ object RST_Height extends WithExpressionInfo { } } - diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmpty.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmpty.scala index e24680234..435913315 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmpty.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmpty.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns true if the raster is empty. */ @@ -18,9 +17,7 @@ case class RST_IsEmpty(raster: Expression, expressionConfig: MosaicExpressionCon /** Returns true if the raster is empty. */ override def rasterTransform(raster: MosaicRaster): Any = { - val result = (raster.ySize == 0 && raster.xSize == 0) || raster.isEmpty - RasterCleaner.dispose(raster) - result + (raster.ySize == 0 && raster.xSize == 0) || raster.isEmpty } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSize.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSize.scala index 8c66a5710..d885d19d4 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSize.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSize.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the memory size of the raster in bytes. */ @@ -17,11 +16,7 @@ case class RST_MemSize(raster: Expression, expressionConfig: MosaicExpressionCon with CodegenFallback { /** Returns the memory size of the raster in bytes. */ - override def rasterTransform(raster: MosaicRaster): Any = { - val result = raster.getMemSize - RasterCleaner.dispose(raster) - result - } + override def rasterTransform(raster: MosaicRaster): Any = raster.getMemSize } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala index 94404b890..979f6a6bc 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala @@ -1,7 +1,6 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.core.raster.operator.merge.MergeRasters import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterArrayExpression @@ -21,7 +20,7 @@ case class RST_Merge( ) extends RasterArrayExpression[RST_Merge]( rastersExpr, BinaryType, - returnsRaster = false, + returnsRaster = true, expressionConfig = expressionConfig ) with NullIntolerant @@ -31,11 +30,7 @@ case class RST_Merge( * Returns a set of new rasters with the specified tile size (tileWidth x * tileHeight). */ - override def rasterTransform(rasters: Seq[MosaicRaster]): Any = { - val result = MergeRasters.merge(rasters) - rasters.foreach(RasterCleaner.dispose) - rasterAPI.writeRasters(Seq(result), expressionConfig.getRasterCheckpoint, BinaryType).head - } + override def rasterTransform(rasters: Seq[MosaicRaster]): Any = MergeRasters.merge(rasters) } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBands.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBands.scala index a6e80c727..685fc0424 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBands.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBands.scala @@ -1,7 +1,6 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.core.raster.operator.merge.MergeBands import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterArrayExpression @@ -21,7 +20,7 @@ case class RST_MergeBands( ) extends RasterArrayExpression[RST_MergeBands]( bandsExpr, BinaryType, - returnsRaster = false, + returnsRaster = true, expressionConfig = expressionConfig ) with NullIntolerant @@ -31,11 +30,7 @@ case class RST_MergeBands( * Returns a set of new rasters with the specified tile size (tileWidth x * tileHeight). */ - override def rasterTransform(rasters: Seq[MosaicRaster]): Any = { - val result = MergeBands.merge(rasters, "bilinear") - rasters.foreach(RasterCleaner.dispose) - rasterAPI.writeRasters(Seq(result), expressionConfig.getRasterCheckpoint, BinaryType).head - } + override def rasterTransform(rasters: Seq[MosaicRaster]): Any = MergeBands.merge(rasters, "bilinear") } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetaData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetaData.scala index cfdf4ee47..80f5a484b 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetaData.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetaData.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the metadata of the raster. */ @@ -17,12 +16,8 @@ case class RST_MetaData(raster: Expression, expressionConfig: MosaicExpressionCo with CodegenFallback { /** Returns the metadata of the raster. */ - override def rasterTransform(raster: MosaicRaster): Any = { - val metaData = raster.metadata - val result = buildMapString(metaData) - RasterCleaner.dispose(raster) - result - } + override def rasterTransform(raster: MosaicRaster): Any = buildMapString(raster.metadata) + } /** Expression info required for the expression registration for spark SQL. */ diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala index 0696259ac..bd3432518 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala @@ -24,32 +24,32 @@ case class RST_NDVI( redIndex, nirIndex, BinaryType, - returnsRaster = false, + returnsRaster = true, expressionConfig = expressionConfig ) with NullIntolerant with CodegenFallback { /** - * The function to be overridden by the extending class. It is called when - * the expression is evaluated. It provides the raster and the arguments to - * the expression. It abstracts spark serialization from the caller. - * - * @param raster - * The raster to be used. - * @param arg1 - * The first argument. - * @param arg2 - * The second argument. - * @return - * A result of the expression. - */ + * The function to be overridden by the extending class. It is called when + * the expression is evaluated. It provides the raster and the arguments to + * the expression. It abstracts spark serialization from the caller. + * + * @param raster + * The raster to be used. + * @param arg1 + * The first argument. + * @param arg2 + * The second argument. + * @return + * A result of the expression. + */ override def rasterTransform(raster: MosaicRaster, arg1: Any, arg2: Any): Any = { val redInd = arg1.asInstanceOf[Int] val nirInd = arg2.asInstanceOf[Int] - val result = NDVI.compute(raster, redInd, nirInd) - rasterAPI.writeRasters(Seq(result), expressionConfig.getRasterCheckpoint, BinaryType).head + NDVI.compute(raster, redInd, nirInd) } + } /** Expression info required for the expression registration for spark SQL. */ diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBands.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBands.scala index 0756d8dde..22d71f270 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBands.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBands.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the number of bands in the raster. */ @@ -17,11 +16,7 @@ case class RST_NumBands(raster: Expression, expressionConfig: MosaicExpressionCo with CodegenFallback { /** Returns the number of bands in the raster. */ - override def rasterTransform(raster: MosaicRaster): Any = { - val result = raster.numBands - RasterCleaner.dispose(raster) - result - } + override def rasterTransform(raster: MosaicRaster): Any = raster.numBands } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeight.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeight.scala index 56a5b055f..67aeeb154 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeight.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeight.scala @@ -22,6 +22,7 @@ case class RST_PixelHeight(raster: Expression, expressionConfig: MosaicExpressio val skewX = raster.getRaster.GetGeoTransform()(2) // when there is no skew the height is scaleY, but we cant assume 0-only skew // skew is not to be confused with rotation + // TODO - check if this is correct val result = math.sqrt(scaleY * scaleY + skewX * skewX) RasterCleaner.dispose(raster) result diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidth.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidth.scala index aeb17f31e..c6265d9e6 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidth.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidth.scala @@ -22,6 +22,7 @@ case class RST_PixelWidth(raster: Expression, expressionConfig: MosaicExpression val skewY = raster.getRaster.GetGeoTransform()(4) // when there is no skew width is scaleX, but we cant assume 0-only skew // skew is not to be confused with rotation + // TODO check if this is correct val result = math.sqrt(scaleX * scaleX + skewY * skewY) RasterCleaner.dispose(raster) result diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoord.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoord.scala index 3f7503f60..74d824833 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoord.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoord.scala @@ -2,13 +2,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.Raster2ArgExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the world coordinates of the raster (x,y) pixel. */ @@ -35,9 +34,7 @@ case class RST_RasterToWorldCoord( val geometryAPI = GeometryAPI(expressionConfig.getGeometryAPI) val point = geometryAPI.fromCoords(Seq(xGeo, yGeo)) - val result = geometryAPI.serialize(point, StringType) - RasterCleaner.dispose(raster) - result + geometryAPI.serialize(point, StringType) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordX.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordX.scala index c9f8c6e96..477f74ba2 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordX.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordX.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.Raster2ArgExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the world coordinates of the raster (x,y) pixel. */ @@ -30,7 +29,6 @@ case class RST_RasterToWorldCoordX( val gt = raster.getRaster.GetGeoTransform() val (xGeo, _) = rasterAPI.toWorldCoord(gt, x, y) - RasterCleaner.dispose(raster) xGeo } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordY.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordY.scala index 493db4233..0de930521 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordY.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordY.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.Raster2ArgExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the world coordinates of the raster (x,y) pixel. */ @@ -30,7 +29,6 @@ case class RST_RasterToWorldCoordY( val gt = raster.getRaster.GetGeoTransform() val (_, yGeo) = rasterAPI.toWorldCoord(gt, x, y) - RasterCleaner.dispose(raster) yGeo } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala index 05cfdbd02..1f9249509 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala @@ -1,7 +1,6 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.core.raster.operator.retile.ReTile import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterGeneratorExpression @@ -30,9 +29,7 @@ case class RST_ReTile( override def rasterGenerator(raster: MosaicRaster): Seq[MosaicRaster] = { val tileWidthValue = tileWidthExpr.eval().asInstanceOf[Int] val tileHeightValue = tileHeightExpr.eval().asInstanceOf[Int] - val result = ReTile.reTile(raster, tileWidthValue, tileHeightValue, geometryAPI, rasterAPI) - RasterCleaner.dispose(raster) - result + ReTile.reTile(raster, tileWidthValue, tileHeightValue) } override def children: Seq[Expression] = Seq(rasterExpr, tileWidthExpr, tileHeightExpr) diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Rotation.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Rotation.scala index df671cba2..2e699379c 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Rotation.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Rotation.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the rotation angle of the raster. */ @@ -20,9 +19,7 @@ case class RST_Rotation(raster: Expression, expressionConfig: MosaicExpressionCo override def rasterTransform(raster: MosaicRaster): Any = { val gt = raster.getRaster.GetGeoTransform() // arctan of y_skew and x_scale - val result = math.atan(gt(4) / gt(1)) - RasterCleaner.dispose(raster) - result + math.atan(gt(4) / gt(1)) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRID.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRID.scala index d5ded0ecc..e7734abb5 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRID.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRID.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ import org.gdal.osr.SpatialReference @@ -24,9 +23,7 @@ case class RST_SRID(raster: Expression, expressionConfig: MosaicExpressionConfig // Reference: https://gis.stackexchange.com/questions/267321/extracting-epsg-from-a-raster-using-gdal-bindings-in-python val proj = new SpatialReference(raster.getRaster.GetProjection()) Try(proj.AutoIdentifyEPSG()) - val result = Try(proj.GetAttrValue("AUTHORITY", 1).toInt).getOrElse(0) - RasterCleaner.dispose(raster) - result + Try(proj.GetAttrValue("AUTHORITY", 1).toInt).getOrElse(0) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleX.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleX.scala index 5038fa1d3..fe2f4b3df 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleX.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleX.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig -import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.{FunctionBuilder, expressions} -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} +import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the scale x of the raster. */ @@ -18,9 +17,7 @@ case class RST_ScaleX(raster: Expression, expressionConfig: MosaicExpressionConf /** Returns the scale x of the raster. */ override def rasterTransform(raster: MosaicRaster): Any = { - val scaleX = raster.getRaster.GetGeoTransform()(1) - RasterCleaner.dispose(raster) - scaleX + raster.getRaster.GetGeoTransform()(1) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleY.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleY.scala index 601399a0e..2c88d14ae 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleY.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleY.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the scale y of the raster. */ @@ -18,9 +17,7 @@ case class RST_ScaleY(raster: Expression, expressionConfig: MosaicExpressionConf /** Returns the scale y of the raster. */ override def rasterTransform(raster: MosaicRaster): Any = { - val scaleY = raster.getRaster.GetGeoTransform()(5) - RasterCleaner.dispose(raster) - scaleY + raster.getRaster.GetGeoTransform()(5) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewX.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewX.scala index f915b099d..22ad14d94 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewX.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewX.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the skew x of the raster. */ @@ -18,9 +17,7 @@ case class RST_SkewX(raster: Expression, expressionConfig: MosaicExpressionConfi /** Returns the skew x of the raster. */ override def rasterTransform(raster: MosaicRaster): Any = { - val skewX = raster.getRaster.GetGeoTransform()(2) - RasterCleaner.dispose(raster) - skewX + raster.getRaster.GetGeoTransform()(2) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewY.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewY.scala index cb8bcebde..e4c01942b 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewY.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewY.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the skew y of the raster. */ @@ -18,9 +17,7 @@ case class RST_SkewY(raster: Expression, expressionConfig: MosaicExpressionConfi /** Returns the skew y of the raster. */ override def rasterTransform(raster: MosaicRaster): Any = { - val skewY = raster.getRaster.GetGeoTransform()(4) - RasterCleaner.dispose(raster) - skewY + raster.getRaster.GetGeoTransform()(4) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdatasets.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdatasets.scala index 13d705007..8ad794a80 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdatasets.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdatasets.scala @@ -1,28 +1,27 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the subdatasets of the raster. */ case class RST_Subdatasets(raster: Expression, expressionConfig: MosaicExpressionConfig) - extends RasterExpression[RST_Subdatasets](raster, MapType(StringType, StringType), returnsRaster = false, expressionConfig) + extends RasterExpression[RST_Subdatasets]( + raster, + MapType(StringType, StringType), + returnsRaster = false, + expressionConfig + ) with NullIntolerant with CodegenFallback { /** Returns the subdatasets of the raster. */ - override def rasterTransform(raster: MosaicRaster): Any = { - val subdatasets = raster.subdatasets - val result = buildMapString(subdatasets) - RasterCleaner.dispose(raster) - result - } + override def rasterTransform(raster: MosaicRaster): Any = buildMapString(raster.subdatasets) } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdivide.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdivide.scala index 098a71f8a..96b4b770c 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdivide.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Subdivide.scala @@ -30,8 +30,7 @@ case class RST_Subdivide( */ override def rasterGenerator(raster: MosaicRaster): Seq[MosaicRaster] = { val targetSize = sizeInMB.eval().asInstanceOf[Int] - val tiles = BalancedSubdivision.splitRaster(raster, targetSize, geometryAPI, rasterAPI) - tiles + BalancedSubdivision.splitRaster(raster, targetSize) } override def children: Seq[Expression] = Seq(rasterExpr, sizeInMB) @@ -59,7 +58,7 @@ object RST_Subdivide extends WithExpressionInfo { | """.stripMargin override def builder(expressionConfig: MosaicExpressionConfig): FunctionBuilder = { - GenericExpressionFactory.getBaseBuilder[RST_Subdivide](3, expressionConfig) + GenericExpressionFactory.getBaseBuilder[RST_Subdivide](2, expressionConfig) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Summary.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Summary.scala index 3f4072d2b..3da88917f 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Summary.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Summary.scala @@ -1,17 +1,16 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String -import org.gdal.gdal.gdal.GDALInfo import org.gdal.gdal.InfoOptions +import org.gdal.gdal.gdal.GDALInfo import java.util.{Vector => JVector} @@ -29,9 +28,7 @@ case class RST_Summary(raster: Expression, expressionConfig: MosaicExpressionCon vector.add("-json") val infoOptions = new InfoOptions(vector) val gdalInfo = GDALInfo(raster.getRaster, infoOptions) - val result = UTF8String.fromString(gdalInfo) - RasterCleaner.dispose(raster) - result + UTF8String.fromString(gdalInfo) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Tessellate.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Tessellate.scala index d60752cfd..e49103e02 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Tessellate.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Tessellate.scala @@ -1,8 +1,6 @@ package com.databricks.labs.mosaic.expressions.raster -import com.databricks.labs.mosaic.core.index.BNGIndexSystem import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.core.raster.operator.retile.RasterTessellate import com.databricks.labs.mosaic.core.types.model.MosaicRasterChip import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} @@ -29,16 +27,12 @@ case class RST_Tessellate( * tileHeight). */ override def rasterGenerator(raster: MosaicRaster, resolution: Int): Seq[MosaicRasterChip] = { - val result = RasterTessellate.tessellate( + RasterTessellate.tessellate( raster, resolution, indexSystem, geometryAPI ) - RasterCleaner.dispose(raster) - result - .filter(c => indexSystem.isValid(c.indexAsLong(indexSystem))) - .map(_.formatCellId(indexSystem)) } override def children: Seq[Expression] = Seq(rasterExpr, resolutionExpr) diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpen.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpen.scala index ecc1174a9..c80c3f5e0 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpen.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpen.scala @@ -1,7 +1,6 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig @@ -18,9 +17,7 @@ case class RST_TryOpen(raster: Expression, expressionConfig: MosaicExpressionCon /** Returns true if the raster can be opened. */ override def rasterTransform(raster: MosaicRaster): Any = { - val result = Option(raster.getRaster).isDefined - RasterCleaner.dispose(raster) - result + Option(raster.getRaster).isDefined } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftX.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftX.scala index 11f79a528..b0d5fb69d 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftX.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftX.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the upper left x of the raster. */ @@ -18,9 +17,7 @@ case class RST_UpperLeftX(raster: Expression, expressionConfig: MosaicExpression /** Returns the upper left x of the raster. */ override def rasterTransform(raster: MosaicRaster): Any = { - val upperLeftX = raster.getRaster.GetGeoTransform()(0) - RasterCleaner.dispose(raster) - upperLeftX + raster.getRaster.GetGeoTransform()(0) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftY.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftY.scala index 2a28efba7..c2523dc61 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftY.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftY.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the upper left y of the raster. */ @@ -18,9 +17,7 @@ case class RST_UpperLeftY(raster: Expression, expressionConfig: MosaicExpression /** Returns the upper left y of the raster. */ override def rasterTransform(raster: MosaicRaster): Any = { - val upperLeftY = raster.getRaster.GetGeoTransform()(3) - RasterCleaner.dispose(raster) - upperLeftY + raster.getRaster.GetGeoTransform()(3) } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Width.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Width.scala index b3fc5f2ef..ee05dad04 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Width.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Width.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.RasterExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types._ /** Returns the width of the raster. */ @@ -17,11 +16,7 @@ case class RST_Width(raster: Expression, expressionConfig: MosaicExpressionConfi with CodegenFallback { /** Returns the width of the raster. */ - override def rasterTransform(raster: MosaicRaster): Any = { - val result = raster.xSize - RasterCleaner.dispose(raster) - result - } + override def rasterTransform(raster: MosaicRaster): Any = raster.xSize } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoord.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoord.scala index fca929abc..a77394e59 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoord.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoord.scala @@ -1,14 +1,13 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.Raster2ArgExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} /** Returns the world coordinate of the raster. */ case class RST_WorldToRasterCoord( @@ -30,8 +29,6 @@ case class RST_WorldToRasterCoord( val gt = raster.getRaster.GetGeoTransform() val (x, y) = rasterAPI.fromWorldCoord(gt, xGeo, yGeo) - RasterCleaner.dispose(raster) - InternalRow.fromSeq(Seq(x, y)) } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordX.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordX.scala index 8ea8e6b99..0685344a9 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordX.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordX.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.Raster2ArgExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types.IntegerType /** Returns the x coordinate of the raster. */ @@ -27,9 +26,7 @@ case class RST_WorldToRasterCoordX( override def rasterTransform(raster: MosaicRaster, arg1: Any, arg2: Any): Any = { val xGeo = arg1.asInstanceOf[Double] val gt = raster.getRaster.GetGeoTransform() - val result = rasterAPI.fromWorldCoord(gt, xGeo, 0)._1 - RasterCleaner.dispose(raster) - result + rasterAPI.fromWorldCoord(gt, xGeo, 0)._1 } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordY.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordY.scala index e4c8e527e..992acaa3f 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordY.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordY.scala @@ -1,13 +1,12 @@ package com.databricks.labs.mosaic.expressions.raster import com.databricks.labs.mosaic.core.raster.MosaicRaster -import com.databricks.labs.mosaic.core.raster.gdal_raster.RasterCleaner import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo} import com.databricks.labs.mosaic.expressions.raster.base.Raster2ArgExpression import com.databricks.labs.mosaic.functions.MosaicExpressionConfig import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder -import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant} import org.apache.spark.sql.types.IntegerType /** Returns the Y coordinate of the raster. */ @@ -27,9 +26,7 @@ case class RST_WorldToRasterCoordY( override def rasterTransform(raster: MosaicRaster, arg1: Any, arg2: Any): Any = { val xGeo = arg1.asInstanceOf[Double] val gt = raster.getRaster.GetGeoTransform() - val result = rasterAPI.fromWorldCoord(gt, xGeo, 0)._2 - RasterCleaner.dispose(raster) - result + rasterAPI.fromWorldCoord(gt, xGeo, 0)._2 } } diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala index 534b281bb..e687f464b 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala @@ -10,7 +10,6 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{CollectionGenerator, Expression, NullIntolerant} import org.apache.spark.sql.types._ -import java.nio.file.{Files, Paths} import scala.reflect.ClassTag /** @@ -37,7 +36,7 @@ abstract class RasterGeneratorExpression[T <: Expression: ClassTag]( with NullIntolerant with Serializable { - override def dataType: DataType = rasterExpr.dataType + override def dataType: DataType = BinaryType val uuid: String = java.util.UUID.randomUUID().toString.replace("-", "_") @@ -58,7 +57,7 @@ abstract class RasterGeneratorExpression[T <: Expression: ClassTag]( * needs to be wrapped in a StructType. The actually type is that of the * structs element. */ - override def elementSchema: StructType = StructType(Array(StructField("raster", dataType))) + override def elementSchema: StructType = StructType(Array(StructField("raster", BinaryType))) /** * The function to be overridden by the extending class. It is called when @@ -72,13 +71,12 @@ abstract class RasterGeneratorExpression[T <: Expression: ClassTag]( def rasterGenerator(raster: MosaicRaster): Seq[MosaicRaster] override def eval(input: InternalRow): TraversableOnce[InternalRow] = { - val checkpointPath = expressionConfig.getRasterCheckpoint - val inRaster = rasterAPI.readRaster(rasterExpr.eval(input), rasterExpr.dataType) val generatedRasters = rasterGenerator(inRaster) // Writing rasters disposes of the written raster - val rows = rasterAPI.writeRasters(generatedRasters, checkpointPath, dataType) + val rows = generatedRasters.map(_.writeToBytes()) + generatedRasters.foreach(RasterCleaner.dispose) RasterCleaner.dispose(inRaster) rows.map(row => InternalRow.fromSeq(Seq(row))) diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/package.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/package.scala index fe03cb672..a229aae89 100644 --- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/package.scala +++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/package.scala @@ -21,8 +21,8 @@ package object raster { * The measure type of the resulting pixel value. * * @return - * The datatype to be used for serialization of the result of - * [[RasterToGridExpression]]. + * The datatype to be used for serialization of the result of + * [[com.databricks.labs.mosaic.expressions.raster.base.RasterToGridExpression]]. */ def RasterToGridType(cellIDType: DataType, measureType: DataType): DataType = { ArrayType( diff --git a/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala b/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala index a7fec219d..9f9945ec6 100644 --- a/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala +++ b/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala @@ -258,7 +258,6 @@ class MosaicContext(indexSystem: IndexSystem, geometryAPI: GeometryAPI, rasterAP mosaicRegistry.registerExpression[RST_BandMetaData](expressionConfig) mosaicRegistry.registerExpression[RST_Clip](expressionConfig) mosaicRegistry.registerExpression[RST_GeoReference](expressionConfig) - mosaicRegistry.registerExpression[RST_GridTiles](expressionConfig) mosaicRegistry.registerExpression[RST_Height](expressionConfig) mosaicRegistry.registerExpression[RST_IsEmpty](expressionConfig) mosaicRegistry.registerExpression[RST_MemSize](expressionConfig) @@ -672,12 +671,6 @@ class MosaicContext(indexSystem: IndexSystem, geometryAPI: GeometryAPI, rasterAP ColumnAdapter(RST_ReTile(raster.expr, lit(tileWidth).expr, lit(tileHeight).expr, expressionConfig)) def rst_retile(raster: String, tileWidth: Int, tileHeight: Int): Column = ColumnAdapter(RST_ReTile(lit(raster).expr, lit(tileWidth).expr, lit(tileHeight).expr, expressionConfig)) - def rst_gridtiles(raster: Column, resolution: Column): Column = - ColumnAdapter(RST_GridTiles(raster.expr, resolution.expr, expressionConfig)) - def rst_gridtiles(raster: String, resolution: Column): Column = - ColumnAdapter(RST_GridTiles(lit(raster).expr, resolution.expr, expressionConfig)) - def rst_gridtiles(raster: Column, resolution: Int): Column = - ColumnAdapter(RST_GridTiles(raster.expr, lit(resolution).expr, expressionConfig)) def rst_rotation(raster: Column): Column = ColumnAdapter(RST_Rotation(raster.expr, expressionConfig)) def rst_rotation(raster: String): Column = ColumnAdapter(RST_Rotation(lit(raster).expr, expressionConfig)) def rst_scalex(raster: Column): Column = ColumnAdapter(RST_ScaleX(raster.expr, expressionConfig)) diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala index 3efccc6b3..a11710f9e 100644 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala +++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala @@ -120,73 +120,4 @@ trait ST_IsValidBehaviors extends MosaicSpatialQueryTest { } - def issue(mosaicContext: MosaicContext): Unit = { - spark.sparkContext.setLogLevel("FATAL") - val sc = spark - val mc = mosaicContext - import mc.functions._ - import sc.implicits._ - mc.register(spark) - - val wkt = "MULTIPOLYGON (((-73.193199311173331 45.032701793372674," + - " -73.193199422881179 45.032701420158844, -73.193199489594178 45.03270104157604," + - " -73.193199510831988 45.032700660346435, -73.193199486441742 45.032700279212058," + - " -73.193189346414215 45.032619101872115, -73.193189281895911 45.032618745525369," + - " -73.193189177484285 45.032618393964931, -73.193189033845357 45.032618049432905," + - " -73.1931888518943 45.032617714126, -73.1931886327919 45.032617390182267," + - " -73.193188377935726 45.032617079667538, -73.1931880889503 45.032616784561895," + - " -73.19318776767841 45.032616506747111, -73.193187416168712 45.032616247994795," + - " -73.193187036662977 45.032616009954921, -73.193186631581028 45.032615794145443," + - " -73.193186203505959 45.032615601942169, -73.193185755167562 45.032615434570921," + - " -73.19318528942442 45.03261529309902, -73.193184809246617 45.032615178428671," + - " -73.193184317696492 45.032615091291255, -73.193184032908164 45.032615054081091," + - " -73.193183746022484 45.032615026123324, -73.1931834576363 45.03261500747665," + - " -73.193183168350487 45.032614998179142, -73.193157790801948 45.032614596556655," + - " -73.193157479312177 45.03261459701848, -73.193157168229348 45.032614608319179," + - " -73.193156858304 45.032614630431752, -73.193156550281969 45.032614663302773," + - " -73.19315624490693 45.032614706853145, -73.193155942913648 45.032614760977545," + - " -73.193155645030487 45.032614825546155, -73.1931553519753 45.032614900403061," + - " -73.193155064454146 45.032614985367509, -73.19315478316048 45.032615080235296," + - " -73.193154508771826 45.032615184777434, -73.193154241949046 45.03261529874225," + - " -73.193153983336273 45.032615421854715, -73.193153733555562 45.032615553818367," + - " -73.19315349320992 45.032615694315325, -73.193153262878127 45.032615843006582," + - " -73.193153043114989 45.032615999534251, -73.193152834450643 45.0326161635207, " + - "-73.193152637387428 45.032616334570868, -73.1931524524011 45.032616512272384," + - " -73.193152279936839 45.032616696197373, -73.1931521204103 45.03261688590225," + - " -73.193151974206032 45.032617080929874, -73.193151841676766 45.032617280810378," + - " -73.193151723141241 45.032617485062, -73.193151618885651 45.032617693192357," + - " -73.193151529160616 45.03261790469967, -73.193151454183351 45.032618119074712," + - " -73.193151394134 45.032618335800606, -73.19315134915702 45.032618554355118," + - " -73.193151319361135 45.032618774211166, -73.193151304818045 45.032618994839076," + - " -73.193148472795158 45.032708968760623, -73.193148475581978 45.032709235046163," + - " -73.193148500581927 45.032709500748645, -73.19314854770748 45.032709764936783," + - " -73.1931486167935 45.032710026685322, -73.193148707597757 45.032710285077187," + - " -73.19314881980246 45.032710539207216, -73.1931489530145 45.032710788184829," + - " -73.193149106766924 45.032711031138192, -73.193149280521069 45.032711267216158, " + - "-73.19314947366874 45.032711495591407, -73.193149685532674 45.032711715463954," + - " -73.1931499153713 45.032711926063705, -73.193150162378771 45.032712126652996, " + - "-73.193150425690291 45.03271231652851, -73.193150704383228 45.032712495025585," + - " -73.193150997480871 45.032712661518758, -73.1931513039572 45.032712815424929," + - " -73.193151622737844 45.032712956204534, -73.193151952706273 45.032713083364762," + - " -73.193152292706571 45.0327131964597, -73.193152641548025 45.032713295093714," + - " -73.193152998007633 45.032713378920867, -73.193153360837016 45.032713447647481," + - " -73.193153728765438 45.032713501032966, -73.193154100503861 45.032713538890214," + - " -73.193154474749718 45.032713561086922, -73.193154850192016 45.032713567544562," + - " -73.193155225515255 45.032713558241319, -73.193155599404861 45.032713533209154," + - " -73.1931559705514 45.032713492536345, -73.193156337653889 45.032713436364965," + - " -73.193156699426623 45.032713364891968, -73.193195049010967 45.0327049699328, " + - "-73.193195555597441 45.032704841346956, -73.19319604499826 45.0327046827904, " + - "-73.1931965136942 45.032704495403159, -73.193196958314829 45.032704280532862, " + - "-73.193197375662976 45.032704039724337, -73.19319776273754 45.032703774709454," + - " -73.193198116754672 45.032703487393924, -73.193198435168952 45.0327031798439," + - " -73.193198715690087 45.032702854270973, -73.193198956301572 45.032702513016254," + - " -73.1931991552728 45.032702158533887, -73.193199311173331 45.032701793372674," + - " -73.193199311173331 45.032701793372674)))" - - val df = Seq(wkt).toDF("wkt") - - df.where(st_isvalid(col("wkt"))).count() should be(1) - - } - } diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidTest.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidTest.scala index 8b43abcc1..03ab3e939 100644 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidTest.scala +++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidTest.scala @@ -5,10 +5,9 @@ import org.apache.spark.sql.test.SharedSparkSession class ST_IsValidTest extends MosaicSpatialQueryTest with SharedSparkSession with ST_IsValidBehaviors { - //testAllGeometriesNoCodegen("Testing stIsValid NO_CODEGEN") { isValidBehaviour } - //testAllGeometriesCodegen("Testing stIsValid CODEGEN compilation") { isValidCodegen } - //testAllGeometriesCodegen("Testing stIsValid CODEGEN") { isValidBehaviour } - //testAllGeometriesNoCodegen("Testing auxiliary methods") { auxiliaryMethods } - testAllGeometriesNoCodegen("Testing stIsValid issue") { issue } + testAllGeometriesNoCodegen("Testing stIsValid NO_CODEGEN") { isValidBehaviour } + testAllGeometriesCodegen("Testing stIsValid CODEGEN compilation") { isValidCodegen } + testAllGeometriesCodegen("Testing stIsValid CODEGEN") { isValidBehaviour } + testAllGeometriesNoCodegen("Testing auxiliary methods") { auxiliaryMethods } } diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesBehaviors.scala deleted file mode 100644 index 5633115b0..000000000 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesBehaviors.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.databricks.labs.mosaic.expressions.raster - -import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI -import com.databricks.labs.mosaic.core.index.IndexSystem -import com.databricks.labs.mosaic.functions.MosaicContext -import org.apache.spark.sql.QueryTest -import org.apache.spark.sql.functions.lit -import org.scalatest.matchers.should.Matchers._ - -trait RST_GridTilesBehaviors extends QueryTest { - - //noinspection MapGetGet - def geoReferenceBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = { - val mc = MosaicContext.build(indexSystem, geometryAPI) - mc.register() - val sc = spark - import mc.functions._ - import sc.implicits._ - - val rastersAsPaths = spark.read - .format("gdal") - .option("raster_storage", "disk") - .load("src/test/resources/modis") - - val rastersInMemory = spark.read - .format("gdal") - .option("raster_storage", "in-memory") - .load("src/test/resources/modis") - - val gridTiles = rastersAsPaths - .withColumn("tiles", rst_gridtiles($"path", 3)) - .select("tiles") - - rastersInMemory - .createOrReplaceTempView("source") - - noException should be thrownBy spark.sql(""" - |select rst_gridtiles(raster, 3) from source - |""".stripMargin) - - noException should be thrownBy rastersInMemory - .withColumn("tiles", rst_gridtiles($"raster", 3)) - .select("tiles") - - val result = gridTiles.as[String].collect() - - result.length should be (980) - - } - -} diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesTest.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesTest.scala deleted file mode 100644 index 4f9db17aa..000000000 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GridTilesTest.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.databricks.labs.mosaic.expressions.raster - -import com.databricks.labs.mosaic.core.geometry.api.JTS -import com.databricks.labs.mosaic.core.index.H3IndexSystem -import org.apache.spark.sql.QueryTest -import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.test.SharedSparkSessionGDAL - -import scala.util.Try - -class RST_GridTilesTest extends QueryTest with SharedSparkSessionGDAL with RST_GridTilesBehaviors { - - private val noCodegen = - withSQLConf( - SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false", - SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString - ) _ - - // Hotfix for SharedSparkSession afterAll cleanup. - override def afterAll(): Unit = Try(super.afterAll()) - - // These tests are not index system nor geometry API specific. - // Only testing one pairing is sufficient. - test("Testing RST_GridTiles with manual GDAL registration (H3, JTS).") { - noCodegen { - assume(System.getProperty("os.name") == "Linux") - geoReferenceBehavior(H3IndexSystem, JTS) - } - } - -} diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala index 765ebd42d..210b8889a 100644 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala +++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala @@ -49,7 +49,7 @@ trait RST_MergeAggBehaviors extends QueryTest { |""".stripMargin) noException should be thrownBy rastersInMemory - .withColumn("tiles", rst_gridtiles($"path", 3)) + .withColumn("tiles", rst_tessellate($"path", 3)) .select("path", "tiles") .groupBy("path") .agg( diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala index 67fbc221a..822cf91ac 100644 --- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala +++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala @@ -28,7 +28,7 @@ trait RST_MergeBehaviors extends QueryTest { .load("src/test/resources/modis") val gridTiles = rastersAsPaths - .withColumn("tiles", rst_gridtiles($"path", 3)) + .withColumn("tiles", rst_tessellate($"path", 3)) .select("path", "tiles") .groupBy("path") .agg( @@ -55,7 +55,7 @@ trait RST_MergeBehaviors extends QueryTest { |""".stripMargin) noException should be thrownBy rastersInMemory - .withColumn("tiles", rst_gridtiles($"path", 3)) + .withColumn("tiles", rst_tessellate($"path", 3)) .select("path", "tiles") .groupBy("path") .agg(