paused, switching back to focal

databrickslabs · Aug 14, 2023 · 639c3a9 · 639c3a9
1 parent 8a83eed
commit 639c3a9
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 35 deletions.
diff --git a/src/main/resources/scripts/install-gdal-databricks.sh b/src/main/resources/scripts/install-gdal-databricks.sh
@@ -7,27 +7,35 @@
 
 MOSAIC_GDAL_JNI_DIR="${MOSAIC_GDAL_JNI_DIR:-__DEFAULT_JNI_PATH__}"
 
-sudo rm -r /var/lib/apt/lists/*
-sudo add-apt-repository main
-sudo add-apt-repository universe
-sudo add-apt-repository restricted
-sudo add-apt-repository multiverse
-sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable
-sudo apt clean && sudo apt -o Acquire::Retries=3 update --fix-missing -y
-sudo apt-get -o Acquire::Retries=3 update -y
-sudo apt-get -o Acquire::Retries=3 install -y gdal-bin=3.4.3+dfsg-1~focal0 libgdal-dev=3.4.3+dfsg-1~focal0 python3-gdal=3.4.3+dfsg-1~focal0
+rm -rf /var/lib/apt/lists/
+mkdir -p /var/lib/apt/lists/
+add-apt-repository main
+add-apt-repository universe
+add-apt-repository restricted
+add-apt-repository multiverse
+add-apt-repository ppa:ubuntugis/ubuntugis-unstable
+apt clean && sudo apt -o Acquire::Retries=3 update --fix-missing -y
+apt-get -o Acquire::Retries=3 update -y
+apt-get -o Acquire::Retries=3 install -y gdal-bin=3.6.4+dfsg-1~jammy0 libgdal-dev=3.6.4+dfsg-1~jammy0 python3-gdal=3.6.4+dfsg-1~jammy0
 
 # fix python file naming in osgeo package
-cd /usr/lib/python3/dist-packages/osgeo \
-  && mv _gdal.cpython-38-x86_64-linux-gnu.so _gdal.so \
-  && mv _gdal_array.cpython-38-x86_64-linux-gnu.so _gdal_array.so \
-  && mv _gdalconst.cpython-38-x86_64-linux-gnu.so _gdalconst.so \
-  && mv _ogr.cpython-38-x86_64-linux-gnu.so _ogr.so \
-  && mv _gnm.cpython-38-x86_64-linux-gnu.so _gnm.so \
-  && mv _osr.cpython-38-x86_64-linux-gnu.so _osr.so
+cd /usr/lib/python3/dist-packages/osgeo
+# Loop over all .so files
+for file in *.so; do
+    # Strip part of name bookended with full-stops and rename file
+    new_name=$(echo "$file" | sed 's/\.cpython.*\./\./')
+    mv "$file" "$new_name"
+done
+#cd /usr/lib/python3/dist-packages/osgeo \
+#  && mv _gdal.cpython-38-x86_64-linux-gnu.so _gdal.so \
+#  && mv _gdal_array.cpython-38-x86_64-linux-gnu.so _gdal_array.so \
+#  && mv _gdalconst.cpython-38-x86_64-linux-gnu.so _gdalconst.so \
+#  && mv _ogr.cpython-38-x86_64-linux-gnu.so _ogr.so \
+#  && mv _gnm.cpython-38-x86_64-linux-gnu.so _gnm.so \
+#  && mv _osr.cpython-38-x86_64-linux-gnu.so _osr.so
 
 # add pre-build JNI shared object to the path
 # please run MosaicGDAL.copySharedObjects("/dbfs/FileStore/geospatial/mosaic/gdal/") before enabling this init script
-mkdir -p /usr/lib/jni
-cp "${MOSAIC_GDAL_JNI_DIR}/libgdalalljni.so" /usr/lib/jni
-cp "${MOSAIC_GDAL_JNI_DIR}/libgdalalljni.so.30" /usr/lib/jni
+#mkdir -p /usr/lib/jni
+#cp "${MOSAIC_GDAL_JNI_DIR}/libgdalalljni.so" /usr/lib/jni
+#cp "${MOSAIC_GDAL_JNI_DIR}/libgdalalljni.so.30" /usr/lib/jni
diff --git a/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala b/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala
@@ -60,7 +60,7 @@ object MosaicGDAL extends Logging {
         val so = readResourceBytes("/gdal/ubuntu/libgdalalljni.so")
         val so30 = readResourceBytes("/gdal/ubuntu/libgdalalljni.so.30")
 
-        val usrGDALPath = Paths.get("/usr/lib/jni/")
+        val usrGDALPath = Paths.get(path)
         if (!Files.exists(mosaicGDALPath)) Files.createDirectories(mosaicGDALPath)
         if (!Files.exists(usrGDALPath)) Files.createDirectories(usrGDALPath)
         Files.write(Paths.get(s"$mosaicGDALAbsolutePath/libgdalalljni.so"), so)

diff --git a/src/test/scala/com/databricks/labs/mosaic/test/TestMosaicGDAL.scala b/src/test/scala/com/databricks/labs/mosaic/test/TestMosaicGDAL.scala
@@ -12,32 +12,37 @@ import scala.sys.process._
 
 object TestMosaicGDAL extends Logging {
 
-    def installGDAL(spark: SparkSession): Unit = {
-        if (!wasEnabled(spark) && !isEnabled) installGDAL(Some(spark))
+    def installGDAL(spark: SparkSession, initScriptPath: String, sharedObjectsPath: String): Unit = {
+        if (!wasEnabled(spark) && !isEnabled) installGDAL(Some(spark), initScriptPath, sharedObjectsPath)
     }
 
-    def installGDAL(spark: Option[SparkSession]): Unit = {
+    def installGDAL(spark: Option[SparkSession], initScriptPath: String, sharedObjectsPath: String): Unit = {
         val sc = spark.map(_.sparkContext)
         val numExecutors = sc.map(_.getExecutorMemoryStatus.size - 1)
-        val script = getScript
-        for (cmd <- script.getLines.toList) {
-            try {
-                if (!cmd.startsWith("#") || cmd.nonEmpty) cmd.!!
-                sc.map { sparkContext =>
-                    if (!sparkContext.isLocal) {
-                        sparkContext.parallelize(1 to numExecutors.get).pipe(cmd).collect
-                    }
-                }
+        val scriptPath = s"$initScriptPath/mosaic-gdal-init.sh"
+        sc.foreach(_.addFile(scriptPath))
+        try {
+            s"sudo sh $scriptPath".!!
+//        val script = getScript
+//        for (cmd <- script.getLines.toList) {
+//            try {
+//                if (!cmd.startsWith("#") & cmd.nonEmpty) {
+//                    s"sudo bash -c '$cmd'".!!
+//                    sc.map { sparkContext =>
+//                        if (!sparkContext.isLocal) {
+//                            sparkContext.parallelize(1 to numExecutors.get).pipe(cmd).collect
+//                        }
+//                    }
+//                }
             } catch {
                 case e: IOException           => logError(e.getMessage)
                 case e: IllegalStateException => logError(e.getMessage)
                 case e: SparkException        => logError(e.getMessage)
                 case e: Throwable             => logError(e.getMessage)
             } finally {
-                script.close
+//                script.close
             }
         }
-    }
 
     def getScript: BufferedSource = {
         System.getProperty("os.name").toLowerCase() match {

diff --git a/src/test/scala/org/apache/spark/sql/test/SharedSparkSessionGDAL.scala b/src/test/scala/org/apache/spark/sql/test/SharedSparkSessionGDAL.scala
@@ -23,9 +23,9 @@ trait SharedSparkSessionGDAL extends SharedSparkSession {
         val session = new TestSparkSession(conf)
         if (conf.get(MOSAIC_GDAL_NATIVE, "false").toBoolean) {
             Try {
-                TestMosaicGDAL.installGDAL(session)
                 val tempPath = Files.createTempDirectory("mosaic-gdal")
                 MosaicGDAL.prepareEnvironment(session, tempPath.toAbsolutePath.toString, "/usr/lib/jni")
+                TestMosaicGDAL.installGDAL(session, tempPath.toAbsolutePath.toString, "/usr/lib/jni")
                 MosaicGDAL.enableGDAL(session)
             }
         }