Hadoop Ingestion job configured with hdfs is failing with IllegalArgu…

…mentException (#14236) (#14244)
apache · Oct 17, 2024 · fe47073 · fe47073
1 parent ad37bd8
commit fe47073
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/...java/org/apache/pinot/plugin/ingestion/batch/hadoop/HadoopSegmentGenerationJobRunner.java b/...java/org/apache/pinot/plugin/ingestion/batch/hadoop/HadoopSegmentGenerationJobRunner.java
@@ -349,7 +349,11 @@ protected void addMapperJarToDistributedCache(Job job, PinotFS outputDirFS, URI
       throws Exception {
     File ourJar = new File(getClass().getProtectionDomain().getCodeSource().getLocation().toURI());
     Path distributedCacheJar = new Path(stagingDirURI.toString(), ourJar.getName());
-    outputDirFS.copyFromLocalDir(ourJar, distributedCacheJar.toUri());
+    if (ourJar.isDirectory()) {
+      outputDirFS.copyFromLocalDir(ourJar, distributedCacheJar.toUri());
+    } else {
+      outputDirFS.copyFromLocalFile(ourJar, distributedCacheJar.toUri());
+    }
     job.addFileToClassPath(distributedCacheJar);
   }