Merge branch 'develop'

intranda · Dec 6, 2022 · 291450b · 291450b
2 parents fc64270 + a96a3f9
commit 291450b
Show file tree

Hide file tree

Showing 19 changed files with 361 additions and 104 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,10 @@
 FROM maven:3.6-jdk-11 AS BUILD
-# build indexer jar
+# you can use --build-arg build=false to skip viewer.war compilation, a viewer.war file needs to be available in target/viewer.war then
+ARG build=true
 
 COPY ./ /indexer
 WORKDIR /indexer
-RUN mvn -f goobi-viewer-indexer clean package
+RUN echo $build; if [ "$build" = "true" ]; then mvn -f goobi-viewer-indexer clean package; elif [ -f "/indexer/goobi-viewer-indexer/target/solr-Indexer.jar" ]; then echo "using existing indexer jar"; else echo "not supposed to build, but no indexer jar found either"; exit 1; fi 
 
 
 # start assembling the final image
@@ -14,6 +15,11 @@ LABEL org.opencontainers.image.authors="Matthias Geerdsen <matthias.geerdsen@int
 ENV SOLR_URL http://solr:8983/solr/collection1
 ENV VIEWER_URL http://viewer:8080/viewer
 
+RUN apt-get update && \
+	apt-get -y install libopenjp2-7 && \
+	apt-get -y clean && \
+	rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
 RUN mkdir -p /opt/digiverso/indexer
 
 COPY --from=BUILD  /indexer/goobi-viewer-indexer/target/solr-Indexer.jar /usr/local/bin/solrIndexer.jar

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -8,12 +8,6 @@ pipeline {
   }
 
   stages {
-    stage('prepare') {
-      agent any
-      steps {
-        sh 'git clean -fdx'
-      }
-    }
     stage('build') {
       agent {
         docker {
@@ -25,9 +19,11 @@ pipeline {
         }
       }
       steps {
-              sh 'mvn -f goobi-viewer-indexer/pom.xml -DskipTests=false clean verify -U'
-              recordIssues enabledForFailure: true, aggregatingResults: true, tools: [java(), javaDoc()]
-              dependencyCheckPublisher pattern: '**/target/dependency-check-report.xml'
+        sh 'git clean -fdx'
+        sh 'mvn -f goobi-viewer-indexer/pom.xml -DskipTests=false clean verify -U'
+        recordIssues enabledForFailure: true, aggregatingResults: true, tools: [java(), javaDoc()]
+        dependencyCheckPublisher pattern: '**/target/dependency-check-report.xml'
+        stash includes: '**/target/*.jar, */src/main/resources/*.xml, */src/main/resources/other/schema.xml', name:  'app'
       }
     }
     stage('sonarcloud') {
@@ -75,15 +71,17 @@ pipeline {
     stage('build, test and publish docker image') {
       agent {label 'controller'}
       steps {
+        unstash 'app'
+
         script{
           docker.withRegistry('https://nexus.intranda.com:4443','jenkins-docker'){
-            dockerimage = docker.build("goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
-            dockerimage_public = docker.build("intranda/goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
+            dockerimage = docker.build("goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}", "--no-cache --build-arg build=false .")
+            dockerimage_public = docker.build("intranda/goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}",  "--build-arg build=false .")
           }
         }
         script {
           dockerimage.inside {
-            sh 'test -f  /opt/digiverso/indexer/solrIndexer.jar || echo "/opt/digiverso/indexer/solrIndexer.jar missing"'
+            sh 'test -f /usr/local/bin/solrIndexer.jar || ( echo "/usr/local/bin/solrIndexer.jar missing"; exit 1 )'
           }
         }
         script {
@@ -152,7 +150,7 @@ pipeline {
     }
     success {
       node('controller') {
-        archiveArtifacts artifacts: '**/target/*.jar, */src/main/resources/indexerconfig_solr.xml, */src/main/resources/other/schema.xml, */src/main/resources/other/solrindexer.service', fingerprint: true
+        archiveArtifacts artifacts: '**/target/*.jar, */src/main/resources/*.xml, */src/main/resources/other/schema.xml, */src/main/resources/other/solrindexer.service', fingerprint: true
       }
     }
     changed {

diff --git a/goobi-viewer-indexer/.gitignore b/goobi-viewer-indexer/.gitignore
@@ -2,4 +2,3 @@
 /bin
 /doc
 /target/
-/rebel.xml
diff --git a/goobi-viewer-indexer/.project b/goobi-viewer-indexer/.project
@@ -20,11 +20,6 @@
 			<arguments>
 			</arguments>
 		</buildCommand>
-		<buildCommand>
-			<name>org.zeroturnaround.eclipse.rebelXmlBuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
 		<buildCommand>
 			<name>org.eclipse.m2e.core.maven2Builder</name>
 			<arguments>
@@ -36,6 +31,5 @@
 		<nature>org.eclipse.jdt.core.javanature</nature>
 		<nature>org.openmrs.generatetestcases.bddNature</nature>
 		<nature>net.sourceforge.pmd.eclipse.plugin.pmdNature</nature>
-		<nature>org.zeroturnaround.eclipse.jrebelNature</nature>
 	</natures>
 </projectDescription>
diff --git a/goobi-viewer-indexer/pom.xml b/goobi-viewer-indexer/pom.xml
@@ -6,7 +6,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>io.goobi.viewer</groupId>
 	<artifactId>viewer-indexer</artifactId>
-	<version>22.10</version>
+	<version>22.11-SNAPSHOT</version>
 
 
 	<name>Goobi viewer - Indexer</name>
@@ -35,7 +35,7 @@
 
 		<!-- intranda libraries -->
 		<alto.version>1.5.9</alto.version>
-		<iiif-api-model.version>2.2.2</iiif-api-model.version>
+		<iiif-api-model.version>2.3.0</iiif-api-model.version>
 		<normdataimporter.version>1.9.4</normdataimporter.version>
 
 		<!-- other libraries -->
@@ -45,12 +45,12 @@
 		<commons-jxpath.version>1.3</commons-jxpath.version>
 		<commons-lang3.version>3.12.0</commons-lang3.version>
         <commons-text.version>1.10.0</commons-text.version>
-		<httpclient.version>4.5.13</httpclient.version>
-		<httpcore.version>4.4.15</httpcore.version>
+		<httpclient.version>4.5.14</httpclient.version>
+		<httpcore.version>4.4.16</httpcore.version>
 		<icu.version>72.1</icu.version>
 		<imageio-openjpeg.version>0.6.2</imageio-openjpeg.version>
 		<log4j.version>2.19.0</log4j.version>
-		<jackson.version>2.13.4</jackson.version>
+		<jackson.version>2.14.1</jackson.version>
 		<jaxen.version>1.2.0</jaxen.version>
 		<jai.version>1.4.0</jai.version>
         <javax.mail.version>1.6.2</javax.mail.version>
@@ -218,7 +218,7 @@
 		<dependency>
 			<groupId>com.fasterxml.jackson.core</groupId>
 			<artifactId>jackson-databind</artifactId>
-			<version>2.13.4.2</version>
+			<version>${jackson.version}</version>
         </dependency>
 		<dependency>
 			<groupId>com.fasterxml.jackson.datatype</groupId>

diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/DublinCoreIndexer.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/DublinCoreIndexer.java
@@ -327,10 +327,11 @@ public String[] index(Path dcFile, Map<String, Path> dataFolders, ISolrWriteStra
                 writeUserGeneratedContents(writeStrategy, dataFolders, indexObj);
             }
 
-            // WRITE TO SOLR (POINT OF NO RETURN: any indexObj modifications from here on will not be included in the index!)
-            logger.debug("Writing document to index...");
             SolrInputDocument rootDoc = SolrSearchIndex.createDocument(indexObj.getLuceneFields());
             writeStrategy.setRootDoc(rootDoc);
+
+            // WRITE TO SOLR (POINT OF NO RETURN: any indexObj modifications from here on will not be included in the index!)
+            logger.debug("Writing document to index...");
             writeStrategy.writeDocs(Configuration.getInstance().isAggregateRecords());
             logger.info("Successfully finished indexing '{}'.", dcFile.getFileName());
         } catch (Exception e) {

diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/MetsIndexer.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/MetsIndexer.java
@@ -40,7 +40,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
@@ -505,6 +504,7 @@ public String[] index(Path metsFile, boolean fromReindexQueue, Map<String, Path>
             logger.debug("Writing document to index...");
             SolrInputDocument rootDoc = SolrSearchIndex.createDocument(indexObj.getLuceneFields());
             writeStrategy.setRootDoc(rootDoc);
+
             writeStrategy.writeDocs(Configuration.getInstance().isAggregateRecords());
             if (indexObj.isVolume() && (!indexObj.isUpdate() || indexedChildrenFileList)) {
                 logger.info("Re-indexing anchor...");
@@ -531,7 +531,7 @@ public String[] index(Path metsFile, boolean fromReindexQueue, Map<String, Path>
         return ret;
     }
 
-    private String getFilePathBannerFromFileSec(JDomXP xp, String filegroup) throws FatalIndexerException {
+    private static String getFilePathBannerFromFileSec(JDomXP xp, String filegroup) throws FatalIndexerException {
         String filePathBanner = "";
         String xpath = "/mets:mets/mets:fileSec/mets:fileGrp[@USE=\"" + filegroup + "\"]/mets:file[@USE=\"banner\"]/mets:FLocat/@xlink:href";
         filePathBanner = xp.evaluateToAttributeStringValue(xpath, null);
@@ -542,22 +542,23 @@ private String getFilePathBannerFromFileSec(JDomXP xp, String filegroup) throws
         }
         return "";
     }
-    
-    private String getFilePathBannerFromPhysicalStructMap(JDomXP xp, String filegroup) throws FatalIndexerException {
+
+    private static String getFilePathBannerFromPhysicalStructMap(JDomXP xp, String filegroup) throws FatalIndexerException {
         String filePathBanner = "";
-
-        String xpathFilePtr = "/mets:mets/mets:structMap[@TYPE='PHYSICAL']/mets:div[@TYPE='physSequence']/mets:div[@xlink:label=\"START_PAGE\"]/mets:fptr/@FILEID";
+
+        String xpathFilePtr =
+                "/mets:mets/mets:structMap[@TYPE='PHYSICAL']/mets:div[@TYPE='physSequence']/mets:div[@xlink:label=\"START_PAGE\"]/mets:fptr/@FILEID";
         List<String> fileIds = xp.evaluateToAttributes(xpathFilePtr, null).stream().map(Attribute::getValue).collect(Collectors.toList());
         for (String fileId : fileIds) {
-            String xpath = "/mets:mets/mets:fileSec/mets:fileGrp[@USE=\"" + filegroup + "\"]/mets:file[@ID='"+ fileId +"']/mets:FLocat/@xlink:href";
+            String xpath = "/mets:mets/mets:fileSec/mets:fileGrp[@USE=\"" + filegroup + "\"]/mets:file[@ID='" + fileId + "']/mets:FLocat/@xlink:href";
             filePathBanner = xp.evaluateToAttributeStringValue(xpath, null);
-            if(StringUtils.isNotBlank(filePathBanner)) {
+            if (StringUtils.isNotBlank(filePathBanner)) {
                 return filePathBanner;
             }
         }
         return filePathBanner;
     }
-    
+
     /**
      * Generates thumbnail info fields for the given docstruct. Also generates page docs mapped to this docstruct. <code>IndexObj.topstructPi</code>
      * must be set before calling this method.
@@ -593,11 +594,11 @@ private List<LuceneField> mapPagesToDocstruct(IndexObject indexObj, boolean isWo
         String filePathBanner = null;
         if (isWork) {
             filePathBanner = getFilePathBannerFromFileSec(xp, useFileGroupGlobal);
-            if(StringUtils.isNotBlank(filePathBanner)) {
+            if (StringUtils.isNotBlank(filePathBanner)) {
                 logger.debug("Found representation thumbnail for {} in METS filesec: {}", indexObj.getLogId(), filePathBanner);
             } else {
                 filePathBanner = getFilePathBannerFromPhysicalStructMap(xp, useFileGroupGlobal);
-                if(StringUtils.isNotBlank(filePathBanner)) {
+                if (StringUtils.isNotBlank(filePathBanner)) {
                     logger.debug("Found representation thumbnail for {} in METS physical structMap: {}", indexObj.getLogId(), filePathBanner);
                 } else if (StringUtils.isNotEmpty(indexObj.getThumbnailRepresent())) {
                     filePathBanner = indexObj.getThumbnailRepresent();

diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/WorldViewsIndexer.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/WorldViewsIndexer.java
@@ -354,10 +354,11 @@ public String[] index(Path mainFile, boolean fromReindexQueue, Map<String, Path>
                 indexedChildrenFileList = true;
             }
 
-            // WRITE TO SOLR (POINT OF NO RETURN: any indexObj modifications from here on will not be included in the index!)
-            logger.debug("Writing document to index...");
             SolrInputDocument rootDoc = SolrSearchIndex.createDocument(indexObj.getLuceneFields());
             writeStrategy.setRootDoc(rootDoc);
+
+            // WRITE TO SOLR (POINT OF NO RETURN: any indexObj modifications from here on will not be included in the index!)
+            logger.debug("Writing document to index...");
             writeStrategy.writeDocs(Configuration.getInstance().isAggregateRecords());
             if (indexObj.isVolume() && (!indexObj.isUpdate() || indexedChildrenFileList)) {
                 logger.info("Re-indexing anchor...");
@@ -382,7 +383,7 @@ public String[] index(Path mainFile, boolean fromReindexQueue, Map<String, Path>
      * Generates thumbnail info fields for the given docstruct. Also generates page docs mapped to this docstruct. <code>IndexObj.topstructPi</code>
      * must be set before calling this method.
      * 
-     * @param indexObj {@link IndexObject}
+     * @param rootIndexObj {@link IndexObject}
      * @param writeStrategy
      * @param dataFolders
      * @param depth Depth of the current docstruct in the docstruct hierarchy.

diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java
@@ -333,6 +333,11 @@ public Hotfolder(String confFilename, SolrClient solrClient, SolrClient oldSolrC
             logger.info("Authority data retrieval is disabled.");
         }
 
+        // REST API token configuration
+        if (StringUtils.isEmpty(Configuration.getInstance().getViewerAuthorizationToken())) {
+            logger.warn("Goobi viewer REST API token not found, communications disabled.");
+        }
+
         // E-mail configuration
         emailConfigurationComplete = checkEmailConfiguration();
         if (emailConfigurationComplete) {
@@ -1018,7 +1023,7 @@ protected void addMetsToIndex(Path metsFile, boolean fromReindexQueue, Map<Strin
                 // Delete all data folders for this record from the hotfolder
                 DataRepository.deleteDataFoldersFromHotfolder(dataFolders, reindexSettings);
             }
-            handleError(metsFile, resp[1]);
+            handleError(metsFile, resp[1], FileFormat.METS);
             try {
                 Files.delete(metsFile);
             } catch (IOException e) {
@@ -1164,8 +1169,6 @@ private void addLidoToIndex(Path lidoFile, Map<String, Boolean> reindexSettings)
                                 logger.info("{} MIX file(s) copied.", counter);
                             }
                         }
-                    } else {
-                        handleError(lidoFile, resp[1]);
                     }
 
                     // Update data repository cache map in the Goobi viewer
@@ -1176,6 +1179,8 @@ private void addLidoToIndex(Path lidoFile, Map<String, Boolean> reindexSettings)
                             logger.error(e.getMessage(), e);
                         }
                     }
+                } else {
+                    handleError(lidoFile, resp[1], FileFormat.LIDO);
                 }
             }
         } finally {
@@ -1333,6 +1338,8 @@ private void addDenkXwebToIndex(Path denkxwebFile, Map<String, Boolean> reindexS
                             logger.error(e.getMessage(), e);
                         }
                     }
+                } else {
+                    handleError(denkxwebFile, resp[1], FileFormat.DENKXWEB);
                 }
             }
         } finally {
@@ -1531,6 +1538,7 @@ private void addDublinCoreToIndex(Path dcFile, Map<String, Boolean> reindexSetti
                 // Delete all data folders for this record from the hotfolder
                 DataRepository.deleteDataFoldersFromHotfolder(dataFolders, reindexSettings);
             }
+            handleError(dcFile, resp[1], FileFormat.DUBLINCORE);
             try {
                 Files.delete(dcFile);
             } catch (IOException e) {
@@ -1669,7 +1677,7 @@ private void addWorldViewsToIndex(Path mainFile, boolean fromReindexQueue, Map<S
                 // Delete all data folders for this record from the hotfolder
                 DataRepository.deleteDataFoldersFromHotfolder(dataFolders, reindexSettings);
             }
-            handleError(mainFile, resp[1]);
+            handleError(mainFile, resp[1], FileFormat.WORLDVIEWS);
             try {
                 Files.delete(mainFile);
             } catch (IOException e) {
@@ -1780,7 +1788,7 @@ public boolean accept(Path entry) throws IOException {
                     Utils.deleteDirectory(dataFolders.get(DataRepository.PARAM_UGC));
                 }
             }
-            handleError(dataFile, resp[1]);
+            handleError(dataFile, resp[1], FileFormat.UNKNOWN);
             try {
                 Files.delete(dataFile);
             } catch (IOException e) {
@@ -1794,18 +1802,22 @@ public boolean accept(Path entry) throws IOException {
      * 
      * @param dataFile {@link File}
      * @param error
+     * @param format
      */
-    private void handleError(Path dataFile, String error) {
+    private void handleError(Path dataFile, String error, FileFormat format) {
+        logger.error("Failed to process '{}'.", dataFile.getFileName());
         // Error log file
-        File logFile = new File(errorMets.toFile(), FilenameUtils.getBaseName(dataFile.getFileName().toString()) + ".log");
-        try (FileWriter fw = new FileWriter(logFile); BufferedWriter out = new BufferedWriter(fw)) {
-            Files.copy(dataFile, Paths.get(errorMets.toAbsolutePath().toString(), dataFile.getFileName().toString()),
-                    StandardCopyOption.REPLACE_EXISTING);
-            if (error != null) {
-                out.write(error);
+        if (FileFormat.METS.equals(format)) {
+            File logFile = new File(errorMets.toFile(), FilenameUtils.getBaseName(dataFile.getFileName().toString()) + ".log");
+            try (FileWriter fw = new FileWriter(logFile); BufferedWriter out = new BufferedWriter(fw)) {
+                Files.copy(dataFile, Paths.get(errorMets.toAbsolutePath().toString(), dataFile.getFileName().toString()),
+                        StandardCopyOption.REPLACE_EXISTING);
+                if (error != null) {
+                    out.write(error);
+                }
+            } catch (IOException e) {
+                logger.error("Data file could not be moved to errorMets!", e);
             }
-        } catch (IOException e) {
-            logger.error("Data file could not be moved to errorMets!", e);
         }
     }
 
@@ -1870,14 +1882,14 @@ public DataFolderSizeCounter(String recordFileName) {
         }
 
         @Override
-        public boolean accept(File pathname) {
-            if (pathname.getName().startsWith(FilenameUtils.getBaseName(recordFileName) + "_")) {
+        public boolean accept(File pathName) {
+            if (pathName != null && pathName.getName().startsWith(FilenameUtils.getBaseName(recordFileName) + "_")) {
                 try {
-                    if (pathname.isFile()) {
-                        total += FileUtils.sizeOf(pathname);
-                    } else {
-                        pathname.listFiles(this);
-                        total += FileUtils.sizeOfDirectory(pathname);
+                    if (pathName.isFile()) {
+                        total += FileUtils.sizeOf(pathName);
+                    } else if (pathName.isDirectory()) {
+                        pathName.listFiles(this);
+                        total += FileUtils.sizeOfDirectory(pathName);
                     }
                 } catch (IllegalArgumentException e) {
                     logger.error(e.getMessage());
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,4 +2,3 @@ @@
     /bin
     /doc
     /target/
-    /rebel.xml