Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
@janvonde
janvonde committed Nov 2, 2022
2 parents 2e52e7c + 667340e commit 1eeb26f
Show file tree
Hide file tree
Showing 10 changed files with 218 additions and 143 deletions.
27 changes: 10 additions & 17 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ pipeline {
stage('build') {
agent {
docker {
label 'controller'
image 'nexus.intranda.com:4443/goobi-viewer-testing-index:latest'
args '-v $HOME/.m2:/var/maven/.m2:z -v $HOME/.config:/var/maven/.config -v $HOME/.sonar:/var/maven/.sonar -u 1000 -ti -e _JAVA_OPTIONS=-Duser.home=/var/maven -e MAVEN_CONFIG=/var/maven/.m2'
registryUrl 'https://nexus.intranda.com:4443/'
Expand All @@ -38,6 +39,7 @@ pipeline {
}
agent {
docker {
label 'controller'
image 'nexus.intranda.com:4443/goobi-viewer-testing-index:latest'
args '-v $HOME/.m2:/var/maven/.m2:z -v $HOME/.config:/var/maven/.config -v $HOME/.sonar:/var/maven/.sonar -u 1000 -ti -e _JAVA_OPTIONS=-Duser.home=/var/maven -e MAVEN_CONFIG=/var/maven/.m2'
registryUrl 'https://nexus.intranda.com:4443/'
Expand All @@ -53,6 +55,7 @@ pipeline {
stage('deployment to maven repository') {
agent {
docker {
label 'controller'
image 'nexus.intranda.com:4443/goobi-viewer-testing-index:latest'
args '-v $HOME/.m2:/var/maven/.m2:z -v $HOME/.config:/var/maven/.config -v $HOME/.sonar:/var/maven/.sonar -u 1000 -ti -e _JAVA_OPTIONS=-Duser.home=/var/maven -e MAVEN_CONFIG=/var/maven/.m2'
registryUrl 'https://nexus.intranda.com:4443/'
Expand All @@ -69,30 +72,20 @@ pipeline {
sh 'mvn -f goobi-viewer-indexer/pom.xml deploy'
}
}
stage('build docker image') {
agent any
stage('build, test and publish docker image') {
agent {label 'controller'}
steps {
script{
docker.withRegistry('https://nexus.intranda.com:4443','jenkins-docker'){
dockerimage = docker.build("goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
dockerimage_public = docker.build("intranda/goobi-viewer-indexer:${BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
}
}
}
}
stage('basic tests'){
agent any
steps{
script {
dockerimage.inside {
sh 'test -f /opt/digiverso/indexer/solrIndexer.jar || echo "/opt/digiverso/indexer/solrIndexer.jar missing"'
}
}
}
}
stage('publish docker devel image to internal repository'){
agent any
steps{
script {
docker.withRegistry('https://nexus.intranda.com:4443','jenkins-docker'){
dockerimage.push("${env.BRANCH_NAME}-${env.BUILD_ID}_${env.GIT_COMMIT}")
Expand All @@ -102,7 +95,7 @@ pipeline {
}
}
stage('publish docker production image to internal repository'){
agent any
agent {label 'controller'}
when {
tag "v*"
}
Expand All @@ -117,7 +110,7 @@ pipeline {
}
}
stage('publish develop image to Docker Hub'){
agent any
agent {label 'controller'}
when {
branch 'develop'
}
Expand All @@ -130,7 +123,7 @@ pipeline {
}
}
stage('publish production image to Docker Hub'){
agent any
agent {label 'controller'}
when {
tag "v*"
}
Expand All @@ -146,7 +139,7 @@ pipeline {
}
post {
always {
node(null) {
node('controller') {
junit "**/target/surefire-reports/*.xml"
step([
$class : 'JacocoPublisher',
Expand All @@ -158,7 +151,7 @@ pipeline {
}
}
success {
node(null){
node('controller') {
archiveArtifacts artifacts: '**/target/*.jar, */src/main/resources/indexerconfig_solr.xml, */src/main/resources/other/schema.xml, */src/main/resources/other/solrindexer.service', fingerprint: true
}
}
Expand Down
6 changes: 3 additions & 3 deletions goobi-viewer-indexer/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>io.goobi.viewer</groupId>
<artifactId>viewer-indexer</artifactId>
<version>22.09</version>
<version>22.10-SNAPSHOT</version>


<name>Goobi viewer - Indexer</name>
Expand Down Expand Up @@ -47,7 +47,7 @@
<commons-text.version>1.10.0</commons-text.version>
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.15</httpcore.version>
<icu.version>71.1</icu.version>
<icu.version>72.1</icu.version>
<imageio-openjpeg.version>0.6.2</imageio-openjpeg.version>
<log4j.version>2.19.0</log4j.version>
<jackson.version>2.13.4</jackson.version>
Expand Down Expand Up @@ -218,7 +218,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
<version>2.13.4.2</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
Expand Down
15 changes: 7 additions & 8 deletions goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Configuration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Timer;

import org.apache.commons.configuration2.HierarchicalConfiguration;
Expand Down Expand Up @@ -202,11 +203,9 @@ public void initNamespaces() {
namespaces.put("pnx", Namespace.getNamespace("pnx", "http://www.exlibrisgroup.com/xsd/primo/primo_nm_bib"));

Map<String, String> additionalNamespaces = getListConfiguration("init.namespaces");
if (additionalNamespaces != null) {
for (String key : additionalNamespaces.keySet()) {
namespaces.put(key, Namespace.getNamespace(key, additionalNamespaces.get(key)));
logger.info("Added custom namespace '{}'.", key);
}
for (Entry<String, String> entry : additionalNamespaces.entrySet()) {
namespaces.put(entry.getKey(), Namespace.getNamespace(entry.getKey(), entry.getValue()));
logger.info("Added custom namespace '{}'.", entry.getKey());
}
}

Expand Down Expand Up @@ -412,7 +411,7 @@ public int getThreads() {
public boolean isCountHotfolderFiles() {
return getBoolean("performance.countHotfolderFiles", true);
}

/**
*
* @return
Expand All @@ -421,7 +420,7 @@ public boolean isCountHotfolderFiles() {
public boolean isAuthorityDataCacheEnabled() {
return getBoolean("performance.authorityDataCache[@enabled]", true);
}

/**
*
* @return
Expand All @@ -430,7 +429,7 @@ public boolean isAuthorityDataCacheEnabled() {
public int getAuthorityDataCacheRecordTTL() {
return getInt("performance.authorityDataCache.recordTTL", 24);
}

/**
*
* @return
Expand Down
52 changes: 25 additions & 27 deletions goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/DateTools.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,29 +182,6 @@ static List<PrimitiveDate> normalizeDate(String dateString, int normalizeYearMin
return ret;
}

/**
*
* @param value
* @param useUTC If true, UTC time zone will be used; default time zone otherwise
* @return Converted datetime string
* @should convert date correctly
*/
public static String convertDateStringForSolrField(String value, boolean useUTC) {
List<PrimitiveDate> dates = normalizeDate(value, 4);
if (!dates.isEmpty()) {
PrimitiveDate date = dates.get(0);
if (date.getYear() != null) {
ZonedDateTime ld =
LocalDateTime.of(date.getYear(), date.getMonth() != null ? date.getMonth() : 1, date.getDay() != null ? date.getDay() : 1, 0,
0, 0, 0).atZone(useUTC ? ZoneOffset.UTC : ZoneId.systemDefault());
return ld.format(formatterISO8601DateTimeInstant);
}
}

logger.warn("Could not parse date from value: {}", value);
return null;
}

/**
*
* @param value Raw value
Expand All @@ -214,6 +191,7 @@ public static String convertDateStringForSolrField(String value, boolean useUTC)
* @should format iso local time correctly
*/
public static String normalizeDateFieldValue(String value) {
logger.trace("normalizeDateFieldValue: {}", value);
if (StringUtils.isEmpty(value)) {
return "";
}
Expand All @@ -232,11 +210,31 @@ public static String normalizeDateFieldValue(String value) {
logger.trace(e.getMessage());
}

List<PrimitiveDate> dates = DateTools.normalizeDate(value, 4);
if (dates.isEmpty()) {
return "";
return convertDateStringForSolrField(value, true);
}

/**
* Converts non-ISO date/time strings to ISO instant (at UTC or system time zone).
*
* @param value
* @param useUTC If true, UTC time zone will be used; default time zone otherwise
* @return Converted ISO instant
* @should convert date correctly
*/
static String convertDateStringForSolrField(String value, boolean useUTC) {
List<PrimitiveDate> dates = normalizeDate(value, 4);
if (!dates.isEmpty()) {
PrimitiveDate date = dates.get(0);
if (date.getYear() != null) {
ZonedDateTime ld =
LocalDateTime.of(date.getYear(), date.getMonth() != null ? date.getMonth() : 1, date.getDay() != null ? date.getDay() : 1, 0,
0, 0, 0).atZone(useUTC ? ZoneOffset.UTC : ZoneId.systemDefault());
return ld.format(formatterISO8601DateTimeInstant);
}
}

return dates.get(0).toLocalDateTime().atZone(ZoneOffset.UTC).format(formatterISO8601DateTimeInstant);
logger.warn("Could not parse date from value: {}", value);
return "";
}

}
15 changes: 10 additions & 5 deletions goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,12 +320,17 @@ public Hotfolder(String confFilename, SolrClient solrClient, SolrClient oldSolrC
logger.info("Content files will be PRESERVED in the hotfolder in case of indexing errors.");
}

// Norm data fields to be added to DEFAULT
MetadataHelper.addAuthorityDataFieldsToDefault = config.getList("init.addNormDataFieldsToDefault.field");
if (MetadataHelper.addAuthorityDataFieldsToDefault != null) {
for (String field : MetadataHelper.addAuthorityDataFieldsToDefault) {
logger.info("{} values will be added to DEFAULT", field);
MetadataHelper.authorityDataEnabled = config.getBoolean("init.authorityData[@enabled]", true);
if (MetadataHelper.authorityDataEnabled) {
// Authority data fields to be added to DEFAULT
MetadataHelper.addAuthorityDataFieldsToDefault = config.getList("init.authorityData.addFieldsToDefault.field");
if (MetadataHelper.addAuthorityDataFieldsToDefault != null) {
for (String field : MetadataHelper.addAuthorityDataFieldsToDefault) {
logger.info("{} values will be added to DEFAULT", field);
}
}
} else {
logger.info("Authority data retrieval is disabled.");
}

// E-mail configuration
Expand Down
Loading

0 comments on commit 1eeb26f

Please sign in to comment.