diff --git a/goobi-viewer-indexer/pom.xml b/goobi-viewer-indexer/pom.xml
index 47c5b233..a89f0037 100644
--- a/goobi-viewer-indexer/pom.xml
+++ b/goobi-viewer-indexer/pom.xml
@@ -6,7 +6,7 @@
4.0.0
io.goobi.viewer
viewer-indexer
- 22.07
+ 22.08-SNAPSHOT
Goobi viewer - Indexer
@@ -50,12 +50,12 @@
71.1
0.6.2
2.18.0
- 2.13.3
+ 2.13.4
1.2.0
1.4.0
1.6.2
2.0.6.1
- 1.15.2
+ 1.15.3
20220320
4.13.2
2.18.0
diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java
index 185e37c4..d81fb9dd 100644
--- a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java
+++ b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java
@@ -23,6 +23,7 @@
import java.util.Map.Entry;
import org.apache.commons.lang3.StringUtils;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrInputDocument;
import org.json.JSONException;
import org.json.JSONObject;
@@ -59,8 +60,16 @@ public UsageStatisticsIndexer(Hotfolder hotfolder) {
* @param sourceFile
* @throws IOException
* @throws FatalIndexerException
+ * @throws SolrServerException
*/
- public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexerException {
+ public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexerException, SolrServerException {
+ String solrDateString = getStatisticsDate(sourceFile);
+ if(statisticsExists(solrDateString)) {
+ logger.info("Don't index usage statistics for " + solrDateString + ": Statistics already exist for that date");
+ return null;
+ }
+
+
String jsonString = Files.readString(sourceFile);
if (StringUtils.isBlank(jsonString)) {
throw new IllegalArgumentException("Usage statistics file {} is empty".replace("{}", sourceFile.toString()));
@@ -75,6 +84,7 @@ public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexer
ISolrWriteStrategy writeStrategy = AbstractWriteStrategy.create(sourceFile, Collections.emptyMap(), this.hotfolder);
writeStrategy.setRootDoc(rootDoc);
writeStrategy.writeDocs(Configuration.getInstance().isAggregateRecords());
+ logger.info("Written usage statistics from " + sourceFile.toString() + " to index with IDDOC " + rootDoc.getFieldValue("IDDOC"));
return rootDoc;
} catch (JSONException | IndexerException e) {
throw new IllegalArgumentException("Usage statistics file {} contains invalid json".replace("{}", sourceFile.toString()));
@@ -82,6 +92,17 @@ public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexer
}
+ /**
+ * @param solrDateString
+ * @return
+ * @throws IOException
+ * @throws SolrServerException
+ */
+ private boolean statisticsExists(String solrDateString) throws SolrServerException, IOException {
+ String query = "+" + StatisticsLuceneFields.DATE + ":\"" + solrDateString + "\" +" + SolrConstants.DOCTYPE + ":" + StatisticsLuceneFields.USAGE_STATISTICS_DOCTYPE;
+ return hotfolder.getSearchIndex().getNumHits(query) > 0;
+ }
+
/**
* @param stats
* @return
@@ -90,6 +111,7 @@ public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexer
private IndexObject createIndexObject(DailyUsageStatistics stats) throws FatalIndexerException {
IndexObject indexObj = new IndexObject(getNextIddoc(hotfolder.getSearchIndex()));
indexObj.addToLucene(SolrConstants.IDDOC, Long.toString(indexObj.getIddoc()));
+ indexObj.addToLucene(SolrConstants.GROUPFIELD, Long.toString(indexObj.getIddoc()));
indexObj.addToLucene(SolrConstants.DOCTYPE, StatisticsLuceneFields.USAGE_STATISTICS_DOCTYPE);
indexObj.addToLucene(StatisticsLuceneFields.VIEWER_NAME, stats.getViewerName());
indexObj.addToLucene(StatisticsLuceneFields.DATE, StatisticsLuceneFields.solrDateFormatter.format(stats.getDate().atStartOfDay()));
@@ -119,12 +141,11 @@ private IndexObject createIndexObject(DailyUsageStatistics stats) throws FatalIn
* @throws FatalIndexerException
*/
public boolean removeFromIndex(Path sourceFile) throws FatalIndexerException {
- String dateString = sourceFile.getFileName().toString().replaceAll("statistics-usage-([\\d-]+).\\w+", "$1");
- LocalDate date = LocalDate.parse(dateString, DailyUsageStatistics.getDateformatter());
- String solrDateString = StatisticsLuceneFields.solrDateFormatter.format(date.atStartOfDay());
-
+ String solrDateString = getStatisticsDate(sourceFile);
+
try {
String query = "+" + StatisticsLuceneFields.DATE + ":\"" + solrDateString + "\" +" + SolrConstants.DOCTYPE + ":" + StatisticsLuceneFields.USAGE_STATISTICS_DOCTYPE;
+ logger.info("Deleting usage statistics for " + StatisticsLuceneFields.DATE + ":" + solrDateString);
return hotfolder.getSearchIndex().deleteByQuery(query);
} finally {
hotfolder.getSearchIndex().commit(false);
@@ -132,4 +153,11 @@ public boolean removeFromIndex(Path sourceFile) throws FatalIndexerException {
}
+ private String getStatisticsDate(Path sourceFile) {
+ String dateString = sourceFile.getFileName().toString().replaceAll("statistics-usage-([\\d-]+).\\w+", "$1");
+ LocalDate date = LocalDate.parse(dateString, DailyUsageStatistics.getDateformatter());
+ String solrDateString = StatisticsLuceneFields.solrDateFormatter.format(date.atStartOfDay());
+ return solrDateString;
+ }
+
}
diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java
index fe568aa8..bf4a6aa9 100644
--- a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java
+++ b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java
@@ -480,7 +480,7 @@ public boolean scan() throws FatalIndexerException {
return true;
}
logger.trace("Hotfolder: Listing files...");
- try (DirectoryStream stream = Files.newDirectoryStream(hotfolderPath, "*.{xml,delete,purge,docupdate,UPDATED}")) {
+ try (DirectoryStream stream = Files.newDirectoryStream(hotfolderPath, "*.{xml,json,delete,purge,docupdate,UPDATED}")) {
for (Path path : stream) {
// Only one file at a time right now
if (currentIndexer != null) {
@@ -648,10 +648,12 @@ private boolean handleSourceFile(Path sourceFile, boolean fromReindexQueue, Map<
} else if (filename.endsWith(".json")) {
if (filename.startsWith(FILENAME_PREFIX_STATISTICS_USAGE)) {
addUsageStatisticsToIndex(sourceFile);
+ Files.delete(sourceFile);
}
} else if (filename.endsWith(FILENAME_EXTENSION_DELETE)) {
if (filename.startsWith(FILENAME_PREFIX_STATISTICS_USAGE)) {
removeUsageStatisticsFromIndex(sourceFile);
+ Files.delete(sourceFile);
} else {
// DELETE
DataRepository[] repositories = dataRepositoryStrategy.selectDataRepository(null, sourceFile, null, searchIndex, oldSearchIndex);
@@ -661,6 +663,7 @@ private boolean handleSourceFile(Path sourceFile, boolean fromReindexQueue, Map<
} else if (filename.endsWith(FILENAME_EXTENSION_PURGE)) {
if (filename.startsWith(FILENAME_PREFIX_STATISTICS_USAGE)) {
removeUsageStatisticsFromIndex(sourceFile);
+ Files.delete(sourceFile);
} else {
// PURGE (delete with no "deleted" doc)
DataRepository[] repositories = dataRepositoryStrategy.selectDataRepository(null, sourceFile, null, searchIndex, oldSearchIndex);
@@ -1200,7 +1203,7 @@ private void addUsageStatisticsToIndex(Path sourceFile) {
try {
this.currentIndexer = new UsageStatisticsIndexer(this);
((UsageStatisticsIndexer) this.currentIndexer).index(sourceFile);
- } catch (IOException | IllegalArgumentException | FatalIndexerException e) {
+ } catch (IOException | IllegalArgumentException | FatalIndexerException | SolrServerException e) {
logger.error("Error indexing file {}. Reason: {}", sourceFile, e.getMessage());
} finally {
this.currentIndexer = null;
diff --git a/goobi-viewer-indexer/src/main/resources/config_indexer.xml b/goobi-viewer-indexer/src/main/resources/config_indexer.xml
index 5bb648fb..6ce78dc0 100644
--- a/goobi-viewer-indexer/src/main/resources/config_indexer.xml
+++ b/goobi-viewer-indexer/src/main/resources/config_indexer.xml
@@ -3420,7 +3420,20 @@
-
-
+
+
+ -
+
+
+ - mets:xmlData/mods:mods/mods:accessCondition[@type="use and reproduction"]
+ - dc:rights[contains(text(),'http')]
+ - mets:xmlData/mods:mods/mods:accessCondition[@lang="eng"][@type='work-license']
+
+
+ true
+ false
+
+
+