diff --git a/goobi-viewer-indexer/pom.xml b/goobi-viewer-indexer/pom.xml index 47c5b233..a89f0037 100644 --- a/goobi-viewer-indexer/pom.xml +++ b/goobi-viewer-indexer/pom.xml @@ -6,7 +6,7 @@ 4.0.0 io.goobi.viewer viewer-indexer - 22.07 + 22.08-SNAPSHOT Goobi viewer - Indexer @@ -50,12 +50,12 @@ 71.1 0.6.2 2.18.0 - 2.13.3 + 2.13.4 1.2.0 1.4.0 1.6.2 2.0.6.1 - 1.15.2 + 1.15.3 20220320 4.13.2 2.18.0 diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java index 185e37c4..d81fb9dd 100644 --- a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java +++ b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/UsageStatisticsIndexer.java @@ -23,6 +23,7 @@ import java.util.Map.Entry; import org.apache.commons.lang3.StringUtils; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrInputDocument; import org.json.JSONException; import org.json.JSONObject; @@ -59,8 +60,16 @@ public UsageStatisticsIndexer(Hotfolder hotfolder) { * @param sourceFile * @throws IOException * @throws FatalIndexerException + * @throws SolrServerException */ - public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexerException { + public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexerException, SolrServerException { + String solrDateString = getStatisticsDate(sourceFile); + if(statisticsExists(solrDateString)) { + logger.info("Don't index usage statistics for " + solrDateString + ": Statistics already exist for that date"); + return null; + } + + String jsonString = Files.readString(sourceFile); if (StringUtils.isBlank(jsonString)) { throw new IllegalArgumentException("Usage statistics file {} is empty".replace("{}", sourceFile.toString())); @@ -75,6 +84,7 @@ public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexer ISolrWriteStrategy writeStrategy = AbstractWriteStrategy.create(sourceFile, Collections.emptyMap(), this.hotfolder); writeStrategy.setRootDoc(rootDoc); writeStrategy.writeDocs(Configuration.getInstance().isAggregateRecords()); + logger.info("Written usage statistics from " + sourceFile.toString() + " to index with IDDOC " + rootDoc.getFieldValue("IDDOC")); return rootDoc; } catch (JSONException | IndexerException e) { throw new IllegalArgumentException("Usage statistics file {} contains invalid json".replace("{}", sourceFile.toString())); @@ -82,6 +92,17 @@ public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexer } + /** + * @param solrDateString + * @return + * @throws IOException + * @throws SolrServerException + */ + private boolean statisticsExists(String solrDateString) throws SolrServerException, IOException { + String query = "+" + StatisticsLuceneFields.DATE + ":\"" + solrDateString + "\" +" + SolrConstants.DOCTYPE + ":" + StatisticsLuceneFields.USAGE_STATISTICS_DOCTYPE; + return hotfolder.getSearchIndex().getNumHits(query) > 0; + } + /** * @param stats * @return @@ -90,6 +111,7 @@ public SolrInputDocument index(Path sourceFile) throws IOException, FatalIndexer private IndexObject createIndexObject(DailyUsageStatistics stats) throws FatalIndexerException { IndexObject indexObj = new IndexObject(getNextIddoc(hotfolder.getSearchIndex())); indexObj.addToLucene(SolrConstants.IDDOC, Long.toString(indexObj.getIddoc())); + indexObj.addToLucene(SolrConstants.GROUPFIELD, Long.toString(indexObj.getIddoc())); indexObj.addToLucene(SolrConstants.DOCTYPE, StatisticsLuceneFields.USAGE_STATISTICS_DOCTYPE); indexObj.addToLucene(StatisticsLuceneFields.VIEWER_NAME, stats.getViewerName()); indexObj.addToLucene(StatisticsLuceneFields.DATE, StatisticsLuceneFields.solrDateFormatter.format(stats.getDate().atStartOfDay())); @@ -119,12 +141,11 @@ private IndexObject createIndexObject(DailyUsageStatistics stats) throws FatalIn * @throws FatalIndexerException */ public boolean removeFromIndex(Path sourceFile) throws FatalIndexerException { - String dateString = sourceFile.getFileName().toString().replaceAll("statistics-usage-([\\d-]+).\\w+", "$1"); - LocalDate date = LocalDate.parse(dateString, DailyUsageStatistics.getDateformatter()); - String solrDateString = StatisticsLuceneFields.solrDateFormatter.format(date.atStartOfDay()); - + String solrDateString = getStatisticsDate(sourceFile); + try { String query = "+" + StatisticsLuceneFields.DATE + ":\"" + solrDateString + "\" +" + SolrConstants.DOCTYPE + ":" + StatisticsLuceneFields.USAGE_STATISTICS_DOCTYPE; + logger.info("Deleting usage statistics for " + StatisticsLuceneFields.DATE + ":" + solrDateString); return hotfolder.getSearchIndex().deleteByQuery(query); } finally { hotfolder.getSearchIndex().commit(false); @@ -132,4 +153,11 @@ public boolean removeFromIndex(Path sourceFile) throws FatalIndexerException { } + private String getStatisticsDate(Path sourceFile) { + String dateString = sourceFile.getFileName().toString().replaceAll("statistics-usage-([\\d-]+).\\w+", "$1"); + LocalDate date = LocalDate.parse(dateString, DailyUsageStatistics.getDateformatter()); + String solrDateString = StatisticsLuceneFields.solrDateFormatter.format(date.atStartOfDay()); + return solrDateString; + } + } diff --git a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java index fe568aa8..bf4a6aa9 100644 --- a/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java +++ b/goobi-viewer-indexer/src/main/java/io/goobi/viewer/indexer/helper/Hotfolder.java @@ -480,7 +480,7 @@ public boolean scan() throws FatalIndexerException { return true; } logger.trace("Hotfolder: Listing files..."); - try (DirectoryStream stream = Files.newDirectoryStream(hotfolderPath, "*.{xml,delete,purge,docupdate,UPDATED}")) { + try (DirectoryStream stream = Files.newDirectoryStream(hotfolderPath, "*.{xml,json,delete,purge,docupdate,UPDATED}")) { for (Path path : stream) { // Only one file at a time right now if (currentIndexer != null) { @@ -648,10 +648,12 @@ private boolean handleSourceFile(Path sourceFile, boolean fromReindexQueue, Map< } else if (filename.endsWith(".json")) { if (filename.startsWith(FILENAME_PREFIX_STATISTICS_USAGE)) { addUsageStatisticsToIndex(sourceFile); + Files.delete(sourceFile); } } else if (filename.endsWith(FILENAME_EXTENSION_DELETE)) { if (filename.startsWith(FILENAME_PREFIX_STATISTICS_USAGE)) { removeUsageStatisticsFromIndex(sourceFile); + Files.delete(sourceFile); } else { // DELETE DataRepository[] repositories = dataRepositoryStrategy.selectDataRepository(null, sourceFile, null, searchIndex, oldSearchIndex); @@ -661,6 +663,7 @@ private boolean handleSourceFile(Path sourceFile, boolean fromReindexQueue, Map< } else if (filename.endsWith(FILENAME_EXTENSION_PURGE)) { if (filename.startsWith(FILENAME_PREFIX_STATISTICS_USAGE)) { removeUsageStatisticsFromIndex(sourceFile); + Files.delete(sourceFile); } else { // PURGE (delete with no "deleted" doc) DataRepository[] repositories = dataRepositoryStrategy.selectDataRepository(null, sourceFile, null, searchIndex, oldSearchIndex); @@ -1200,7 +1203,7 @@ private void addUsageStatisticsToIndex(Path sourceFile) { try { this.currentIndexer = new UsageStatisticsIndexer(this); ((UsageStatisticsIndexer) this.currentIndexer).index(sourceFile); - } catch (IOException | IllegalArgumentException | FatalIndexerException e) { + } catch (IOException | IllegalArgumentException | FatalIndexerException | SolrServerException e) { logger.error("Error indexing file {}. Reason: {}", sourceFile, e.getMessage()); } finally { this.currentIndexer = null; diff --git a/goobi-viewer-indexer/src/main/resources/config_indexer.xml b/goobi-viewer-indexer/src/main/resources/config_indexer.xml index 5bb648fb..6ce78dc0 100644 --- a/goobi-viewer-indexer/src/main/resources/config_indexer.xml +++ b/goobi-viewer-indexer/src/main/resources/config_indexer.xml @@ -3420,7 +3420,20 @@ - - + + + + + + mets:xmlData/mods:mods/mods:accessCondition[@type="use and reproduction"] + dc:rights[contains(text(),'http')] + mets:xmlData/mods:mods/mods:accessCondition[@lang="eng"][@type='work-license'] + + + true + false + + +