diff --git a/README.md b/README.md index ccb757c0..6789cd03 100644 --- a/README.md +++ b/README.md @@ -508,6 +508,11 @@ a single data elements (a DataELement in the API). Its properties are: against * `indexField` (String): the name which can be used in a search engine connected to the application (at the time of writing Apache Solr is supported) +* `inactive` (boolean): the data element is inactive, do not run checks on this +* `identifierField` (boolean): the data element is the identifier of the record +* `asLanguageTagged` (boolean): treat the data element as language tagged. It works + for JSON where the content of the data element is encoded with an associated + array, where the keys are the language tags. Optionaly you can set the "canonical list" of categories. It provides two additional functionalities diff --git a/src/main/java/de/gwdg/metadataqa/api/calculator/CalculatorFacade.java b/src/main/java/de/gwdg/metadataqa/api/calculator/CalculatorFacade.java index 627d88dc..93374053 100644 --- a/src/main/java/de/gwdg/metadataqa/api/calculator/CalculatorFacade.java +++ b/src/main/java/de/gwdg/metadataqa/api/calculator/CalculatorFacade.java @@ -240,6 +240,7 @@ protected Object measureCsvWithGenerics(List result = calculator.measure(cache); collector.addResult(calculator, result, compressionLevel); } diff --git a/src/main/java/de/gwdg/metadataqa/api/calculator/FieldExtractor.java b/src/main/java/de/gwdg/metadataqa/api/calculator/FieldExtractor.java index 1378b762..494e4f7e 100644 --- a/src/main/java/de/gwdg/metadataqa/api/calculator/FieldExtractor.java +++ b/src/main/java/de/gwdg/metadataqa/api/calculator/FieldExtractor.java @@ -4,6 +4,7 @@ import de.gwdg.metadataqa.api.counter.FieldCounter; import de.gwdg.metadataqa.api.interfaces.Calculator; import de.gwdg.metadataqa.api.interfaces.MetricResult; +import de.gwdg.metadataqa.api.json.DataElement; import de.gwdg.metadataqa.api.model.EdmFieldInstance; import de.gwdg.metadataqa.api.model.selector.Selector; import de.gwdg.metadataqa.api.model.XmlFieldInstance; @@ -56,10 +57,13 @@ public List measure(Selector cache) if (schema != null) { String path; + DataElement dataELement; + boolean asLanguageTagged; for (String fieldName : schema.getExtractableFields().keySet()) { if (idPath == null || !fieldName.equals(FIELD_NAME)) { + dataELement = schema.getPathByLabel(fieldName); path = schema.getExtractableFields().get(fieldName); - extractSingleField(cache, resultMap, path, fieldName); + extractSingleField(cache, resultMap, path, fieldName, dataELement); } } } @@ -67,7 +71,20 @@ public List measure(Selector cache) } private void extractSingleField(Selector cache, FieldCounter resultMap, String path, String fieldName) { - List values = cache.get(path); + extractSingleField(cache, resultMap, path, fieldName,null); + } + + private void extractSingleField(Selector cache, + FieldCounter resultMap, + String path, + String fieldName, + DataElement dataELement) { + List values; + if (dataELement != null) { + values = cache.get(dataELement); + } else { + values = cache.get(path); + } String value = null; if (values == null || values.isEmpty() || values.get(0) == null) { value = nullValue; diff --git a/src/main/java/de/gwdg/metadataqa/api/configuration/schema/Field.java b/src/main/java/de/gwdg/metadataqa/api/configuration/schema/Field.java index 2f0fc1b5..e15c150b 100644 --- a/src/main/java/de/gwdg/metadataqa/api/configuration/schema/Field.java +++ b/src/main/java/de/gwdg/metadataqa/api/configuration/schema/Field.java @@ -11,6 +11,7 @@ public class Field { private List rules; private String indexField; private boolean identifierField; + private boolean asLanguageTagged; public String getName() { return name; @@ -75,4 +76,12 @@ public boolean isIdentifierField() { public void setIdentifierField(boolean isRecordId) { this.identifierField = isRecordId; } + + public boolean isAsLanguageTagged() { + return asLanguageTagged; + } + + public void setAsLanguageTagged(boolean asLanguageTagged) { + this.asLanguageTagged = asLanguageTagged; + } } diff --git a/src/main/java/de/gwdg/metadataqa/api/json/DataElement.java b/src/main/java/de/gwdg/metadataqa/api/json/DataElement.java index 436b50e9..db331242 100644 --- a/src/main/java/de/gwdg/metadataqa/api/json/DataElement.java +++ b/src/main/java/de/gwdg/metadataqa/api/json/DataElement.java @@ -35,6 +35,7 @@ public class DataElement implements Cloneable, Serializable { private List rules; private Schema schema; private String indexField; + private boolean asLanguageTagged = false; public DataElement(String label, String path, String solrFieldName) { this.label = label; @@ -292,4 +293,13 @@ public DataElement setIndexField(String indexField) { public String generateIndexField() { return label.replaceAll("\\W", "_"); } + + public DataElement setAsLanguageTagged() { + this.asLanguageTagged = true; + return this; + } + + public boolean isAsLanguageTagged() { + return asLanguageTagged; + } } diff --git a/src/main/java/de/gwdg/metadataqa/api/json/JsonUtils.java b/src/main/java/de/gwdg/metadataqa/api/json/JsonUtils.java index e18804d2..e0015c67 100644 --- a/src/main/java/de/gwdg/metadataqa/api/json/JsonUtils.java +++ b/src/main/java/de/gwdg/metadataqa/api/json/JsonUtils.java @@ -9,6 +9,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -82,6 +83,13 @@ public static List extractList(Object value) { public static List extractFieldInstanceList(Object value, String recordId, String jsonPath) { + return extractFieldInstanceList(value, recordId, jsonPath, false); + } + + public static List extractFieldInstanceList(Object value, + String recordId, + String jsonPath, + boolean asLanguageTagged) { List extracted = new ArrayList<>(); if (value.getClass() == String.class) { extracted.add(new EdmFieldInstance((String) value)); @@ -110,9 +118,13 @@ public static List extractFieldInstanceList(Object v } else if (outerVal.getClass() == BigInteger.class) { extracted.add(new EdmFieldInstance(((BigInteger) outerVal).toString())); } else if (outerVal.getClass() == JSONArray.class) { - extracted.addAll(extractInnerArray(outerVal, recordId, jsonPath)); + extracted.addAll(extractInnerArray(outerVal, recordId, jsonPath, asLanguageTagged)); } else if (outerVal.getClass() == LinkedHashMap.class) { - extracted.add(hashToFieldInstance(outerVal, recordId, jsonPath)); + if (asLanguageTagged) { + extracted.addAll(convertLanguageTaggedMap(outerVal, recordId, jsonPath)); + } else { + extracted.add(hashToFieldInstance(outerVal, recordId, jsonPath, asLanguageTagged)); + } } else { LOGGER.severe(String.format( "Unhandled outerArray type: %s, %s [record ID: %s, path: %s]", @@ -121,7 +133,11 @@ public static List extractFieldInstanceList(Object v } } } else if (value.getClass() == LinkedHashMap.class) { - extracted.add(hashToFieldInstance(value, recordId, jsonPath)); + if (asLanguageTagged) { + extracted.addAll(convertLanguageTaggedMap(value, recordId, jsonPath)); + } else { + extracted.add(hashToFieldInstance(value, recordId, jsonPath, asLanguageTagged)); + } } else if (value.getClass() == Integer.class) { extracted.add(new EdmFieldInstance(Integer.toString((int) value))); } else if (value.getClass() == Float.class) { @@ -143,7 +159,8 @@ public static List extractFieldInstanceList(Object v return extracted; } - private static List extractInnerArray(Object outerVal, String recordId, String jsonPath) { + private static List extractInnerArray(Object outerVal, String recordId, String jsonPath, + boolean asLanguageTagged) { List extracted = new ArrayList<>(); JSONArray array = (JSONArray) outerVal; for (int j = 0, l2 = array.size(); j < l2; j++) { @@ -151,9 +168,13 @@ private static List extractInnerArray(Object outerVal, String if (innerVal.getClass() == String.class) { extracted.add(new EdmFieldInstance((String) innerVal)); } else if (innerVal.getClass() == LinkedHashMap.class) { - extracted.add(hashToFieldInstance(innerVal, recordId, jsonPath)); + if (asLanguageTagged) { + extracted.addAll(convertLanguageTaggedMap(innerVal, recordId, jsonPath)); + } else { + extracted.add(hashToFieldInstance(innerVal, recordId, jsonPath, asLanguageTagged)); + } } else if (innerVal.getClass() == JSONArray.class) { - extracted.addAll(extractInnerArray(innerVal, recordId, jsonPath)); + extracted.addAll(extractInnerArray(innerVal, recordId, jsonPath, asLanguageTagged)); } else { LOGGER.severe(String.format( "Unhandled inner array type: %s, [record ID: %s, path: %s]", @@ -164,7 +185,34 @@ private static List extractInnerArray(Object outerVal, String return extracted; } - public static EdmFieldInstance hashToFieldInstance(Object innerVal, String recordId, String jsonPath) { + private static Collection convertLanguageTaggedMap(Object innerVal, + String recordId, + String jsonPath) { + List instances = new ArrayList<>(); + Map map = (LinkedHashMap) innerVal; + for (Map.Entry entry : map.entrySet()) { + String languageTag = entry.getKey(); + if (entry.getValue() instanceof JSONArray) { + JSONArray values = (JSONArray) entry.getValue(); + for (Object value : values) { + if (value instanceof String) { + instances.add(new EdmFieldInstance(value.toString(), languageTag)); + } else { + LOGGER.severe("Other type of entry value: " + entry.getValue().getClass().getCanonicalName() + " " + entry.getValue()); + } + } + } else { + LOGGER.severe("Other type of entry value: " + entry.getValue().getClass().getCanonicalName() + + " " + entry.getValue()); + } + } + return instances; + } + + public static EdmFieldInstance hashToFieldInstance(Object innerVal, + String recordId, + String jsonPath, + boolean asLanguageTagged) { Map map = (LinkedHashMap) innerVal; var instance = new EdmFieldInstance(); for (Map.Entry entry : map.entrySet()) { @@ -197,6 +245,14 @@ public static EdmFieldInstance hashToFieldInstance(Object innerVal, String recor // instance.setValue(map.get("def")); } else if (entry.getKey().equals("@lang")) { instance.setLanguage((String) value); + } else if (asLanguageTagged) { + instance.setLanguage(entry.getKey()); + if (entry.getValue() instanceof JSONArray) { + JSONArray values = (JSONArray) entry.getValue(); + instance.setValue((String) values.get(0)); + } else { + LOGGER.severe("Other type of entry value: " + entry.getValue().getClass().getCanonicalName()); + } } else { LOGGER.severe(String.format( "Other type (%s) of map: %s, [record ID: %s, path: %s]", diff --git a/src/main/java/de/gwdg/metadataqa/api/model/selector/BaseSelector.java b/src/main/java/de/gwdg/metadataqa/api/model/selector/BaseSelector.java index 6d16140b..2574265a 100644 --- a/src/main/java/de/gwdg/metadataqa/api/model/selector/BaseSelector.java +++ b/src/main/java/de/gwdg/metadataqa/api/model/selector/BaseSelector.java @@ -1,5 +1,6 @@ package de.gwdg.metadataqa.api.model.selector; +import de.gwdg.metadataqa.api.json.DataElement; import de.gwdg.metadataqa.api.model.XmlFieldInstance; import java.util.HashMap; @@ -20,6 +21,9 @@ public abstract class BaseSelector implements Select public List get(String path) { return get(path, path, null, null); } + public List get(DataElement dataElement) { + return get(dataElement.getPath(), dataElement.getPath(), null, null); + } public E get(String path, Class clazz) { if (!typedCache.containsKey(path)) { diff --git a/src/main/java/de/gwdg/metadataqa/api/model/selector/JsonSelector.java b/src/main/java/de/gwdg/metadataqa/api/model/selector/JsonSelector.java index 7ccdb7ef..33ae2222 100644 --- a/src/main/java/de/gwdg/metadataqa/api/model/selector/JsonSelector.java +++ b/src/main/java/de/gwdg/metadataqa/api/model/selector/JsonSelector.java @@ -1,5 +1,6 @@ package de.gwdg.metadataqa.api.model.selector; +import de.gwdg.metadataqa.api.json.DataElement; import de.gwdg.metadataqa.api.json.JsonUtils; import de.gwdg.metadataqa.api.model.XmlFieldInstance; import de.gwdg.metadataqa.api.util.ExceptionUtils; @@ -41,13 +42,50 @@ public JsonSelector(Object jsonDocument) { this.document = jsonDocument; } + public List get(DataElement dataElement) { + if (!cache.containsKey(dataElement.getPath())) { + set(dataElement, null, null); + } + return cache.get(dataElement.getPath()); + } + + protected void set(DataElement dataElement, Object jsonFragment, Class clazz) { + List instances = null; + Object value = read(dataElement.getPath(), jsonFragment); + if (value != null) { + if (clazz == null) { + instances = (List) JsonUtils.extractFieldInstanceList( + value, recordId, dataElement.getPath(), dataElement.isAsLanguageTagged() + ); + } else { + if (value instanceof JSONArray) { + typedCache.put(dataElement.getPath(), clazz.cast(((JSONArray) value).get(0))); + } else { + typedCache.put(dataElement.getPath(), value); + } + } + } + cache.put(dataElement.getPath(), instances); + } + + public List get(String path) { + return get(path, path, null, null); + } + + public List get(String address, String path, Object jsonFragment, Class clazz) { + if (!cache.containsKey(address)) { + set(address, path, jsonFragment, clazz); + } + return cache.get(address); + } + @Override protected void set(String address, String path, Object jsonFragment, Class clazz) { List instances = null; Object value = read(path, jsonFragment); if (value != null) { if (clazz == null) { - instances = (List) JsonUtils.extractFieldInstanceList(value, recordId, path); + instances = (List) JsonUtils.extractFieldInstanceList(value, recordId, path, false); } else { if (value instanceof JSONArray) { typedCache.put(address, clazz.cast(((JSONArray) value).get(0))); diff --git a/src/main/java/de/gwdg/metadataqa/api/model/selector/Selector.java b/src/main/java/de/gwdg/metadataqa/api/model/selector/Selector.java index 72375ef8..ed926f49 100644 --- a/src/main/java/de/gwdg/metadataqa/api/model/selector/Selector.java +++ b/src/main/java/de/gwdg/metadataqa/api/model/selector/Selector.java @@ -1,5 +1,6 @@ package de.gwdg.metadataqa.api.model.selector; +import de.gwdg.metadataqa.api.json.DataElement; import de.gwdg.metadataqa.api.model.XmlFieldInstance; import java.io.Serializable; @@ -11,6 +12,7 @@ public interface Selector extends Serializable { Object read(String path, Object jsonFragment); List get(String path); + List get(DataElement dataElement); // E get(String path, Class clazz); List get(String address, String path, Object jsonFragment); List get(String address, String path, Object jsonFragment, Class clazz); diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/logical/AndChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/logical/AndChecker.java index 26b4af9c..be546b01 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/logical/AndChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/logical/AndChecker.java @@ -37,7 +37,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = false; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { FieldCounter localResults = new FieldCounter<>(); for (RuleChecker checker : checkers) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/logical/NotChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/logical/NotChecker.java index ac4a5262..e663eb15 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/logical/NotChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/logical/NotChecker.java @@ -37,7 +37,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = false; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { FieldCounter localResults = new FieldCounter<>(); for (RuleChecker checker : checkers) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/logical/OrChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/logical/OrChecker.java index 3c1ec362..59a420e9 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/logical/OrChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/logical/OrChecker.java @@ -40,7 +40,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = false; var isNA = false; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { FieldCounter localResults = new FieldCounter<>(); for (RuleChecker checker : checkers) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ContentTypeChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ContentTypeChecker.java index 650b6dc2..a60d8350 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ContentTypeChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ContentTypeChecker.java @@ -38,7 +38,7 @@ public void update(Selector cache, FieldCounter results, Rule var isNA = true; int instanceCount = 0; int failureCount = 0; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/DependencyChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/DependencyChecker.java index 759ae2b0..9dfebd2b 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/DependencyChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/DependencyChecker.java @@ -52,7 +52,7 @@ public void update(Selector cache, FieldCounter localResults, var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/EnumerationChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/EnumerationChecker.java index 050c7e69..8058d7b0 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/EnumerationChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/EnumerationChecker.java @@ -32,7 +32,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/HasValueChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/HasValueChecker.java index 3458ec37..720cef4b 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/HasValueChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/HasValueChecker.java @@ -36,7 +36,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = false; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionChecker.java index fbeb8087..f16a1c6d 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionChecker.java @@ -38,7 +38,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxCountChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxCountChecker.java index 678fb257..772bf728 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxCountChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxCountChecker.java @@ -24,7 +24,6 @@ public MaxCountChecker(DataElement field, int maxCount, boolean allowEmptyInstan this.allowEmptyInstances = allowEmptyInstances; } - public MaxCountChecker(DataElement field, String header, int maxCount) { super(field, header + ":" + PREFIX); this.maxCount = maxCount; diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxLengthChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxLengthChecker.java index 2e50802e..9fdb7289 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxLengthChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxLengthChecker.java @@ -32,7 +32,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxWordsChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxWordsChecker.java index a25197c3..9ba6bbe7 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxWordsChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MaxWordsChecker.java @@ -32,7 +32,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinLengthChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinLengthChecker.java index 2f6bd082..e9141c64 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinLengthChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinLengthChecker.java @@ -32,7 +32,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinWordsChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinWordsChecker.java index 0762a898..e4927d02 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinWordsChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/MinWordsChecker.java @@ -33,7 +33,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/NumericValueChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/NumericValueChecker.java index 64be02ee..398fbfce 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/NumericValueChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/NumericValueChecker.java @@ -47,7 +47,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/PatternChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/PatternChecker.java index 260671af..e6bdcadc 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/PatternChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/PatternChecker.java @@ -33,7 +33,7 @@ public void update(Selector cache, FieldCounter results, Rule var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/UniquenessChecker.java b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/UniquenessChecker.java index cd83d05f..5b178242 100644 --- a/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/UniquenessChecker.java +++ b/src/main/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/UniquenessChecker.java @@ -34,7 +34,7 @@ public void update(Selector cache, FieldCounter results, Rule LOGGER.info(this.getClass() + " " + this.id); var allPassed = true; var isNA = true; - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) { for (XmlFieldInstance instance : instances) { if (instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/util/ImageDimensionExtractor.java b/src/main/java/de/gwdg/metadataqa/api/util/ImageDimensionExtractor.java index 2c2c69b1..22d70940 100644 --- a/src/main/java/de/gwdg/metadataqa/api/util/ImageDimensionExtractor.java +++ b/src/main/java/de/gwdg/metadataqa/api/util/ImageDimensionExtractor.java @@ -14,7 +14,7 @@ public static DimensionDao extractRemote(String url) throws IOException { HttpURLConnection urlConnection = (HttpURLConnection) urlObj.openConnection(); - int timeout = 1000; + int timeout = 2000; urlConnection.setConnectTimeout(timeout); urlConnection.setReadTimeout(timeout); urlConnection.connect(); diff --git a/src/main/java/de/gwdg/metadataqa/api/util/InstanceCounter.java b/src/main/java/de/gwdg/metadataqa/api/util/InstanceCounter.java index 55f91963..50ecbdc0 100644 --- a/src/main/java/de/gwdg/metadataqa/api/util/InstanceCounter.java +++ b/src/main/java/de/gwdg/metadataqa/api/util/InstanceCounter.java @@ -28,7 +28,7 @@ public InstanceCounter(Selector cache, DataElement field, boolean allowEmptyInst } private void count() { - List instances = cache.get(field.getPath()); + List instances = cache.get(field); if (instances != null && !instances.isEmpty()) for (XmlFieldInstance instance : instances) if (allowEmptyInstances || instance.hasValue()) { diff --git a/src/main/java/de/gwdg/metadataqa/api/util/SchemaFactory.java b/src/main/java/de/gwdg/metadataqa/api/util/SchemaFactory.java index 51bfafc4..2e7af5df 100644 --- a/src/main/java/de/gwdg/metadataqa/api/util/SchemaFactory.java +++ b/src/main/java/de/gwdg/metadataqa/api/util/SchemaFactory.java @@ -29,10 +29,10 @@ public static Schema fromConfig(SchemaConfiguration config) { schema.setCategories(config.getCategories()); for (Field field : config.getFields()) { - var branch = new DataElement(field.getName()); + var dataElement = new DataElement(field.getName()); if (StringUtils.isNotBlank(field.getPath())) - branch.setPath(field.getPath()); + dataElement.setPath(field.getPath()); if (field.getCategories() != null) { List categories = new ArrayList<>(); @@ -47,25 +47,28 @@ public static Schema fromConfig(SchemaConfiguration config) { categories.add(category); } } - branch.setCategories(categories); + dataElement.setCategories(categories); } if (field.isExtractable()) - branch.setExtractable(); + dataElement.setExtractable(); if (field.isInactive()) - branch.setActive(false); + dataElement.setActive(false); if (field.getRules() != null) - branch.setRule(field.getRules()); + dataElement.setRule(field.getRules()); if (StringUtils.isNotBlank(field.getIndexField())) - branch.setIndexField(field.getIndexField()); + dataElement.setIndexField(field.getIndexField()); - schema.addField(branch); + if (field.isAsLanguageTagged()) + dataElement.setAsLanguageTagged(); + + schema.addField(dataElement); if (field.isIdentifierField()) - schema.setRecordId(branch); + schema.setRecordId(dataElement); } if (config.getGroups() != null) diff --git a/src/test/java/de/gwdg/metadataqa/api/calculator/FieldExtractorTest.java b/src/test/java/de/gwdg/metadataqa/api/calculator/FieldExtractorTest.java index cc157963..9ad4619d 100644 --- a/src/test/java/de/gwdg/metadataqa/api/calculator/FieldExtractorTest.java +++ b/src/test/java/de/gwdg/metadataqa/api/calculator/FieldExtractorTest.java @@ -37,7 +37,8 @@ public class FieldExtractorTest { @Before public void setUp() throws URISyntaxException, IOException { calculator = new FieldExtractor("$.identifier"); - cache = new JsonSelector(FileUtils.readFirstLineFromResource("general/test.json")); + String content = FileUtils.readFirstLineFromResource("general/test.json"); + cache = new JsonSelector(content); } @Test diff --git a/src/test/java/de/gwdg/metadataqa/api/configuration/SchemaConfigurationTest.java b/src/test/java/de/gwdg/metadataqa/api/configuration/SchemaConfigurationTest.java index 21a2e99c..7da32db0 100644 --- a/src/test/java/de/gwdg/metadataqa/api/configuration/SchemaConfigurationTest.java +++ b/src/test/java/de/gwdg/metadataqa/api/configuration/SchemaConfigurationTest.java @@ -348,4 +348,10 @@ public void yaml_contentType() throws FileNotFoundException { assertEquals(List.of("image/jpeg", "image/png", "image/tiff", "image/tiff-fx", "image/gif", "image/svg+xml", "application/pdf"), contentTypeRule.getContentType()); } + + @Test + public void yaml_ssLanguageTagged() throws FileNotFoundException { + Schema schema = ConfigurationReader.readSchemaYaml("src/test/resources/configuration/schema/asLanguageTagged.yaml").asSchema(); + assertEquals(true, schema.getPathByLabel("description").isAsLanguageTagged()); + } } diff --git a/src/test/java/de/gwdg/metadataqa/api/io/reader/JSONArrayRecordReaderTest.java b/src/test/java/de/gwdg/metadataqa/api/io/reader/JSONArrayRecordReaderTest.java new file mode 100644 index 00000000..690d6b0a --- /dev/null +++ b/src/test/java/de/gwdg/metadataqa/api/io/reader/JSONArrayRecordReaderTest.java @@ -0,0 +1,89 @@ +package de.gwdg.metadataqa.api.io.reader; + +import de.gwdg.metadataqa.api.calculator.CalculatorFacade; +import de.gwdg.metadataqa.api.configuration.MeasurementConfiguration; +import de.gwdg.metadataqa.api.interfaces.MetricResult; +import de.gwdg.metadataqa.api.io.IOTestBase; +import de.gwdg.metadataqa.api.json.DataElement; +import de.gwdg.metadataqa.api.schema.BaseSchema; +import de.gwdg.metadataqa.api.schema.Format; +import de.gwdg.metadataqa.api.schema.Schema; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class JSONArrayRecordReaderTest extends IOTestBase { + + String inputFile = "src/test/resources/json/edm.json"; + BufferedReader inputReader; + RecordReader reader; + + @Before + public void setUp() throws Exception { + inputReader = Files.newBufferedReader(Paths.get(inputFile)); + } + + @Test + public void testHasNext() throws IOException { + reader = new JSONArrayRecordReader(inputReader, getCalculator(Format.JSON)); + assertTrue(reader.hasNext()); + } + + @Test + public void testNext_dcTitle() throws IOException { + reader = new JSONArrayRecordReader(inputReader, getCalculator(Format.JSON, "dcTitle")); + + Map> result = reader.next(); + assertNotNull(result); + assertTrue(result instanceof LinkedHashMap); + assertEquals(1, result.size()); + } + + @Test + public void testNext_dcDate() throws IOException { + reader = new JSONArrayRecordReader(inputReader, getCalculator(Format.JSON, "dcDate")); + + Map> result = reader.next(); + assertNotNull(result); + assertTrue(result instanceof LinkedHashMap); + assertEquals(1, result.size()); + } + + protected CalculatorFacade getCalculator(Format format, String field) { + MeasurementConfiguration config = new MeasurementConfiguration() + .enableFieldExtractor() + .disableCompletenessMeasurement(); + + CalculatorFacade calculator = new CalculatorFacade(config).setSchema(getSchema(format, field)); + + return calculator; + } + + protected Schema getSchema(Format format, String field) { + System.err.println(getPath(field)); + return new BaseSchema() + .setFormat(format) + .addField( + new DataElement(getPath(field)) + .setExtractable() + .setAsLanguageTagged() + ) + // .addField(new DataElement("name").setExtractable()) + ; + } + + private String getPath(String field) { + return String.format("$.['object']['proxies'][?(@['europeanaProxy'] == false)]['%s']", field); + } +} \ No newline at end of file diff --git a/src/test/java/de/gwdg/metadataqa/api/io/reader/JSONRecordReaderTest.java b/src/test/java/de/gwdg/metadataqa/api/io/reader/JSONRecordReaderTest.java index a62c5f29..2feae317 100644 --- a/src/test/java/de/gwdg/metadataqa/api/io/reader/JSONRecordReaderTest.java +++ b/src/test/java/de/gwdg/metadataqa/api/io/reader/JSONRecordReaderTest.java @@ -42,4 +42,6 @@ public void next() { assertEquals("https://neurovault.org/images/384958/", fields.get(0).getResultMap().get("url")); assertEquals("massivea uditory lexical decision", fields.get(0).getResultMap().get("name")); } + + // path: $.['object']['proxies'][?(@['europeanaProxy'] == false)]['dcTitle'][*] } \ No newline at end of file diff --git a/src/test/java/de/gwdg/metadataqa/api/json/JsonUtilsTest.java b/src/test/java/de/gwdg/metadataqa/api/json/JsonUtilsTest.java index a3b870ce..3e688d23 100644 --- a/src/test/java/de/gwdg/metadataqa/api/json/JsonUtilsTest.java +++ b/src/test/java/de/gwdg/metadataqa/api/json/JsonUtilsTest.java @@ -299,10 +299,25 @@ public void testExtractFieldInstanceList() { } + @Test + public void testExtractFieldInstanceMap() { + List expected = Arrays.asList( + new EdmFieldInstance("2023-05-12", "def"), + new EdmFieldInstance("07:30", "def") + ); + List list = extractFieldInstanceListTagged( + "{\"def\": [\"2023-05-12\",\"07:30\"]}", "dcDate"); + assertEquals(expected, list); + } + private List extractFieldInstanceList(String json, String path) { return JsonUtils.extractFieldInstanceList(PARSER.parse(json), "1", path); } + private List extractFieldInstanceListTagged(String json, String path) { + return JsonUtils.extractFieldInstanceList(PARSER.parse(json), "1", path, true); + } + private List createFieldList(String... args) { List list = new ArrayList<>(); for (String var : args) diff --git a/src/test/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionCheckerTest.java b/src/test/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionCheckerTest.java index 72867ba3..10de7ca2 100644 --- a/src/test/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionCheckerTest.java +++ b/src/test/java/de/gwdg/metadataqa/api/rule/singlefieldchecker/ImageDimensionCheckerTest.java @@ -58,7 +58,9 @@ public void successMinShortLong() { "\"https://iiif.deutsche-digitale-bibliothek.de/image/2/ec863e48-7e20-4e9c-95fd-babd708b6eaf/full/full/0/default.jpg\""); cache.setCsvReader(new CsvReader().setHeader( ((CsvAwareSchema) schema).getHeader() )); - ImageDimensionChecker checker = new ImageDimensionChecker(schema.getPathByLabel("name"), new Dimension().withMinShortside(200).withMinLongside(200)); + ImageDimensionChecker checker = new ImageDimensionChecker( + schema.getPathByLabel("name"), + new Dimension().withMinShortside(200).withMinLongside(200)); FieldCounter fieldCounter = new FieldCounter<>(); checker.update(cache, fieldCounter, RuleCheckingOutputType.BOTH); diff --git a/src/test/resources/configuration/schema/asLanguageTagged.yaml b/src/test/resources/configuration/schema/asLanguageTagged.yaml new file mode 100644 index 00000000..f84db69b --- /dev/null +++ b/src/test/resources/configuration/schema/asLanguageTagged.yaml @@ -0,0 +1,9 @@ +format: json +fields: + - name: about + path: $.['about'] + rules: + - lessThanOrEquals: description + - name: description + path: $.['description'] + asLanguageTagged: true diff --git a/src/test/resources/json/edm.json b/src/test/resources/json/edm.json new file mode 100644 index 00000000..545bb590 --- /dev/null +++ b/src/test/resources/json/edm.json @@ -0,0 +1,1151 @@ +[ + { + "success": true, + "statsDuration": 202, + "requestNumber": 999, + "object": { + "about": "/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "aggregations": [ + { + "about": "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "edmDataProvider": { + "def": [ + "http://data.europeana.eu/organization/1482250000000370517" + ] + }, + "edmIsShownBy": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/XC808089-230512_0100a.mp3", + "edmIsShownAt": "https://data.biodiversitydata.nl/xeno-canto/observation/XC808089", + "edmObject": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/wave/XC808089-large.png", + "edmProvider": { + "def": [ + "http://data.europeana.eu/organization/1482250000004671150" + ] + }, + "edmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "edmUgc": "false", + "hasView": [ + "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/ffts/XC808089-large.png", + "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/wave/XC808089-large.png" + ], + "aggregatedCHO": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "webResources": [ + { + "about": "https://data.biodiversitydata.nl/xeno-canto/observation/XC808089", + "dcFormat": { + "en": [ + "text/html" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808089. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
" + }, + { + "webResourceEdmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "about": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/ffts/XC808089-large.png", + "dcFormat": { + "en": [ + "image/png" + ] + }, + "dcCreator": { + "def": [ + "Stichting Xeno-canto voor Natuurgeluiden" + ] + }, + "dcType": { + "def": [ + "StillImage" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089. Stichting Xeno-canto voor Natuurgeluiden. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808089. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Creator
Stichting Xeno-canto voor Natuurgeluiden
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
", + "dctermsIsReferencedBy": [ + "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089/manifest" + ], + "ebucoreHasMimeType": "image/png", + "ebucoreFileByteSize": 203697, + "ebucoreWidth": 1022, + "ebucoreHeight": 396, + "edmHasColorSpace": "sRGB", + "edmComponentColor": [ + "#696969", + "#808080", + "#F5F5F5", + "#FAF0E6", + "#BC8F8F", + "#FFFAFA" + ], + "ebucoreOrientation": "landscape" + }, + { + "webResourceEdmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "about": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/wave/XC808089-large.png", + "dcFormat": { + "en": [ + "image/png" + ] + }, + "dcCreator": { + "def": [ + "Stichting Xeno-canto voor Natuurgeluiden" + ] + }, + "dcType": { + "def": [ + "StillImage" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089. Stichting Xeno-canto voor Natuurgeluiden. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808089. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Creator
Stichting Xeno-canto voor Natuurgeluiden
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
", + "dctermsIsReferencedBy": [ + "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089/manifest" + ], + "ebucoreHasMimeType": "image/png", + "ebucoreFileByteSize": 3165, + "ebucoreWidth": 960, + "ebucoreHeight": 320, + "edmHasColorSpace": "sRGB", + "edmComponentColor": [ + "#FFFAFA", + "#E6E6FA", + "#C0C0C0", + "#FAF0E6", + "#FFFFFF", + "#000000" + ], + "ebucoreOrientation": "landscape" + }, + { + "webResourceEdmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "about": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/XC808089-230512_0100a.mp3", + "dcDescription": { + "en": [ + "4760 s" + ] + }, + "dcFormat": { + "en": [ + "audio/mp3" + ] + }, + "dcCreator": { + "def": [ + "Cedric Mroczko" + ] + }, + "dcType": { + "def": [ + "Sound" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089. Cedric Mroczko. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808089. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Creator
Cedric Mroczko
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
", + "dctermsIsReferencedBy": [ + "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089/manifest" + ], + "edmCodecName": "mp3", + "ebucoreHasMimeType": "audio/mpeg", + "ebucoreFileByteSize": 114246100, + "ebucoreAudioChannelNumber": 2, + "ebucoreDuration": "4760033", + "ebucoreSampleSize": 0, + "ebucoreSampleRate": 44100, + "ebucoreBitRate": 192000 + }, + { + "about": "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089/manifest", + "rdfType": "http://iiif.io/api/presentation/3#Manifest" + } + ], + "dqvHasQualityAnnotation": [ + { + "created": "2024-01-18T13:34:48.847776Z", + "target": [ + "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + ], + "body": "http://www.europeana.eu/schemas/epf/contentTier3" + }, + { + "created": "2024-01-18T13:34:48.847787Z", + "target": [ + "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + ], + "body": "http://www.europeana.eu/schemas/epf/metadataTierA" + } + ] + } + ], + "edmDatasetName": [ + "999__Naturalis_Wildlife_Sounds_batch3" + ], + "europeanaAggregation": { + "about": "/aggregation/europeana/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "aggregatedCHO": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "edmCountry": { + "def": [ + "Netherlands" + ] + }, + "edmLanguage": { + "def": [ + "en" + ] + }, + "edmPreview": "https://api.europeana.eu/thumbnail/v2/url.json?uri=https%3A%2F%2Fxeno-canto.org%2Fsounds%2Fuploaded%2FTJSRKPHQNP%2Fwave%2FXC808089-large.png&type=SOUND", + "edmLandingPage": "https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + }, + "europeanaCollectionName": [ + "999__Naturalis_Wildlife_Sounds_batch3" + ], + "europeanaCompleteness": 6, + "organizations": [ + { + "about": "http://data.europeana.eu/organization/1482250000000370517", + "prefLabel": { + "en": [ + "Naturalis Biodiversity Center" + ] + } + }, + { + "about": "http://data.europeana.eu/organization/1482250000004671150", + "prefLabel": { + "en": [ + "OpenUp!" + ] + } + } + ], + "places": [ + { + "about": "geo:48.4696,23.7651", + "prefLabel": { + "def": [ + "Kolochava, Mizhhirs'kyi district, Zakarpattia Oblast" + ] + }, + "note": { + "def": [ + "Ukraine" + ] + }, + "isPartOf": { + "def": [ + "https://sws.geonames.org/690791/" + ] + }, + "latitude": 48.4696, + "longitude": 23.7651 + } + ], + "providedCHOs": [ + { + "about": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + } + ], + "proxies": [ + { + "about": "/proxy/europeana/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "dcDate": { + "def": [ + "#2023-05-12" + ] + }, + "dcIdentifier": { + "def": [ + "WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + ] + }, + "proxyIn": [ + "/aggregation/europeana/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + ], + "proxyFor": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "lineage": [ + "/proxy/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + ], + "europeanaProxy": true + }, + { + "about": "/proxy/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "dcContributor": { + "def": [ + "Cedric Mroczko (collector)" + ] + }, + "dcDate": { + "def": [ + "2023-05-12", + "07:30" + ] + }, + "dcDescription": { + "def": [ + "High-pass filtered. ls14 + stereo clippy em272" + ] + }, + "dcIdentifier": { + "def": [ + "808089@XC", + "https://data.biodiversitydata.nl/xeno-canto/observation/XC808089", + "XC808089" + ] + }, + "dcRelation": { + "def": [ + "http://www.biodiversitylibrary.org/name/Sonus+naturalis" + ] + }, + "dcRights": { + "def": [ + "Stichting Xeno-canto voor Natuurgeluiden", + "Cedric Mroczko" + ] + }, + "dcSubject": { + "en": [ + "Wildlife sounds - Soundscape" + ] + }, + "dcTitle": { + "la": [ + "Sonus naturalis" + ] + }, + "dcType": { + "en": [ + "Machine Observation" + ] + }, + "dctermsAlternative": { + "def": [ + "Soundscape" + ] + }, + "dctermsSpatial": { + "def": [ + "1300 m", + "geo:48.4696,23.7651" + ] + }, + "edmHasMet": { + "def": [ + "808089XXCX2023X05X12", + "geo:48.4696,23.7651" + ] + }, + "edmHasType": { + "def": [ + "http://rs.tdwg.org/dwc/terms/MachineObservation" + ] + }, + "proxyIn": [ + "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089" + ], + "proxyFor": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808089", + "edmType": "SOUND", + "europeanaProxy": false + } + ], + "timespans": [ + { + "about": "808089XXCX2023X05X12", + "begin": { + "def": [ + "2023-05-12" + ] + } + }, + { + "about": "http://data.europeana.eu/timespan/21", + "prefLabel": { + "de": [ + "21. Jahrhundert" + ], + "ru": [ + "XXI век" + ], + "fi": [ + "2000-luku" + ], + "pt": [ + "Século XXI" + ], + "bg": [ + "21 век" + ], + "lt": [ + "XXI amžius" + ], + "lv": [ + "21. gadsimts" + ], + "hr": [ + "21. stoljeće" + ], + "fr": [ + "XXIe siècle" + ], + "hu": [ + "21. század" + ], + "sk": [ + "21. storočie" + ], + "sl": [ + "21. stoletje" + ], + "ga": [ + "21ú haois" + ], + "ca": [ + "Segle XXI" + ], + "sv": [ + "2000-talet" + ], + "el": [ + "21ος αιώνας" + ], + "en": [ + "21st century" + ], + "it": [ + "XXI secolo" + ], + "es": [ + "Siglo XXI" + ], + "et": [ + "21. sajand" + ], + "eu": [ + "XXI. mendea" + ], + "cs": [ + "21. století" + ], + "pl": [ + "XXI wiek" + ], + "ro": [ + "Secolul al XXI-lea" + ], + "da": [ + "21. århundrede" + ], + "nl": [ + "21e eeuw" + ] + }, + "altLabel": { + "de": [ + "21. Jh.", + "21. Jhd.", + "21. Jhdt.", + "21. Jhrh.", + "21. Jahrh.", + "21. Zentennium", + "21. Centennium", + "21. Hektode" + ], + "sv": [ + "21:a århundradet", + "21:a seklet", + "2000-tal", + "2000-talet (århundrade)", + "2000-talet (sekel)" + ], + "fi": [ + "21. vuosisata" + ], + "ru": [ + "21 век" + ], + "en": [ + "21st-century", + "21st-century", + "Twenty-first century", + "History, 21st Century", + "XXI Century", + "Current century" + ], + "it": [ + "21° secolo", + "Secolo XXI", + "Ventunesimo secolo", + "XXI Secolo", + "Duemila" + ], + "pl": [ + "21 wiek", + "21 stulecie", + "XXI stulecie", + "XXI w" + ], + "fr": [ + "21e siècle", + "Vingt et unième siècle" + ], + "es": [ + "Historia, siglo 21", + "Siglo 21", + "Este siglo", + "Siglo veintiuno" + ], + "ca": [ + "Segle 21" + ] + }, + "begin": { + "def": [ + "2001-01-01" + ] + }, + "end": { + "def": [ + "2100-12-31" + ] + }, + "owlSameAs": [ + "http://www.wikidata.org/entity/Q6939", + "http://data.culture.fr/thesaurus/resource/ark:/67717/T4-46", + "http://id.loc.gov/authorities/names/sh85139024", + "http://data.bnf.fr/ark:/12148/cb11930812p", + "http://id.nlm.nih.gov/mesh/D049674", + "https://www.freebase.com/m/08b43", + "https://g.co/kg/m/08b43", + "http://id.nlm.nih.gov/mesh/K01.400.504.984", + "http://vocab.getty.edu/aat/300404515", + "http://id.worldcat.org/fast/1159816", + "http://babelnet.org/rdf/s01157225n", + "http://dbpedia.org/resource/21st_century" + ] + }, + { + "about": "#2023-05-12", + "prefLabel": { + "zxx": [ + "2023-05-12" + ] + }, + "begin": { + "def": [ + "2023-05-01" + ] + }, + "end": { + "def": [ + "2023-05-31" + ] + }, + "isPartOf": { + "def": [ + "http://data.europeana.eu/timespan/21" + ] + }, + "skosNotation": { + "def": [ + "2023-05-12" + ] + } + } + ], + "timestamp_created": "2024-01-18T11:16:27.079Z", + "timestamp_created_epoch": 1705576587079, + "timestamp_update": "2024-01-18T11:16:27.079Z", + "timestamp_update_epoch": 1705576587079, + "type": "SOUND" + } + }, + { + "success": true, + "statsDuration": 197, + "requestNumber": 999, + "object": { + "about": "/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "aggregations": [ + { + "about": "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "edmDataProvider": { + "def": [ + "http://data.europeana.eu/organization/1482250000000370517" + ] + }, + "edmIsShownBy": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/XC808057-0608_045233b.mp3", + "edmIsShownAt": "https://data.biodiversitydata.nl/xeno-canto/observation/XC808057", + "edmObject": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/wave/XC808057-large.png", + "edmProvider": { + "def": [ + "http://data.europeana.eu/organization/1482250000004671150" + ] + }, + "edmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "edmUgc": "false", + "hasView": [ + "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/ffts/XC808057-large.png", + "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/wave/XC808057-large.png" + ], + "aggregatedCHO": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "webResources": [ + { + "webResourceEdmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "about": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/wave/XC808057-large.png", + "dcFormat": { + "en": [ + "image/png" + ] + }, + "dcCreator": { + "def": [ + "Stichting Xeno-canto voor Natuurgeluiden" + ] + }, + "dcType": { + "def": [ + "StillImage" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057. Stichting Xeno-canto voor Natuurgeluiden. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808057. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Creator
Stichting Xeno-canto voor Natuurgeluiden
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
", + "dctermsIsReferencedBy": [ + "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057/manifest" + ], + "ebucoreHasMimeType": "image/png", + "ebucoreFileByteSize": 3161, + "ebucoreWidth": 960, + "ebucoreHeight": 320, + "edmHasColorSpace": "sRGB", + "edmComponentColor": [ + "#2F4F4F", + "#F5F5F5", + "#C0C0C0", + "#E6E6FA", + "#FAF0E6", + "#FFFFFF" + ], + "ebucoreOrientation": "landscape" + }, + { + "webResourceEdmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "about": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/ffts/XC808057-large.png", + "dcFormat": { + "en": [ + "image/png" + ] + }, + "dcCreator": { + "def": [ + "Stichting Xeno-canto voor Natuurgeluiden" + ] + }, + "dcType": { + "def": [ + "StillImage" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057. Stichting Xeno-canto voor Natuurgeluiden. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808057. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Creator
Stichting Xeno-canto voor Natuurgeluiden
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
", + "dctermsIsReferencedBy": [ + "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057/manifest" + ], + "ebucoreHasMimeType": "image/png", + "ebucoreFileByteSize": 174416, + "ebucoreWidth": 1022, + "ebucoreHeight": 396, + "edmHasColorSpace": "sRGB", + "edmComponentColor": [ + "#F8F8FF", + "#A9A9A9", + "#FFFAFA", + "#F5F5F5", + "#FFE4E1", + "#556B2F" + ], + "ebucoreOrientation": "landscape" + }, + { + "webResourceEdmRights": { + "def": [ + "http://creativecommons.org/licenses/by-nc-sa/4.0/" + ] + }, + "about": "https://xeno-canto.org/sounds/uploaded/TJSRKPHQNP/XC808057-0608_045233b.mp3", + "dcDescription": { + "en": [ + "1561 s" + ] + }, + "dcFormat": { + "en": [ + "audio/mp3" + ] + }, + "dcCreator": { + "def": [ + "Cedric Mroczko" + ] + }, + "dcType": { + "def": [ + "Sound" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057. Cedric Mroczko. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808057. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Creator
Cedric Mroczko
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
", + "dctermsIsReferencedBy": [ + "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057/manifest" + ], + "edmCodecName": "mp3", + "ebucoreHasMimeType": "audio/mpeg", + "ebucoreFileByteSize": 62447767, + "ebucoreAudioChannelNumber": 2, + "ebucoreDuration": "1561051", + "ebucoreSampleSize": 0, + "ebucoreSampleRate": 44100, + "ebucoreBitRate": 320000 + }, + { + "about": "https://data.biodiversitydata.nl/xeno-canto/observation/XC808057", + "dcFormat": { + "en": [ + "text/html" + ] + }, + "textAttributionSnippet": "(la) Sonus naturalis - https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057. Naturalis Biodiversity Center - https://data.biodiversitydata.nl/xeno-canto/observation/XC808057. CC BY-NC-SA - http://creativecommons.org/licenses/by-nc-sa/4.0/", + "htmlAttributionSnippet": "
Title
Sonus naturalis
Institution
Naturalis Biodiversity Center
Country
Netherlands
Rights
CC BY-NC-SA
" + }, + { + "about": "https://iiif.europeana.eu/presentation/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057/manifest", + "rdfType": "http://iiif.io/api/presentation/3#Manifest" + } + ], + "dqvHasQualityAnnotation": [ + { + "created": "2024-01-18T15:37:47.630360Z", + "target": [ + "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + ], + "body": "http://www.europeana.eu/schemas/epf/contentTier3" + }, + { + "created": "2024-01-18T15:37:47.630373Z", + "target": [ + "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + ], + "body": "http://www.europeana.eu/schemas/epf/metadataTierA" + } + ] + } + ], + "edmDatasetName": [ + "999__Naturalis_Wildlife_Sounds_batch3" + ], + "europeanaAggregation": { + "about": "/aggregation/europeana/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "aggregatedCHO": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "edmCountry": { + "def": [ + "Netherlands" + ] + }, + "edmLanguage": { + "def": [ + "en" + ] + }, + "edmPreview": "https://api.europeana.eu/thumbnail/v2/url.json?uri=https%3A%2F%2Fxeno-canto.org%2Fsounds%2Fuploaded%2FTJSRKPHQNP%2Fwave%2FXC808057-large.png&type=SOUND", + "edmLandingPage": "https://www.europeana.eu/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + }, + "europeanaCollectionName": [ + "999__Naturalis_Wildlife_Sounds_batch3" + ], + "europeanaCompleteness": 7, + "organizations": [ + { + "about": "http://data.europeana.eu/organization/1482250000000370517", + "prefLabel": { + "en": [ + "Naturalis Biodiversity Center" + ] + } + }, + { + "about": "http://data.europeana.eu/organization/1482250000004671150", + "prefLabel": { + "en": [ + "OpenUp!" + ] + } + } + ], + "places": [ + { + "about": "geo:42.9178,2.2289", + "prefLabel": { + "def": [ + "Espéraza, Aude, Occitanie" + ] + }, + "note": { + "def": [ + "France" + ] + }, + "isPartOf": { + "def": [ + "https://sws.geonames.org/3017382/" + ] + }, + "latitude": 42.9178, + "longitude": 2.2289 + } + ], + "providedCHOs": [ + { + "about": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + } + ], + "proxies": [ + { + "about": "/proxy/europeana/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "dcDate": { + "def": [ + "#2023-06-08" + ] + }, + "dcIdentifier": { + "def": [ + "WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + ] + }, + "proxyIn": [ + "/aggregation/europeana/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + ], + "proxyFor": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "lineage": [ + "/proxy/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + ], + "europeanaProxy": true + }, + { + "about": "/proxy/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "dcContributor": { + "def": [ + "Cedric Mroczko (collector)" + ] + }, + "dcDate": { + "def": [ + "2023-06-08", + "05:30" + ] + }, + "dcDescription": { + "def": [ + "field recording", + "Dawn chorus. r05 +stereo clippy em272; animal seen:no; playback used:no" + ] + }, + "dcIdentifier": { + "def": [ + "808057@XC", + "https://data.biodiversitydata.nl/xeno-canto/observation/XC808057", + "XC808057" + ] + }, + "dcRelation": { + "def": [ + "http://www.biodiversitylibrary.org/name/Sonus+naturalis" + ] + }, + "dcRights": { + "def": [ + "Stichting Xeno-canto voor Natuurgeluiden", + "Cedric Mroczko" + ] + }, + "dcSubject": { + "en": [ + "Wildlife sounds - Soundscape" + ] + }, + "dcTitle": { + "la": [ + "Sonus naturalis" + ] + }, + "dcType": { + "en": [ + "Machine Observation" + ] + }, + "dctermsAlternative": { + "def": [ + "Soundscape" + ] + }, + "dctermsSpatial": { + "def": [ + "360 m", + "geo:42.9178,2.2289" + ] + }, + "edmHasMet": { + "def": [ + "808057XXCX2023X06X08", + "geo:42.9178,2.2289" + ] + }, + "edmHasType": { + "def": [ + "http://rs.tdwg.org/dwc/terms/MachineObservation" + ] + }, + "proxyIn": [ + "/aggregation/provider/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057" + ], + "proxyFor": "/item/999/WILDLIFEXSOUNDSXXXSOUNDSCAPEXXC808057", + "edmType": "SOUND", + "europeanaProxy": false + } + ], + "timespans": [ + { + "about": "#2023-06-08", + "prefLabel": { + "zxx": [ + "2023-06-08" + ] + }, + "begin": { + "def": [ + "2023-06-08" + ] + }, + "end": { + "def": [ + "2023-06-08" + ] + }, + "isPartOf": { + "def": [ + "http://data.europeana.eu/timespan/21" + ] + }, + "skosNotation": { + "def": [ + "2023-06-08" + ] + } + }, + { + "about": "http://data.europeana.eu/timespan/21", + "prefLabel": { + "de": [ + "21. Jahrhundert" + ], + "ru": [ + "XXI век" + ], + "fi": [ + "2000-luku" + ], + "pt": [ + "Século XXI" + ], + "bg": [ + "21 век" + ], + "lt": [ + "XXI amžius" + ], + "lv": [ + "21. gadsimts" + ], + "hr": [ + "21. stoljeće" + ], + "fr": [ + "XXIe siècle" + ], + "hu": [ + "21. század" + ], + "sk": [ + "21. storočie" + ], + "sl": [ + "21. stoletje" + ], + "ga": [ + "21ú haois" + ], + "ca": [ + "Segle XXI" + ], + "sv": [ + "2000-talet" + ], + "el": [ + "21ος αιώνας" + ], + "en": [ + "21st century" + ], + "it": [ + "XXI secolo" + ], + "es": [ + "Siglo XXI" + ], + "et": [ + "21. sajand" + ], + "eu": [ + "XXI. mendea" + ], + "cs": [ + "21. století" + ], + "pl": [ + "XXI wiek" + ], + "ro": [ + "Secolul al XXI-lea" + ], + "da": [ + "21. århundrede" + ], + "nl": [ + "21e eeuw" + ] + }, + "altLabel": { + "de": [ + "21. Jh.", + "21. Jhd.", + "21. Jhdt.", + "21. Jhrh.", + "21. Jahrh.", + "21. Zentennium", + "21. Centennium", + "21. Hektode" + ], + "sv": [ + "21:a århundradet", + "21:a seklet", + "2000-tal", + "2000-talet (århundrade)", + "2000-talet (sekel)" + ], + "fi": [ + "21. vuosisata" + ], + "ru": [ + "21 век" + ], + "en": [ + "21st-century", + "21st-century", + "Twenty-first century", + "History, 21st Century", + "XXI Century", + "Current century" + ], + "it": [ + "21° secolo", + "Secolo XXI", + "Ventunesimo secolo", + "XXI Secolo", + "Duemila" + ], + "pl": [ + "21 wiek", + "21 stulecie", + "XXI stulecie", + "XXI w" + ], + "fr": [ + "21e siècle", + "Vingt et unième siècle" + ], + "es": [ + "Historia, siglo 21", + "Siglo 21", + "Este siglo", + "Siglo veintiuno" + ], + "ca": [ + "Segle 21" + ] + }, + "begin": { + "def": [ + "2001-01-01" + ] + }, + "end": { + "def": [ + "2100-12-31" + ] + }, + "owlSameAs": [ + "http://www.wikidata.org/entity/Q6939", + "http://data.culture.fr/thesaurus/resource/ark:/67717/T4-46", + "http://id.loc.gov/authorities/names/sh85139024", + "http://data.bnf.fr/ark:/12148/cb11930812p", + "http://id.nlm.nih.gov/mesh/D049674", + "https://www.freebase.com/m/08b43", + "https://g.co/kg/m/08b43", + "http://id.nlm.nih.gov/mesh/K01.400.504.984", + "http://vocab.getty.edu/aat/300404515", + "http://id.worldcat.org/fast/1159816", + "http://babelnet.org/rdf/s01157225n", + "http://dbpedia.org/resource/21st_century" + ] + }, + { + "about": "808057XXCX2023X06X08", + "begin": { + "def": [ + "2023-06-08" + ] + } + } + ], + "timestamp_created": "2024-01-18T11:16:27.079Z", + "timestamp_created_epoch": 1705576587079, + "timestamp_update": "2024-01-18T11:16:27.079Z", + "timestamp_update_epoch": 1705576587079, + "type": "SOUND" + } + } +] \ No newline at end of file