Skip to content

Commit

Permalink
#882 support for species-lists
Browse files Browse the repository at this point in the history
  • Loading branch information
Adam Collins committed Feb 21, 2024
1 parent b2badbf commit 58225ac
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 14 deletions.
13 changes: 8 additions & 5 deletions config/biocache-config.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ deleted.file.store=/data/biocache-delete/
# List tool endpoint
list.tool.url=https://lists.ala.org.au

# Is the list tool the newer species-lists
list.tool.useListWs=false

# SDS data file
sds.url=https://sds.ala.org.au/sensitive-species-data.xml

Expand Down Expand Up @@ -82,9 +85,9 @@ download.internal.queue.size=1000
# Defaults to 1 week (604,800,000ms)
download.max.execute.time=604800000

# Maximum total time in milliseconds for downloads to be allowed to normally complete before they are aborted,
# once all of the Solr/etc. queries have been completed or aborted and the RecordWriter is
# reading the remaining download.internal.queue.size items off the queue.
# Maximum total time in milliseconds for downloads to be allowed to normally complete before they are aborted,
# once all of the Solr/etc. queries have been completed or aborted and the RecordWriter is
# reading the remaining download.internal.queue.size items off the queue.
# Defaults to 5 minutes (300,000ms)
download.max.completion.time=300000

Expand Down Expand Up @@ -122,9 +125,9 @@ solr.downloadquery.busywaitsleep=100

# Defines the thread pool structure used for handling offline downloads
# Download threads for matching subsets of offline downloads.
# The default is:
# The default is:
# * 4 threads for index (SOLR) downloads for <50,000 occurrences with 10ms poll delay, 10ms execution delay, and normal thread priority (5)
# * 1 thread for index (SOLR) downloads for <100,000,000 occurrences with 100ms poll delay, 100ms execution delay, and minimum thread priority (1)
# * 1 thread for index (SOLR) downloads for <100,000,000 occurrences with 100ms poll delay, 100ms execution delay, and minimum thread priority (1)
# * 2 threads for db (CASSANDA) downloads for <50,000 occurrences with 10ms poll delay, 10ms execution delay, and normal thread priority (5)
# * 1 thread for either index or db downloads, an unrestricted count, with 300ms poll delay, 100ms execution delay, and minimum thread priority (1)
# * If there are no thread patterns specified here, a single thread with 10ms poll delay and 0ms execution delay, and normal thread priority (5) will be created and used instead.
Expand Down
87 changes: 79 additions & 8 deletions src/main/java/au/org/ala/biocache/service/ListsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ public class ListsService {
@Value("${list.tool.url:https://lists.ala.org.au}")
private String speciesListUrl;

@Value("${list.tool.useListWs:false}")
private Boolean useListWs;

private Map<String, Map<String, Set<String>>> data = RestartDataService.get(this, "data", new TypeReference<HashMap<String, Map<String, Set<String>>>>(){}, HashMap.class);

@PostConstruct
Expand Down Expand Up @@ -86,8 +89,15 @@ public void run() {
try {
HashMap map = new HashMap();

Map threatened = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/?isThreatened=eq:true&isAuthoritative=eq:true"), Map.class);
Map invasive = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/?isInvasive=eq:true&isAuthoritative=eq:true"), Map.class);
Map threatened;
Map invasive;
if (useListWs) {
threatened = restTemplate.getForObject(new URI(speciesListUrl + "/speciesList/?isThreatened=true&isAuthoritative=true&pageSize=1000"), Map.class);
invasive = restTemplate.getForObject(new URI(speciesListUrl + "/speciesList/?isInvasive=true&isAuthoritative=true&pageSize=1000"), Map.class);
} else {
threatened = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/?isThreatened=eq:true&isAuthoritative=eq:true"), Map.class);
invasive = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/?isInvasive=eq:true&isAuthoritative=eq:true"), Map.class);
}

if ((threatened != null && threatened.size() > 0) ||
(invasive != null && invasive.size() > 0)) {
Expand All @@ -111,8 +121,15 @@ private Map<String, Set<String>> getItemsMap(Map speciesLists, boolean conservat
List ja = (List) speciesLists.get("lists");
Map<String, Set<String>> map = new HashMap();
for (int i = 0; i < ja.size(); i++) {
String name = ((Map) ja.get(i)).get("listName").toString();
String dr = ((Map) ja.get(i)).get("dataResourceUid").toString();
String name;
String dr;
if (useListWs) {
name = ((Map) ja.get(i)).get("title").toString();
dr = ((Map) ja.get(i)).get("id").toString();
} else {
name = ((Map) ja.get(i)).get("listName").toString();
dr = ((Map) ja.get(i)).get("dataResourceUid").toString();
}
List<SpeciesListItemDTO> items = getListItems(dr, conservationList);
for (SpeciesListItemDTO item : items) {
Set<String> existing = map.get(item.lsid);
Expand Down Expand Up @@ -143,9 +160,17 @@ public List<SpeciesListItemDTO> getListItems(String dataResourceUid, Boolean inc
boolean hasAnotherPage = true;
int max = 400; // response size can be limited by the gateway
int offset = 0;
int page = 1;

while (hasAnotherPage) {
List<SpeciesListItemDTO> speciesListItems = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesListItems/" + dataResourceUid + "?max=" + max + "&offset=" + offset + "&includeKVP=" + includeKvp), SpeciesListItemsDTO.class);
List<SpeciesListItemDTO> speciesListItems;
if (useListWs) {
speciesListItems = restTemplate.getForObject(new URI(speciesListUrl + "/speciesListItems/" + dataResourceUid + "?pageSize=" + max + "&page=" + page), SpeciesListItemsDTO.class);
page++;
legacyFormatSpeciesListItems(speciesListItems);
} else {
speciesListItems = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesListItems/" + dataResourceUid + "?max=" + max + "&offset=" + offset + "&includeKVP=" + includeKvp), SpeciesListItemsDTO.class);
}

offset += max;
hasAnotherPage = speciesListItems.size() == max;
Expand Down Expand Up @@ -175,10 +200,18 @@ public List<Kvp> getKvp(String dataResourceUid) {
boolean hasAnotherPage = true;
int max = 400; // response size can be limited by api gateway
int offset = 0;
int page = 1;

try {
while (hasAnotherPage) {
SpeciesListItemsDTO speciesListItems = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesListItems/" + dataResourceUid + "?includeKVP=true&max=" + max + "&offset=" + offset), SpeciesListItemsDTO.class);
SpeciesListItemsDTO speciesListItems;
if (useListWs) {
speciesListItems = restTemplate.getForObject(new URI(speciesListUrl + "/speciesListItems/" + dataResourceUid + "?pageSize=" + max + "&page=" + page), SpeciesListItemsDTO.class);
page++;
legacyFormatSpeciesListItems(speciesListItems);
} else {
speciesListItems = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesListItems/" + dataResourceUid + "?includeKVP=true&max=" + max + "&offset=" + offset), SpeciesListItemsDTO.class);
}

offset += max;
hasAnotherPage = speciesListItems.size() == max;
Expand Down Expand Up @@ -238,7 +271,16 @@ public Set<String> get(String type, String lsid) {
@Cacheable("speciesListItems")
public SpeciesListDTO getListInfo(String dr) throws URISyntaxException {

SpeciesListDTO speciesList = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/" + dr), SpeciesListDTO.class);
SpeciesListDTO speciesList;
if (useListWs) {
speciesList = restTemplate.getForObject(new URI(speciesListUrl + "/speciesList/" + dr), SpeciesListDTO.class);
if (speciesList != null) {
speciesList.dataResourceUid = speciesList.id;
speciesList.listName = speciesList.title;
}
} else {
speciesList = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/" + dr), SpeciesListDTO.class);
}

return speciesList;
}
Expand Down Expand Up @@ -346,6 +388,10 @@ public Optional<SpeciesListDTO> findSpeciesListByDataResourceId(@NotNull String
}

public static class SpeciesListDTO {
// species-list fields
public String id;
public String title;

public String dataResourceUid;
public String listName;
public String listType;
Expand All @@ -366,22 +412,47 @@ public static class SpeciesListDTO {
}

public static class SpeciesListItemDTO {
public long id;
public String id;
public String name;
public String commonName;
public String scientificName;
public String lsid;
public String dataResourceUid;
public List<KvpDTO> kvpValues;
public List<KvpDTO> properties;
public ClassificationDTO classification;

public static class KvpDTO {
public String key;
public String value;
}

public static class ClassificationDTO {
public String taxonConceptID;
}
}

public static class SpeciesListItemsDTO extends ArrayList<SpeciesListItemDTO> {

}

/**
* Make SpeciesListItemsDTO modified for species-lists compatible with existing code
*
* @param speciesListItems
*/
private void legacyFormatSpeciesListItems(List<SpeciesListItemDTO> speciesListItems) {
if (speciesListItems == null || !useListWs) {
return;
}

for (SpeciesListItemDTO item : speciesListItems) {
if (item.classification != null) {
item.lsid = item.classification.taxonConceptID;
}

item.kvpValues = item.properties;
}
}
}

2 changes: 1 addition & 1 deletion src/main/java/au/org/ala/biocache/util/QueryFormatUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public class QueryFormatUtils {

//Patterns that are used to prepare a SOLR query for execution
protected Pattern lsidPattern = Pattern.compile("(^|\\s|\"|\\(|\\[|'|-)taxonConceptID:\"?([a-zA-Z0-9/\\.:\\-_]*)\"?");
protected Pattern speciesListPattern = Pattern.compile("(^|\\s|\"|\\(|\\[|'|-)species_list:\"?(dr[0-9]*)\"?");
protected Pattern speciesListPattern = Pattern.compile("(^|\\s|\"|\\(|\\[|'|-)species_list:\"?([0-9a-z]*)\"?");

protected Pattern spatialObjectPattern = Pattern.compile("(^|\\s|\"|\\(|\\[|'|-)spatialObject:\"?([0-9]*)\"?");
protected Pattern urnPattern = Pattern.compile("\\burn:[a-zA-Z0-9\\.:-]*");
Expand Down

0 comments on commit 58225ac

Please sign in to comment.