Skip to content

Commit

Permalink
implement stopgap request retries approach
Browse files Browse the repository at this point in the history
  • Loading branch information
alexdunnjpl committed Oct 19, 2023
1 parent b9e9528 commit 562b2ed
Showing 1 changed file with 19 additions and 0 deletions.
19 changes: 19 additions & 0 deletions src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,20 @@ private String loadBatch(BufferedReader fileReader, String firstLine) throws Exc
* @throws Exception an exception
*/
public int loadBatch(List<String> data, Set<String> errorLidvids) throws Exception
{
int defaultRetries = 5;
return loadBatch(data, errorLidvids, defaultRetries);
}

/**
* Load data into Elasticsearch
* @param data NJSON data. (2 lines per record)
* @param errorLidvids output parameter. If not null, add failed LIDVIDs to this set.
* @param retries number of times to retry the request if an exception is thrown.
* @return Number of loaded documents
* @throws Exception an exception
*/
public int loadBatch(List<String> data, Set<String> errorLidvids, int retries) throws Exception
{
if(data == null || data.isEmpty()) return 0;
if(data.size() % 2 != 0) throw new Exception("Data list size should be an even number.");
Expand Down Expand Up @@ -311,6 +325,11 @@ public int loadBatch(List<String> data, Set<String> errorLidvids) throws Excepti
String msg = EsUtils.extractReasonFromJson(json);
if(msg == null) msg = json;

if (retries > 0) {
log.warn("DataLoader.loadBatch() request failed due to \"" + msg + "\" ("+ retries +" retries remaining)");
return loadBatch(data, errorLidvids, retries);
}

throw new Exception(msg);
}
finally
Expand Down

0 comments on commit 562b2ed

Please sign in to comment.