diff --git a/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java b/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java index 3f8905fb938..4058005bf3d 100644 --- a/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java +++ b/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java @@ -67,14 +67,24 @@ private IssnData extractIssnData(String json) throws JSONException { String cleanTitle = cleanText(title); issnData.setMainTitle(cleanTitle); return issnData; - } else if (jsonArray.getJSONObject(i).has("name")) { - // name and mainTitle always in same object - therefore if - // no mainTitle but name present, no mainTitle in data + } + } + // If we reach this point it means the mainTitle was not available. + // Lets iterate again now looking for key + for (int i = 0; i < jsonArray.length(); i++) { + if (jsonArray.getJSONObject(i).has("name")) { try { - issnData.setMainTitle(jsonArray.getJSONObject(i).getJSONArray("name").getString(0)); + String title = jsonArray.getJSONObject(i).getString("name"); + issnData.setMainTitle(cleanText(title)); } catch (JSONException e) { - // may not be an array - issnData.setMainTitle(jsonArray.getJSONObject(i).getString("name")); + // may be an array + try { + String title = jsonArray.getJSONObject(i).getJSONArray("name").getString(0); + issnData.setMainTitle(cleanText(title)); + } catch(Exception ee) { + // Nothing else to try, propagate the exception + throw ee; + } } return issnData; } diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java index 8924d5d6e3a..f70288a3476 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java @@ -7,6 +7,7 @@ import javax.annotation.Resource; +import org.apache.commons.lang3.StringUtils; import org.orcid.core.groupIds.issn.IssnClient; import org.orcid.core.groupIds.issn.IssnData; import org.orcid.core.groupIds.issn.IssnValidator; @@ -18,6 +19,7 @@ import org.orcid.persistence.jpa.entities.InvalidIssnGroupIdRecordEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component @@ -27,8 +29,12 @@ public class IssnLoadSource { private static Pattern issnGroupTypePattern = Pattern.compile("^issn:(\\d{4}-\\d{3}[\\dXx])$"); - private static final int BATCH_SIZE = 30; + @Value("${org.orcid.scheduler.issnLoadSource.batchSize:50}") + private int batchSize; + @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:30000}") + private int waitBetweenBatches; + @Resource private GroupIdRecordDao groupIdRecordDao; @@ -64,24 +70,18 @@ public void loadIssn(String issnSource) { private void updateIssnGroupIdRecords() { Date start = new Date(); - List issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsNotModifiedSince(BATCH_SIZE, start); + List issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsNotModifiedSince(batchSize, start); int count = 0; while (!issnEntities.isEmpty()) { for (GroupIdRecordEntity issnEntity : issnEntities) { String issn = getIssn(issnEntity); if (issn != null && issnValidator.issnValid(issn)) { + count++; IssnData issnData = issnClient.getIssnData(issn); if (issnData != null) { updateIssnEntity(issnEntity, issnData); - count++; - try { - LOG.info("Updated group id record {} - {}, processed count now {}", - new Object[] { issnEntity.getId(), issnEntity.getGroupId(), Integer.toString(count) }); - Thread.sleep(10000l); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + LOG.info("Updated group id record {} - {}, processed count now {}", + new Object[] { issnEntity.getId(), issnEntity.getGroupId(), Integer.toString(count) }); } else { LOG.warn("ISSN data not found for {}", issn); recordFailure(issnEntity.getId(), "Data not found"); @@ -89,9 +89,20 @@ private void updateIssnGroupIdRecords() { } else { LOG.info("Issn for group record {} not valid: {}", issnEntity.getId(), issnEntity.getGroupId()); recordFailure(issnEntity.getId(), "Invalid record"); + } + try { + // Lets sleep for 30 secs after processing one batch + if(count >= batchSize) { + Thread.sleep(waitBetweenBatches); + // Reset the count + count = 0; + } + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } } - issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsNotModifiedSince(BATCH_SIZE, start); + issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsNotModifiedSince(batchSize, start); } } @@ -104,10 +115,16 @@ private void recordFailure(Long id, String notes) { private void updateIssnEntity(GroupIdRecordEntity issnEntity, IssnData issnData) { String currentGroupName = issnEntity.getGroupName(); - issnEntity.setGroupName(issnData.getMainTitle()); - issnEntity.setClientSourceId(orcidSource.getId()); - LOG.info("group id: " + issnEntity.getGroupId() + " | current group name: " + currentGroupName + " | group name to be updated: " + issnEntity.getGroupName()); - groupIdRecordDao.merge(issnEntity); + String updatedGroupName = issnData.getMainTitle(); + + if(!StringUtils.equals(currentGroupName, updatedGroupName)) { + issnEntity.setGroupName(updatedGroupName); + issnEntity.setClientSourceId(orcidSource.getId()); + LOG.info("group id: " + issnEntity.getGroupId() + " | current group name: " + currentGroupName + " | group name to be updated: " + issnEntity.getGroupName()); + groupIdRecordDao.merge(issnEntity); + } else { + LOG.info("Group id: " + issnEntity.getGroupId() + " is up to date"); + } } private String getIssn(GroupIdRecordEntity issnEntity) {