icewind: enable config of optional retrySkipped mode

opencadc · Oct 26, 2023 · 3a6c72a · 3a6c72a
1 parent 4fb8ac8
commit 3a6c72a
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 6 deletions.
diff --git a/icewind/README.md b/icewind/README.md
@@ -44,6 +44,10 @@ org.opencadc.icewind.basePublisherID={uri}
 
 # (optional) exit after processing collections once
 #org.opencadc.icewind.exitWhenComplete=true
+
+# (optional) retry previously failed (skipped) observations
+# this mode always assumes exitWhenComplete=true
+org.opencadc.icewind.retrySkipped = true
 ```
 
 The _caom_ database account owns and manages (create, alter, drop) CAOM database objects
@@ -73,6 +77,12 @@ allows for the whole "data collection" to be registered in an IVOA registry usin
 `icewind` normally runs forever; the _exitWhenComplete_ flag (optional) can
 be set to `true` to cause the process to exit after syncing each collection once.
 
+`icewind` normally does incremental harvest of new or modified observations from 
+the source; the _retrySkipped_ flag (optional) can be set to `true` to cause it to
+retry previously failed (skipped) observations listed in the `caom2.HarvestSkipURI`
+table. This mode always assumes _exitWhenComplete_ so it terminates after one pass
+through the list.
+
 ### cadcproxy.pem (optional)
 This client certificate can be provided in /config directory. If present, it is used to 
 authenticate to the _repoService_ if the service requests a client certificate. If 

diff --git a/icewind/VERSION b/icewind/VERSION
@@ -1,6 +1,6 @@
 ## deployable containers have a semantic and build tag
 # semantic version tag: major.minor[.patch]
 # build version tag: timestamp
-VER=0.9.7
+VER=0.9.8
 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")"
 unset VER
diff --git a/icewind/src/main/java/org/opencadc/icewind/Main.java b/icewind/src/main/java/org/opencadc/icewind/Main.java
@@ -97,7 +97,8 @@ public class Main {
     private static final String CERTIFICATE_FILE_LOCATION = System.getProperty("user.home") + "/.ssl/cadcproxy.pem";
     private static final String CONFIG_PREFIX = Main.class.getPackage().getName();
     private static final String LOGGING_CONFIG_KEY = CONFIG_PREFIX + ".logging";
-
+
+    private static final String RETRY_SKIPPED_CONFIG_KEY = CONFIG_PREFIX + ".retrySkipped";
     private static final String REPO_SERVICE_CONFIG_KEY = CONFIG_PREFIX + ".repoService";
     private static final String COLLECTION_CONFIG_KEY = CONFIG_PREFIX + ".collection";
     private static final String MAX_IDLE_CONFIG_KEY = CONFIG_PREFIX + ".maxIdle";
@@ -197,9 +198,19 @@ public static void main(final String[] args) {
                                                                  BASE_PUBLISHER_ID_CONFIG_KEY, configuredBasePublisherIDUrl));
             }
 
+            final boolean retrySkipped;
+            final String configuredRetrySkipped = props.getFirstPropertyValue(RETRY_SKIPPED_CONFIG_KEY);
+            if (configuredRetrySkipped == null) {
+                retrySkipped = false;
+            } else {
+                retrySkipped = Boolean.parseBoolean(configuredRetrySkipped);
+            }
+
             final boolean exitWhenComplete;
             final String configuredExitWhenComplete = props.getFirstPropertyValue(EXIT_WHEN_COMPLETE_CONFIG_KEY);
-            if (configuredExitWhenComplete == null) {
+            if (retrySkipped) {
+                exitWhenComplete = true;
+            } else if (configuredExitWhenComplete == null) {
                 exitWhenComplete = DEFAULT_EXIT_WHEN_COMPLETE;
             } else {
                 exitWhenComplete = Boolean.parseBoolean(configuredExitWhenComplete);
@@ -208,13 +219,12 @@ public static void main(final String[] args) {
             final String configuredMaxSleep = props.getFirstPropertyValue(MAX_IDLE_CONFIG_KEY);
             final long maxSleep = Long.parseLong(configuredMaxSleep);
 
-            // full=false, skip=false: incremental harvest
+            // full=false: incremental harvest
             final boolean full = false;
-            final boolean skip = false;
             final boolean noChecksum = false;
             CaomHarvester harvester = new CaomHarvester(sourceHarvestResource, destinationHarvestResource,
                     configuredCollections, basePublisherID, DEFAULT_BATCH_SIZE, DEFAULT_BATCH_SIZE / 10,
-                                                        full, skip, noChecksum, exitWhenComplete, maxSleep);
+                                                        full, retrySkipped, noChecksum, exitWhenComplete, maxSleep);
 
             Subject subject = AuthenticationUtil.getAnonSubject();
             File cert = new File(CERTIFICATE_FILE_LOCATION);