changes in cadc-inventory-db to support using HarvestState in vault

baldur/README.md

@@ -112,6 +112,9 @@ When more than one entry matches an artifact URI, the grants are combined as fol
 * the members of any of the groups in all matching readOnlyGroup lists are allowed to read
 * the members of any of the groups in all matching readWriteGroup lists are allowed to read and write
 
+### cadc-log.properties (optional)
+See <a href="https://github.com/opencadc/core/tree/master/cadc-log">cadc-log</a> for common 
+dynamic logging control.
 
 ## integration testing
 

cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java

@@ -97,7 +97,7 @@
 import org.opencadc.inventory.SiteLocation;
 import org.opencadc.inventory.StorageLocation;
 import org.opencadc.inventory.StoredArtifactComparator;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -126,22 +126,23 @@ public ArtifactDAOTest() throws Exception {
             config.put(SQLGenerator.class.getName(), SQLGenerator.class);
             config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest");
             config.put("database", TestUtil.DATABASE);
-            config.put("schema", TestUtil.SCHEMA);
-
+            config.put("invSchema", TestUtil.SCHEMA);
+            config.put("genSchema", TestUtil.SCHEMA);
+
             originDAO = new ArtifactDAO();
             originDAO.setConfig(config);
-            
+
             nonOriginDAO = new ArtifactDAO(false);
             nonOriginDAO.setConfig(config);
-            
+
             DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest-alt", cc);
             Map<String,Object> altConfig = new TreeMap<String,Object>();
             altConfig.put(SQLGenerator.class.getName(), SQLGenerator.class);
             altConfig.put("jndiDataSourceName", "jdbc/ArtifactDAOTest-alt");
             altConfig.put("database", TestUtil.DATABASE);
-            altConfig.put("schema", TestUtil.SCHEMA);
+            altConfig.put("invSchema", TestUtil.SCHEMA);
+            altConfig.put("genSchema", TestUtil.SCHEMA);
             altDAO.setConfig(altConfig);
-
         } catch (Exception ex) {
             log.error("setup failed", ex);
             throw ex;
@@ -151,7 +152,7 @@ public ArtifactDAOTest() throws Exception {
     @Before
     public void init_cleanup() throws Exception {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(originDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(originDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");
 

cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/EntityEventDAOTest.java

@@ -87,7 +87,7 @@
 import org.opencadc.inventory.DeletedStorageLocationEvent;
 import org.opencadc.inventory.InventoryUtil;
 import org.opencadc.inventory.StorageLocationEvent;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -106,25 +106,31 @@ public class EntityEventDAOTest {
     StorageLocationEventDAO slDAO = new StorageLocationEventDAO();
 
     public EntityEventDAOTest() throws Exception {
-        DBConfig dbrc = new DBConfig();
-        ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE);
-        DBUtil.createJNDIDataSource("jdbc/EntityEventDAOTest", cc);
-
-        Map<String,Object> config = new TreeMap<String,Object>();
-        config.put(SQLGenerator.class.getName(), SQLGenerator.class);
-        config.put("jndiDataSourceName", "jdbc/EntityEventDAOTest");
-        config.put("database", TestUtil.DATABASE);
-        config.put("schema", TestUtil.SCHEMA);
-        daeDAO.setConfig(config);
-        dslDAO.setConfig(config);
-        slDAO.setConfig(config);
+        try {
+            DBConfig dbrc = new DBConfig();
+            ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE);
+            DBUtil.createJNDIDataSource("jdbc/EntityEventDAOTest", cc);
+
+            Map<String,Object> config = new TreeMap<String,Object>();
+            config.put(SQLGenerator.class.getName(), SQLGenerator.class);
+            config.put("jndiDataSourceName", "jdbc/EntityEventDAOTest");
+            config.put("database", TestUtil.DATABASE);
+            config.put("invSchema", TestUtil.SCHEMA);
+            config.put("genSchema", TestUtil.SCHEMA);
+            daeDAO.setConfig(config);
+            dslDAO.setConfig(config);
+            slDAO.setConfig(config);
+        } catch (Exception ex) {
+            log.error("setup failed", ex);
+            throw ex;
+        }
     }
 
     @Before
     public void setup()
         throws Exception {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(daeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(daeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");
 

cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/HarvestStateDAOTest.java

@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2022.                            (c) 2022.
+*  (c) 2024.                            (c) 2024.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -84,7 +84,7 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -112,7 +112,8 @@ public HarvestStateDAOTest() throws Exception {
             config.put(SQLGenerator.class.getName(), SQLGenerator.class);
             config.put("jndiDataSourceName", "jdbc/HarvestStateDAOTest");
             config.put("database", TestUtil.DATABASE);
-            config.put("schema", TestUtil.SCHEMA);
+            config.put("invSchema", TestUtil.SCHEMA);
+            config.put("genSchema", TestUtil.SCHEMA);
             dao.setConfig(config);
         } catch (Exception ex) {
             log.error("setup failed", ex);
@@ -123,7 +124,7 @@ public HarvestStateDAOTest() throws Exception {
     @Before
     public void setup() throws Exception {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");
 
@@ -136,6 +137,11 @@ public void setup() throws Exception {
         log.info("clearing old content... OK");
     }
 
+    @Test
+    public void noop() {
+        log.info("no-op - just setup");
+    }
+
     @Test
     public void testPutGetUpdateDelete() {
         try {
@@ -172,6 +178,7 @@ public void testPutGetUpdateDelete() {
             // update
             hs1.curLastModified = new Date();
             hs1.curID = UUID.randomUUID();
+            hs1.instanceID = UUID.randomUUID();
             dao.put(hs1);
 
             //log.warn("SLEEPING for lock diagnostics: 20 sec");
@@ -181,12 +188,25 @@ public void testPutGetUpdateDelete() {
             HarvestState hs2 = dao.get(hs1.getID());
             log.info("found: " + hs2);
 
-            Assert.assertNotNull("find by uuid", hs1);
+            Assert.assertNotNull("find by uuid", hs2);
             Assert.assertNotEquals(expected.getLastModified(), hs2.getLastModified());
             Assert.assertNotEquals(expected.getMetaChecksum(), hs2.getMetaChecksum());
             Assert.assertEquals("round trip metachecksum", hs1.getMetaChecksum(), hs2.getMetaChecksum());
             Assert.assertEquals("curLastModified", hs1.curLastModified.getTime(), hs2.curLastModified.getTime());
             Assert.assertEquals("curID", hs1.curID, hs2.curID);
+            Assert.assertEquals("instanceID", hs1.instanceID, hs2.instanceID);
+
+            // force update with no state change
+            final Date prevT = hs2.getLastModified();
+            Thread.sleep(100L);
+            dao.put(hs2, true);
+
+            HarvestState tup = dao.get(hs2.getID());
+            Assert.assertNotNull(tup);
+            URI tupCS = tup.computeMetaChecksum(MessageDigest.getInstance("MD5"));
+            Assert.assertEquals("meta: no change", tup.getMetaChecksum(), tupCS);
+            Assert.assertTrue("timestamp: update", tup.getLastModified().after(prevT));
+            log.info("force update OK: " + tup);
 
             // clear tracking state
             hs1.curLastModified = null;

cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ObsoleteStorageLocationDAOTest.java

@@ -67,7 +67,6 @@
 
 package org.opencadc.inventory.db;
 
-import org.opencadc.inventory.ObsoleteStorageLocation;
 import ca.nrc.cadc.db.ConnectionConfig;
 import ca.nrc.cadc.db.DBConfig;
 import ca.nrc.cadc.db.DBUtil;
@@ -83,8 +82,9 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.opencadc.inventory.ObsoleteStorageLocation;
 import org.opencadc.inventory.StorageLocation;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -110,7 +110,8 @@ public ObsoleteStorageLocationDAOTest() throws Exception {
             config.put(SQLGenerator.class.getName(), SQLGenerator.class);
             config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest");
             config.put("database", TestUtil.DATABASE);
-            config.put("schema", TestUtil.SCHEMA);
+            config.put("invSchema", TestUtil.SCHEMA);
+            config.put("genSchema", TestUtil.SCHEMA);
             dao.setConfig(config);
         } catch (Exception ex) {
             log.error("setup failed", ex);
@@ -123,7 +124,7 @@ public void setup()
         throws Exception
     {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");
 

cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java

@@ -86,7 +86,7 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.opencadc.inventory.PreauthKeyPair;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -111,7 +111,8 @@ public PreauthKeyPairDAOTest()throws Exception {
         config.put(SQLGenerator.class.getName(), SQLGenerator.class);
         config.put("jndiDataSourceName", "jdbc/PreauthKeyPairDAOTest");
         config.put("database", TestUtil.DATABASE);
-        config.put("schema", TestUtil.SCHEMA);
+        config.put("invSchema", TestUtil.SCHEMA);
+        config.put("genSchema", TestUtil.SCHEMA);
         dao.setConfig(config);
     }
 
@@ -120,7 +121,7 @@ public void setup()
         throws Exception
     {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");
 

cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/StorageSiteDAOTest.java

@@ -84,7 +84,7 @@
 import org.junit.Test;
 import org.opencadc.inventory.InventoryUtil;
 import org.opencadc.inventory.StorageSite;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -109,7 +109,8 @@ public StorageSiteDAOTest() throws Exception {
         config.put(SQLGenerator.class.getName(), SQLGenerator.class);
         config.put("jndiDataSourceName", "jdbc/StorageSiteDAOTest");
         config.put("database", TestUtil.DATABASE);
-        config.put("schema", TestUtil.SCHEMA);
+        config.put("invSchema", TestUtil.SCHEMA);
+        config.put("genSchema", TestUtil.SCHEMA);
         dao.setConfig(config);
     }
 
@@ -118,7 +119,7 @@ public void setup()
         throws Exception
     {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");
 

cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java

@@ -124,7 +124,8 @@ public NodeDAOTest() throws Exception {
             config.put(SQLGenerator.class.getName(), SQLGenerator.class);
             config.put("jndiDataSourceName", "jdbc/NodeDAOTest");
             config.put("database", TestUtil.DATABASE);
-            config.put("schema", TestUtil.SCHEMA);
+            config.put("invSchema", TestUtil.SCHEMA);
+            config.put("genSchema", TestUtil.VOS_SCHEMA);
             config.put("vosSchema", TestUtil.VOS_SCHEMA);
 
             this.nodeDAO = new NodeDAO();

cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java

@@ -190,8 +190,9 @@ public Map<String, Class> getParams() {
         Map<String, Class> ret = new TreeMap<String, Class>();
         ret.put("jndiDataSourceName", String.class);
         ret.put("database", String.class);
-        ret.put("schema", String.class);
-        ret.put("vosSchema", String.class); // optional
+        ret.put("invSchema", String.class);
+        ret.put("genSchema", String.class);
+        ret.put("vosSchema", String.class);
         ret.put(SQLGenerator.class.getName(), Class.class);
         return ret;
     }
@@ -224,11 +225,12 @@ public void setConfig(Map<String, Object> config) {
         }
 
         String database = (String) config.get("database");
-        String schema = (String) config.get("schema");
+        String invSchema = (String) config.get("invSchema");
+        String genSchema = (String) config.get("genSchema");
         String vosSchema = (String) config.get("vosSchema");
         try {
-            Constructor<?> ctor = genClass.getConstructor(String.class, String.class, String.class);
-            this.gen = (SQLGenerator) ctor.newInstance(database, schema, vosSchema);
+            Constructor<?> ctor = genClass.getConstructor(String.class, String.class, String.class, String.class);
+            this.gen = (SQLGenerator) ctor.newInstance(database, invSchema, genSchema, vosSchema);
         } catch (Exception ex) {
             throw new RuntimeException("failed to instantiate SQLGenerator: " + genClass.getName(), ex);
         }

cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java

@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2020.                            (c) 2020.
+*  (c) 2024.                            (c) 2024.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -70,7 +70,6 @@
 import java.net.URI;
 import java.util.Date;
 import java.util.UUID;
-
 import org.apache.log4j.Logger;
 import org.opencadc.inventory.Entity;
 import org.opencadc.inventory.InventoryUtil;
@@ -95,6 +94,12 @@ public class HarvestState extends Entity {
      */
     public UUID curID;
 
+    /**
+     * The ID of the current running instance. This is optional and only used by applications 
+     * that share workload between instances.
+     */
+    public UUID instanceID;
+
     public HarvestState(String name, URI resourceID) { 
         super();
         InventoryUtil.assertNotNull(HarvestState.class, "name", name);
@@ -117,6 +122,7 @@ public String getName() {
     }
 
     public void setName(String name) {
+        InventoryUtil.assertNotNull(HarvestState.class, "name", name);
         this.name = name;
     }
 

cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java

@@ -176,12 +176,16 @@ public HarvestState get(String name, URI resourceID) {
 
     @Override
     public void put(HarvestState val) {
-        if (curBufferCount < updateBufferCount) {
+        put(val, false);
+    }
+
+    public void put(HarvestState val, boolean forceTimestampUpdate) {
+        if (curBufferCount < updateBufferCount && !forceTimestampUpdate) {
             log.debug("buffering: " + curBufferCount + " < " + updateBufferCount + " " + val);
             curBufferCount++;
             bufferedState = val;
         } else {
-            super.put(val);
+            super.put(val, false, forceTimestampUpdate);
             curBufferCount = 0;
             bufferedState = null;