Merge pull request #565 from opencadc/vos2

merge vos2 feature branch
opencadc · Apr 3, 2024 · ac0bc17 · ac0bc17
2 parents e1dfb6d + 1cf7ea5
commit ac0bc17
Show file tree

Hide file tree

Showing 155 changed files with 10,631 additions and 1,622 deletions.
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -66,19 +66,10 @@ jobs:
 
     - name: java build -- raven
       run: cd raven && ../gradlew --info clean build javadoc checkstyleMain
-
-    - name: java build -- ringhold
-      run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain
 
     - name: java build -- ratik
       run: cd ratik && ../gradlew --info clean build javadoc checkstyleMain
 
-## TODO: docker build depends on cadc-tomcat base image from docker-base.git
-#    - name: docker build -- baldur
-#      run: cd baldur && docker build . --file Dockerfile --tag baldur:$(date +%s)
-#    - name: docker build -- minoc
-#      run: cd minoc && docker build . --file Dockerfile --tag minoc:$(date +%s)
-#    - name: docker build -- luskan
-#      run: cd luskan && docker build . --file Dockerfile --tag luskan:$(date +%s)
-#    - name: docker build -- raven
-#      run: cd raven && docker build . --file Dockerfile --tag raven:$(date +%s)
+    - name: java build -- ringhold
+      run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain
+
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -0,0 +1,70 @@
+# Change Log
+
+This is a cursory summary of changes to various storage-inventory components.
+Check the README in specific modules for details.
+
+## minoc:1.0.0
+```
+added optional `org.opencadc.minoc.trust.preauth` config
+removed optional `org.opencadc.minoc.publicKeyFile` config
+```
+A `minoc` instance will download a public key from each trusted service and
+use the public key(s) to validate URLs that include a _preauth_ token.
+
+```
+added optional `org.opencadc.minoc.readable` and `org.opencadc.minoc.writable` config
+```
+A `minoc` service will advertise (via inventory.StorageSite record in the database) the
+_readable_ and _writable_ status; this information is synced to global inventory and
+used by `raven` to determine if it should generate PUT or GET URLs that use the `minoc`
+service(s) at that site. The configuration of _readGrantProvider_(s) and
+_writeGrantProvider_(s) implicitly determines the status (_readable_ and _writable_
+respectively); configuration of any _trust.preauth_ will also implicitly make make the 
+status _readable_ and _writable_.
+
+The explicit _readable_ and _writable_ configuration options will override the above 
+implicit logic and set the status accordingly. This is currently optional but may be required
+in a future version.
+
+```
+added optional config file: cadc-log.properties
+added optional config file: cadc-vosi.properties
+```
+
+## raven:1.0.0
+```
+added org.opencadc.raven.inventory connection pool
+```
+A `raven` service uses this pool to perform database initialization. This pool is 
+configured in the `catalina.properties` file.
+
+```
+added optional `org.opencadc.raven.keys.preauth` config
+removed optional `org.opencadc.raven.publicKeyFile` and `org.opencadc.minoc.privateKeyFile` config
+```
+When configured to do so, a `raven` service will generate it's own public/private key pair
+and use the private key to _sign_ `minoc` URLs. All the `minoc` services known to the global
+`raven` service must also be configured to _trust_ `raven`.
+
+```
+added optional config file: cadc-log.properties
+added optional config file: cadc-vosi.properties
+```
+
+## luskan:1.0.0
+```
+changed config file: cadc-tap-tmp.properties
+```
+A `luskan` service now uses the DelegatingStorageManager` so this config file must
+specify which storage manager implementation to use along with existing 
+implementation-specific configuration options.
+
+```
+added optional config file: cadc-log.properties
+added optional config file: cadc-vosi.properties
+```
+
+## vault:0.5.0 (NEW)
+This is a new service that implements the IVOA VOSpace 2.1 REST API to provide user-managed
+hierarchical storage. `vault` is deployed with it's own database and an associated inventory
+database, uses inventory services (`minoc`) for file storage and management.
diff --git a/README.md b/README.md
@@ -46,6 +46,10 @@ local artifact selection policy. This is used if the new fenwick policy excludes
 This is an implementation of the **file-validate** process that compares the inventory database against 
 the back end storage at a storage site.
 
+## vault
+UNDER DEVELOPMENT: This is an implementation of an <a href="https://www.ivoa.net/documents/TAP/">IVOA VOSpace</a> 
+service that uses storage-inventory as the back end storage mechanism.
+
 ## cadc-*
 These are libraries used in multiple services and applications.
 

diff --git a/baldur/README.md b/baldur/README.md
@@ -112,6 +112,9 @@ When more than one entry matches an artifact URI, the grants are combined as fol
 * the members of any of the groups in all matching readOnlyGroup lists are allowed to read
 * the members of any of the groups in all matching readWriteGroup lists are allowed to read and write
 
+### cadc-log.properties (optional)
+See <a href="https://github.com/opencadc/core/tree/master/cadc-log">cadc-log</a> for common 
+dynamic logging control.
 
 ## integration testing
 

diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle
@@ -17,16 +17,18 @@ sourceCompatibility = 1.8
 
 group = 'org.opencadc'
 
-version = '0.14.6'
+version = '1.0.0'
 
 description = 'OpenCADC Storage Inventory database library'
 def git_url = 'https://github.com/opencadc/storage-inventory'
 
 mainClassName = 'org.opencadc.inventory.db.version.Main'
 
 dependencies {
-    compile 'org.opencadc:cadc-util:[1.9.5,2.0)'
-    compile 'org.opencadc:cadc-inventory:[0.9.4,)'
+    compile 'org.opencadc:cadc-util:[1.11.0,2.0)'
+    compile 'org.opencadc:cadc-gms:[1.0.0,)'
+    compile 'org.opencadc:cadc-inventory:[1.0.0,)'
+    compile 'org.opencadc:cadc-vos:[2.0.6,3.0)'
 
     testCompile 'junit:junit:[4.0,)'
 

diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java
@@ -94,10 +94,11 @@
 import org.junit.Test;
 import org.opencadc.inventory.Artifact;
 import org.opencadc.inventory.InventoryUtil;
+import org.opencadc.inventory.Namespace;
 import org.opencadc.inventory.SiteLocation;
 import org.opencadc.inventory.StorageLocation;
 import org.opencadc.inventory.StoredArtifactComparator;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -126,22 +127,23 @@ public ArtifactDAOTest() throws Exception {
             config.put(SQLGenerator.class.getName(), SQLGenerator.class);
             config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest");
             config.put("database", TestUtil.DATABASE);
-            config.put("schema", TestUtil.SCHEMA);
-
+            config.put("invSchema", TestUtil.SCHEMA);
+            config.put("genSchema", TestUtil.SCHEMA);
+
             originDAO = new ArtifactDAO();
             originDAO.setConfig(config);
-            
+
             nonOriginDAO = new ArtifactDAO(false);
             nonOriginDAO.setConfig(config);
-            
+
             DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest-alt", cc);
             Map<String,Object> altConfig = new TreeMap<String,Object>();
             altConfig.put(SQLGenerator.class.getName(), SQLGenerator.class);
             altConfig.put("jndiDataSourceName", "jdbc/ArtifactDAOTest-alt");
             altConfig.put("database", TestUtil.DATABASE);
-            altConfig.put("schema", TestUtil.SCHEMA);
+            altConfig.put("invSchema", TestUtil.SCHEMA);
+            altConfig.put("genSchema", TestUtil.SCHEMA);
             altDAO.setConfig(altConfig);
-
         } catch (Exception ex) {
             log.error("setup failed", ex);
             throw ex;
@@ -151,7 +153,7 @@ public ArtifactDAOTest() throws Exception {
     @Before
     public void init_cleanup() throws Exception {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(originDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(originDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");
 
@@ -954,6 +956,9 @@ public void testIteratorClose() {
     public void testArtifactIterator() {
         int num = 10;
         try {
+            final Date startDate = new Date();
+            Thread.sleep(10L);
+
             int numArtifacts = 0;
             int numStuffExpected = 0;
             // artifacts with storageLocation
@@ -976,6 +981,7 @@ public void testArtifactIterator() {
                     numStuffExpected++;
                 }
             }
+
             // some artifacts with no storageLocation
             collection = "STUFF";
             for (int i = num; i < 2 * num; i++) {
@@ -995,6 +1001,12 @@ public void testArtifactIterator() {
                     numStuffExpected++;
                 }
             }
+
+            Thread.sleep(10L);
+            final Date midDate = new Date();
+            Thread.sleep(10L);
+            final int midNumStuff = numStuffExpected;
+
             // some artifacts with siteLocations
             UUID siteID = UUID.randomUUID();
             int numSiteExpected = 0;
@@ -1017,6 +1029,8 @@ public void testArtifactIterator() {
                     numStuffExpected++;
                 }
             }
+            Thread.sleep(10L);
+            final Date endDate = new Date();
             log.info("added: " + numArtifacts);
 
             log.info("count all...");
@@ -1031,13 +1045,14 @@ public void testArtifactIterator() {
             Assert.assertEquals("count", numArtifacts, count);
 
             log.info("count with criteria...");
+            final Namespace ns = new Namespace("cadc:STUFF/");
             count = 0;
-            try (ResourceIterator<Artifact> iter = originDAO.iterator("uri like 'cadc:STUFF/%'", null, false)) {
+            try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, false)) {
                 while (iter.hasNext()) {
                     Artifact actual = iter.next();
                     count++;
                     log.info("found: " + actual.getURI());
-                    Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith("cadc:STUFF/"));
+                    Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
                 }
             }
             Assert.assertEquals("count", numStuffExpected, count);
@@ -1064,7 +1079,8 @@ public void testArtifactIterator() {
                     while (iter.hasNext()) {
                         Artifact actual = iter.next();
                         count++;
-                        log.info("found: " + actual.getURI());
+                        log.info("found: " + actual.getBucket() + " " + actual.getURI());
+                        Assert.assertTrue(actual.getBucket().startsWith(bpre));
                     }
                 }
             }
@@ -1074,18 +1090,56 @@ public void testArtifactIterator() {
             count = 0;
             for (byte b = 0; b < 16; b++) {
                 String bpre = HexUtil.toHex(b).substring(1);
-                log.debug("bucket prefix: " + bpre);
-                try (ResourceIterator<Artifact> iter = originDAO.iterator("uri like 'cadc:STUFF/%'", bpre, false)) {
+                log.info("bucket prefix: " + bpre);
+                try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, bpre, false)) {
                     while (iter.hasNext()) {
                         Artifact actual = iter.next();
                         count++;
-                        log.info("found: " + actual.getURI());
-                        Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith("cadc:STUFF/"));
+                        log.info("found: " + actual.getBucket() + " " + actual.getURI());
+                        Assert.assertTrue(actual.getBucket().startsWith(bpre));
+                        Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
                     }
                 }
             }
             Assert.assertEquals("count", numStuffExpected, count);
 
+            log.info("count vs Namespace incremental from start...");
+            DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC);
+            count = 0;
+            try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, startDate, true)) {
+                while (iter.hasNext()) {
+                    Artifact actual = iter.next();
+                    count++;
+                    log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified()));
+                    Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
+                }
+            }
+            Assert.assertEquals("count", numStuffExpected, count);
+
+            log.info("count vs Namespace incremental from mid...");
+            count = 0;
+            try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, midDate, true)) {
+                while (iter.hasNext()) {
+                    Artifact actual = iter.next();
+                    count++;
+                    log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified()));
+                    Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
+                }
+            }
+            Assert.assertEquals("count", midNumStuff, count);
+
+            log.info("count vs Namespace incremental from end...");
+            count = 0;
+            try (ResourceIterator<Artifact> iter = originDAO.iterator(ns, null, endDate, true)) {
+                while (iter.hasNext()) {
+                    Artifact actual = iter.next();
+                    count++;
+                    log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified()));
+                    Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace()));
+                }
+            }
+            Assert.assertEquals("count", 0, count);
+
         } catch (Exception unexpected) {
             log.error("unexpected exception", unexpected);
             Assert.fail("unexpected exception: " + unexpected);

diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/EntityEventDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/EntityEventDAOTest.java
@@ -87,7 +87,7 @@
 import org.opencadc.inventory.DeletedStorageLocationEvent;
 import org.opencadc.inventory.InventoryUtil;
 import org.opencadc.inventory.StorageLocationEvent;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
 
 /**
  *
@@ -106,25 +106,31 @@ public class EntityEventDAOTest {
     StorageLocationEventDAO slDAO = new StorageLocationEventDAO();
 
     public EntityEventDAOTest() throws Exception {
-        DBConfig dbrc = new DBConfig();
-        ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE);
-        DBUtil.createJNDIDataSource("jdbc/EntityEventDAOTest", cc);
-
-        Map<String,Object> config = new TreeMap<String,Object>();
-        config.put(SQLGenerator.class.getName(), SQLGenerator.class);
-        config.put("jndiDataSourceName", "jdbc/EntityEventDAOTest");
-        config.put("database", TestUtil.DATABASE);
-        config.put("schema", TestUtil.SCHEMA);
-        daeDAO.setConfig(config);
-        dslDAO.setConfig(config);
-        slDAO.setConfig(config);
+        try {
+            DBConfig dbrc = new DBConfig();
+            ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE);
+            DBUtil.createJNDIDataSource("jdbc/EntityEventDAOTest", cc);
+
+            Map<String,Object> config = new TreeMap<String,Object>();
+            config.put(SQLGenerator.class.getName(), SQLGenerator.class);
+            config.put("jndiDataSourceName", "jdbc/EntityEventDAOTest");
+            config.put("database", TestUtil.DATABASE);
+            config.put("invSchema", TestUtil.SCHEMA);
+            config.put("genSchema", TestUtil.SCHEMA);
+            daeDAO.setConfig(config);
+            dslDAO.setConfig(config);
+            slDAO.setConfig(config);
+        } catch (Exception ex) {
+            log.error("setup failed", ex);
+            throw ex;
+        }
     }
 
     @Before
     public void setup()
         throws Exception {
         log.info("init database...");
-        InitDatabase init = new InitDatabase(daeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
+        InitDatabaseSI init = new InitDatabaseSI(daeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA);
         init.doInit();
         log.info("init database... OK");