From 2556e702c3aac476c1c4b5df16748c67bbdb02b8 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 28 Mar 2023 17:51:45 -0700 Subject: [PATCH 001/186] incomplete exploratory dev --- cadc-inventory-db/build.gradle | 1 + .../org/opencadc/inventory/db/TestUtil.java | 16 +- .../org/opencadc/vospace/db/NodeDAOTest.java | 199 ++++++++++++++++++ .../opencadc/inventory/db/AbstractDAO.java | 8 +- .../opencadc/inventory/db/SQLGenerator.java | 54 ++++- .../opencadc/vospace/db/InitDatabaseVOS.java | 113 ++++++++++ .../java/org/opencadc/vospace/db/NodeDAO.java | 100 +++++++++ .../main/resources/vos.DeletedNodeEvent.sql | 17 ++ .../src/main/resources/vos.ModelVersion.sql | 9 + .../src/main/resources/vos.Node.sql | 31 +++ 10 files changed, 538 insertions(+), 10 deletions(-) create mode 100644 cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java create mode 100644 cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java create mode 100644 cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java create mode 100644 cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql create mode 100644 cadc-inventory-db/src/main/resources/vos.ModelVersion.sql create mode 100644 cadc-inventory-db/src/main/resources/vos.Node.sql diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index ff0e2012b..c1352fd70 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -27,6 +27,7 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.6.2,2.0)' compile 'org.opencadc:cadc-inventory:[0.9,)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java index 92ba24bcc..8ab42bf71 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/TestUtil.java @@ -79,10 +79,11 @@ public class TestUtil { private static final Logger log = Logger.getLogger(TestUtil.class); - static String SERVER = "INVENTORY_TEST"; - static String DATABASE = "cadctest"; - static String SCHEMA = "inventory"; - static String TABLE_PREFIX = null; + public static String SERVER = "INVENTORY_TEST"; + public static String DATABASE = "cadctest"; + public static String SCHEMA = "inventory"; + public static String VOS_SCHEMA = "vospace"; + public static String TABLE_PREFIX = null; static { try { @@ -102,12 +103,17 @@ public class TestUtil { if (s != null) { SCHEMA = s.trim(); } + s = props.getProperty("vos_schema"); + if (s != null) { + VOS_SCHEMA = s.trim(); + } s = props.getProperty("tablePrefix"); if (s != null) { TABLE_PREFIX = s.trim(); } } - log.info("intTest database config: " + SERVER + " " + DATABASE + " " + SCHEMA + " " + TABLE_PREFIX); + log.info("intTest database config: " + SERVER + " " + DATABASE + " " + SCHEMA + " " + VOS_SCHEMA + + " tablePrefix=" + TABLE_PREFIX); } catch (Exception oops) { log.debug("failed to load/read optional db config", oops); } diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java new file mode 100644 index 000000000..ccf475b6d --- /dev/null +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -0,0 +1,199 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2022. (c) 2022. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.db.TestUtil; +import org.opencadc.vospace.db.InitDatabaseVOS; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.LinkNode; +import org.opencadc.vospace.Node; + +/** + * + * @author pdowler + */ +public class NodeDAOTest { + private static final Logger log = Logger.getLogger(NodeDAOTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); + } + + NodeDAO nodeDAO; + + public NodeDAOTest() throws Exception { + try { + DBConfig dbrc = new DBConfig(); + ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); + DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest", cc); + + Map config = new TreeMap(); + config.put(SQLGenerator.class.getName(), SQLGenerator.class); + config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); + config.put("database", TestUtil.DATABASE); + config.put("schema", TestUtil.SCHEMA); + config.put("vosSchema", TestUtil.VOS_SCHEMA); + + this.nodeDAO = new NodeDAO(); + nodeDAO.setConfig(config); + + } catch (Exception ex) { + log.error("setup failed", ex); + throw ex; + } + } + + @Before + public void init_cleanup() throws Exception { + log.info("init database..."); + InitDatabaseVOS init = new InitDatabaseVOS(nodeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.VOS_SCHEMA); + init.doInit(); + log.info("init database... OK"); + + log.info("clearing old content..."); + SQLGenerator gen = nodeDAO.getSQLGenerator(); + DataSource ds = nodeDAO.getDataSource(); + String sql = "delete from " + gen.getTable(Node.class); + log.info("pre-test cleanup: " + sql); + ds.getConnection().createStatement().execute(sql); + log.info("clearing old content... OK"); + } + + @Test + public void testGetByID() { + UUID id = UUID.randomUUID(); + Node a = nodeDAO.get(id); + Assert.assertNull(a); + } + + //@Test + public void testPutGetDeleteContainerNode() { + ContainerNode root = new ContainerNode("root", false); + + ContainerNode n = new ContainerNode("container-test", false); + n.parent = root; + nodeDAO.put(n); + + Node a = nodeDAO.get(n.getID()); + Assert.assertNotNull(a); + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(n.getName(), a.getName()); + // these are set in put + Assert.assertEquals(n.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(n.getLastModified(), a.getLastModified()); + ContainerNode ac = nodeDAO.getChildren(c); + Assert.assertNotNull(ac); + Assert.assertTrue(ac.nodes.isEmpty()); + + // add children + ContainerNode cont = new ContainerNode("container1", false); + cont.parent = c; + DataNode data = new DataNode("data1"); + data.parent = c; + LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); + link.parent = c; + nodeDAO.put(cont); + nodeDAO.put(data); + nodeDAO.put(link); + + ac = nodeDAO.getChildren(c); + Assert.assertNotNull(ac); + Assert.assertFalse(ac.nodes.isEmpty()); + Assert.assertEquals(3, ac.nodes.size()); + } + + @Test + public void testPutGetDeleteDataNode() { + log.info("TODO"); + } + + @Test + public void testPutGetDeleteLinkNode() { + log.info("TODO"); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java index 7b823e304..fe8533e75 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java @@ -168,7 +168,7 @@ public DataSource getDataSource() { return dataSource; } - SQLGenerator getSQLGenerator() { + public SQLGenerator getSQLGenerator() { checkInit(); return gen; } @@ -191,6 +191,7 @@ public Map getParams() { ret.put("jndiDataSourceName", String.class); ret.put("database", String.class); ret.put("schema", String.class); + ret.put("vosSchema", String.class); // optional ret.put(SQLGenerator.class.getName(), Class.class); return ret; } @@ -224,9 +225,10 @@ public void setConfig(Map config) { String database = (String) config.get("database"); String schema = (String) config.get("schema"); + String vosSchema = (String) config.get("vosSchema"); try { - Constructor ctor = genClass.getConstructor(String.class, String.class); - this.gen = (SQLGenerator) ctor.newInstance(database, schema); + Constructor ctor = genClass.getConstructor(String.class, String.class, String.class); + this.gen = (SQLGenerator) ctor.newInstance(database, schema, vosSchema); } catch (Exception ex) { throw new RuntimeException("failed to instantiate SQLGenerator: " + genClass.getName(), ex); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 15a34d7b7..172472704 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -98,6 +98,8 @@ import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StorageLocationEvent; import org.opencadc.inventory.StorageSite; +import org.opencadc.vospace.DeletedNodeEvent; +import org.opencadc.vospace.Node; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -115,6 +117,7 @@ public class SQLGenerator { protected final String database; // currently not used in SQL protected final String schema; // may be null + protected final String vosSchema; /** * Constructor. The database name is currently not used in any generated SQL; code assumes @@ -128,8 +131,13 @@ public class SQLGenerator { * @param schema schema name (may be null) */ public SQLGenerator(String database, String schema) { + this(database, schema, null); + } + + public SQLGenerator(String database, String schema, String vosSchema) { this.database = database; this.schema = schema; + this.vosSchema = vosSchema; init(); } @@ -202,6 +210,42 @@ protected void init() { "id" // last column is always PK }; this.columnMap.put(HarvestState.class, cols); + + // optional vospace + log.warn("vosSchema: " + vosSchema); + if (vosSchema != null) { + pref = vosSchema + "."; + tableMap.put(Node.class, pref + Node.class.getSimpleName()); + tableMap.put(DeletedNodeEvent.class, pref + DeletedNodeEvent.class.getSimpleName()); + + cols = new String[] { + "parentID", + "name", + "nodeType", + "ownerID", + "isPublic", + "isLocked", + "readOnlyGroups", + "readWriteGroups", + "properties", + "busyState", + "storageID", + "target", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(Node.class, cols); + + cols = new String[] { + "nodeType", + "storageID", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(DeletedNodeEvent.class, cols); + } } private static class ClassComp implements Comparator { @@ -217,8 +261,7 @@ public String getCurrentTimeSQL() { return "SELECT now()"; } - // test usage - String getTable(Class c) { + public String getTable(Class c) { return tableMap.get(c); } @@ -253,6 +296,13 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { if (HarvestState.class.equals(c)) { return new HarvestStateGet(); } + + if (Node.class.equals(c)) { + + } + if (DeletedNodeEvent.class.equals(c)) { + + } throw new UnsupportedOperationException("entity-get: " + c.getName()); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java new file mode 100644 index 000000000..093618755 --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -0,0 +1,113 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import java.net.URL; +import javax.sql.DataSource; +import org.apache.log4j.Logger; +import org.opencadc.inventory.db.version.InitDatabase; + +/** + * + * @author pdowler + */ +public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { + private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); + + public static final String MODEL_NAME = "vospace-inventory"; + public static final String MODEL_VERSION = "0.1"; + public static final String PREV_MODEL_VERSION = "n/a"; + //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; + + static String[] CREATE_SQL = new String[] { + "vos.ModelVersion.sql", + "vos.Node.sql", + "vos.DeletedNodeEvent.sql", + "inventory.permissions.sql" + }; + + static String[] UPGRADE_SQL = new String[] { + "inventory.permissions.sql" + }; + + public InitDatabaseVOS(DataSource ds, String database, String schema) { + super(ds, database, schema, MODEL_NAME, MODEL_VERSION, PREV_MODEL_VERSION); + for (String s : CREATE_SQL) { + createSQL.add(s); + } + for (String s : UPGRADE_SQL) { + upgradeSQL.add(s); + } + } + + @Override + protected URL findSQL(String fname) { + // SQL files are stored inside the jar file + return InitDatabase.class.getClassLoader().getResource(fname); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java new file mode 100644 index 000000000..1d16fa40a --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -0,0 +1,100 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2022. (c) 2022. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vospace.db; + +import java.util.UUID; +import org.apache.log4j.Logger; +import org.opencadc.inventory.Entity; +import org.opencadc.inventory.db.AbstractDAO; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.Node; + +/** + * + * @author pdowler + */ +public class NodeDAO extends AbstractDAO { + private static final Logger log = Logger.getLogger(NodeDAO.class); + + public NodeDAO() { + super(true); + } + + public Node get(UUID id) { + //return super.get(Node.class, id); + throw new UnsupportedOperationException(); + } + + public void put(Node n) { + throw new UnsupportedOperationException(); + } + + public ContainerNode getChildren(ContainerNode cn) { + throw new UnsupportedOperationException(); + } +} diff --git a/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql b/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql new file mode 100644 index 000000000..341a670e5 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql @@ -0,0 +1,17 @@ + +create table .DeletedNodeEvent ( + -- type is immutable + nodeType char(1) not null, + + -- support cleanup of obsolete artifacts + storageID varchar(512), + + lastModified timestamp not null, + metaChecksum varchar(136) not null, + id uuid not null primary key +); + + + +create index dne_lastmodified on .DeletedNodeEvent(lastModified); + diff --git a/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql b/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql new file mode 100644 index 000000000..ca9126697 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql @@ -0,0 +1,9 @@ + +create table .ModelVersion +( + model varchar(32) not null primary key, + version varchar(32) not null, + lastModified timestamp not null +) +; + diff --git a/cadc-inventory-db/src/main/resources/vos.Node.sql b/cadc-inventory-db/src/main/resources/vos.Node.sql new file mode 100644 index 000000000..7039b300a --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vos.Node.sql @@ -0,0 +1,31 @@ + +create table .Node ( + -- require a special root ID value but prevent bugs + parentID uuid not null, + name varchar(512) not null, + nodeType char(1) not null, + + ownerID varchar(256) not null, + isPublic boolean, + isLocked boolean, + readOnlyGroups text, + readWriteGroups text, + + -- store all props in a 2D array + properties text[][], + + -- DataNode + busyState boolean, + storageID varchar(512), + + -- linkNode + target text, + + lastModified timestamp not null, + metaChecksum varchar(136) not null, + id uuid not null primary key +); + +create unique index node_parent_child on .Node(parentID,name); + +create index node_lastmodified on .Node(lastModified); From ce8a3719b77477d887a8cf148af627484e493343 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 28 Mar 2023 17:56:56 -0700 Subject: [PATCH 002/186] comment out old row lock code --- .../java/org/opencadc/inventory/db/SQLGenerator.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 172472704..767f0bb88 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -320,12 +320,14 @@ public EntityList getEntityList(Class c) { throw new UnsupportedOperationException("entity-list: " + c.getName()); } - public EntityLock getEntityLock(Class c) { + /* + private EntityLock getEntityLock(Class c) { if (Artifact.class.equals(c)) { return new EntityLockImpl(c); } throw new UnsupportedOperationException("entity-list: " + c.getName()); } + */ public EntityGet getSkeletonEntityGet(Class c) { EntityGet ret = new SkeletonGet(c); @@ -361,6 +363,8 @@ public EntityDelete getEntityDelete(Class c) { return new EntityDeleteImpl(c); } + /* + // replaced by select-for-update private class EntityLockImpl implements EntityLock { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final Class entityClass; @@ -395,7 +399,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce return prep; } } - + */ + private class SkeletonGet implements EntityGet { private UUID id; private final Class entityClass; From b2d8b2c77386b7d93ba8758d48bb90eaa972a131 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 5 Apr 2023 07:49:41 -0700 Subject: [PATCH 003/186] NodeDAO work --- .../org/opencadc/vospace/db/NodeDAOTest.java | 4 +- .../opencadc/inventory/db/AbstractDAO.java | 2 +- .../org/opencadc/inventory/db/EntityGet.java | 2 +- .../org/opencadc/inventory/db/EntityLock.java | 2 +- .../org/opencadc/inventory/db/EntityPut.java | 2 +- .../opencadc/inventory/db/SQLGenerator.java | 165 ++++++++++++------ .../java/org/opencadc/vospace/db/NodeDAO.java | 12 +- 7 files changed, 125 insertions(+), 64 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index ccf475b6d..a1890cc69 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -83,7 +83,6 @@ import org.junit.Test; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.TestUtil; -import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.LinkNode; @@ -98,9 +97,10 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); - Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.DEBUG); Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace.db", Level.DEBUG); } NodeDAO nodeDAO; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java index fe8533e75..237668ba9 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java @@ -86,8 +86,8 @@ import javax.naming.NamingException; import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.persist.Entity; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java index c065b58a6..029ca037b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java @@ -68,7 +68,7 @@ package org.opencadc.inventory.db; import java.util.UUID; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java index 238518d78..d99e41742 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java @@ -68,7 +68,7 @@ package org.opencadc.inventory.db; import java.util.UUID; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java index 383347eba..e236c7a2b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java @@ -69,7 +69,7 @@ package org.opencadc.inventory.db; -import org.opencadc.inventory.Entity; +import org.opencadc.persist.Entity; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementCreator; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 424f1d51f..7a83254c1 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -92,13 +92,13 @@ import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.ObsoleteStorageLocation; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StorageLocationEvent; import org.opencadc.inventory.StorageSite; +import org.opencadc.persist.Entity; import org.opencadc.vospace.DeletedNodeEvent; import org.opencadc.vospace.Node; import org.springframework.dao.DataAccessException; @@ -113,8 +113,8 @@ public class SQLGenerator { private static final Logger log = Logger.getLogger(SQLGenerator.class); - private final Map tableMap = new TreeMap(new ClassComp()); - private final Map columnMap = new TreeMap(new ClassComp()); + private final Map tableMap = new TreeMap<>(new ClassComp()); + private final Map columnMap = new TreeMap<>(new ClassComp()); protected final String database; // currently not used in SQL protected final String schema; // may be null @@ -278,6 +278,13 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new StorageSiteGet(forUpdate); } + if (Node.class.equals(c)) { + return new NodeGet(forUpdate); + } + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodeGet(); + } + if (forUpdate) { throw new UnsupportedOperationException("entity-get + forUpdate: " + c.getSimpleName()); } @@ -298,12 +305,6 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new HarvestStateGet(); } - if (Node.class.equals(c)) { - - } - if (DeletedNodeEvent.class.equals(c)) { - - } throw new UnsupportedOperationException("entity-get: " + c.getName()); } @@ -357,6 +358,12 @@ public EntityPut getEntityPut(Class c, boolean update) { if (HarvestState.class.equals(c)) { return new HarvestStatePut(update); } + if (Node.class.equals(c)) { + return new NodePut(update); + } + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodePut(update); + } throw new UnsupportedOperationException("entity-put: " + c.getName()); } @@ -364,44 +371,6 @@ public EntityDelete getEntityDelete(Class c) { return new EntityDeleteImpl(c); } - /* - // replaced by select-for-update - private class EntityLockImpl implements EntityLock { - private final Calendar utc = Calendar.getInstance(DateUtil.UTC); - private final Class entityClass; - private UUID id; - - EntityLockImpl(Class entityClass) { - this.entityClass = entityClass; - } - - @Override - public void setID(UUID id) { - this.id = id; - } - - @Override - public void execute(JdbcTemplate jdbc) throws EntityNotFoundException { - int n = jdbc.update(this); - if (n == 0) { - throw new EntityNotFoundException("not found: " + id); - } - } - - @Override - public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { - String sql = getLockSQL(entityClass); - log.debug("EntityLockImpl: " + sql); - PreparedStatement prep = conn.prepareStatement(sql); - int col = 1; - prep.setObject(col++, id); - prep.setObject(col++, id); - - return prep; - } - } - */ - private class SkeletonGet implements EntityGet { private UUID id; private final Class entityClass; @@ -759,7 +728,6 @@ public ResourceIterator query(DataSource ds) { private class StorageSiteGet implements EntityGet { private UUID id; - private URI uri; private final boolean forUpdate; public StorageSiteGet(boolean forUpdate) { @@ -786,6 +754,9 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } else { throw new IllegalStateException("primary key is null"); } + if (forUpdate) { + sb.append(" FOR UPDATE"); + } String sql = sb.toString(); log.debug("StorageSiteGet: " + sql); PreparedStatement prep = conn.prepareStatement(sql); @@ -814,6 +785,45 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } + private class NodeGet implements EntityGet { + private UUID id; + private final boolean forUpdate; + + public NodeGet(boolean forUpdate) { + this.forUpdate = forUpdate; + } + + @Override + public void setID(UUID id) { + this.id = id; + } + + @Override + public Node execute(JdbcTemplate jdbc) { + return (Node) jdbc.query(this, new NodeExtractor()); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + StringBuilder sb = getSelectFromSQL(Node.class, false); + sb.append(" WHERE "); + if (id != null) { + String col = getKeyColumn(Node.class, true); + sb.append(col).append(" = ?"); + } else { + throw new IllegalStateException("primary key is null"); + } + if (forUpdate) { + sb.append(" FOR UPDATE"); + } + String sql = sb.toString(); + log.debug("Node: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + prep.setObject(1, id); + return prep; + } + } + private void safeSetString(PreparedStatement prep, int col, String value) throws SQLException { log.debug("safeSetString: " + col + " " + value); if (value != null) { @@ -1069,6 +1079,48 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } + private class NodePut implements EntityPut { + private final Calendar utc = Calendar.getInstance(DateUtil.UTC); + private final boolean update; + private Node value; + + NodePut(boolean update) { + this.update = update; + } + + @Override + public void setValue(Node value) { + this.value = value; + } + + @Override + public void execute(JdbcTemplate jdbc) { + jdbc.update(this); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + String sql = null; + if (update) { + sql = getUpdateSQL(Node.class); + + } else { + sql = getInsertSQL(Node.class); + } + log.debug("StorageSitePut: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + int col = 1; + + throw new UnsupportedOperationException("TODO"); + + //prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); + //prep.setString(col++, value.getMetaChecksum().toASCIIString()); + //prep.setObject(col++, value.getID()); + + //return prep; + } + } + private class EntityEventPut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; @@ -1520,4 +1572,19 @@ public StorageLocationEvent extractData(ResultSet rs) throws SQLException, DataA return ret; } } + + private class NodeExtractor implements ResultSetExtractor { + + final Calendar utc = Calendar.getInstance(DateUtil.UTC); + + @Override + public Node extractData(ResultSet rs) throws SQLException, DataAccessException { + if (!rs.next()) { + return null; + } + int col = 1; + throw new UnsupportedOperationException("TODO"); + } + + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 1d16fa40a..67c8fcee5 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -69,7 +69,6 @@ import java.util.UUID; import org.apache.log4j.Logger; -import org.opencadc.inventory.Entity; import org.opencadc.inventory.db.AbstractDAO; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; @@ -78,7 +77,7 @@ * * @author pdowler */ -public class NodeDAO extends AbstractDAO { +public class NodeDAO extends AbstractDAO { private static final Logger log = Logger.getLogger(NodeDAO.class); public NodeDAO() { @@ -86,12 +85,7 @@ public NodeDAO() { } public Node get(UUID id) { - //return super.get(Node.class, id); - throw new UnsupportedOperationException(); - } - - public void put(Node n) { - throw new UnsupportedOperationException(); + return super.get(Node.class, id); } public ContainerNode getChildren(ContainerNode cn) { From d244f50dd00db46b7026993e533b3c95fb9698ff Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 6 Apr 2023 15:18:05 -0700 Subject: [PATCH 004/186] basic function of NodeDAO: put, get, update, delete --- .../org/opencadc/vospace/db/NodeDAOTest.java | 400 ++++++++++++++++-- .../opencadc/inventory/db/SQLGenerator.java | 258 +++++++++-- .../java/org/opencadc/inventory/db/Util.java | 148 +++++-- .../java/org/opencadc/vospace/db/NodeDAO.java | 17 +- .../src/main/resources/vos.Node.sql | 7 +- 5 files changed, 724 insertions(+), 106 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index a1890cc69..71ec77702 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -70,8 +70,11 @@ import ca.nrc.cadc.db.ConnectionConfig; import ca.nrc.cadc.db.DBConfig; import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.util.Log4jInit; +import java.io.IOException; import java.net.URI; +import java.util.Date; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -87,6 +90,8 @@ import org.opencadc.vospace.DataNode; import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; /** * @@ -137,7 +142,7 @@ public void init_cleanup() throws Exception { log.info("clearing old content..."); SQLGenerator gen = nodeDAO.getSQLGenerator(); DataSource ds = nodeDAO.getDataSource(); - String sql = "delete from " + gen.getTable(Node.class); + String sql = "delete from " + gen.getTable(ContainerNode.class); log.info("pre-test cleanup: " + sql); ds.getConnection().createStatement().execute(sql); log.info("clearing old content... OK"); @@ -150,50 +155,393 @@ public void testGetByID() { Assert.assertNull(a); } - //@Test - public void testPutGetDeleteContainerNode() { - ContainerNode root = new ContainerNode("root", false); + @Test + public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + // put + ContainerNode orig = new ContainerNode("container-test", false); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + // get + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); + + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.isPublic = true; + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + Assert.assertTrue(updated instanceof ContainerNode); + ContainerNode uc = (ContainerNode) updated; + Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + // TODO: use get-by-path to find and remove the test node + + ContainerNode orig = new ContainerNode("container-test", false); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.inheritPermissions = false; + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g2")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g4,g5")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6-g7")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6.g7")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6_g7")); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6~g7")); + + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.properties.add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); + orig.properties.add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); + orig.properties.add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); + + Assert.assertTrue(a instanceof ContainerNode); + ContainerNode c = (ContainerNode) a; + Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.readOnlyGroup.clear(); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.clear(); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.clear(); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.inheritPermissions = true; + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + Assert.assertTrue(updated instanceof ContainerNode); + ContainerNode uc = (ContainerNode) updated; + Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteDataNode() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + DataNode orig = new DataNode("data-test", URI.create("cadc:vault/" + UUID.randomUUID())); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); - ContainerNode n = new ContainerNode("container-test", false); - n.parent = root; - nodeDAO.put(n); + Assert.assertTrue(a instanceof DataNode); + DataNode dn = (DataNode) a; + Assert.assertEquals(orig.getStorageID(), dn.getStorageID()); - Node a = nodeDAO.get(n.getID()); + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.readOnlyGroup.clear(); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.clear(); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.clear(); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + // don't change storageID + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + + Assert.assertTrue(a instanceof DataNode); + DataNode udn = (DataNode) updated; + Assert.assertEquals(orig.getStorageID(), udn.getStorageID()); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + // TODO: use get-by-path to find and remove the test node + + LinkNode orig = new LinkNode("data-test", URI.create("vos://opencadc.org~srv/path/to/something")); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(orig.ownerID, a.ownerID); + Assert.assertEquals(orig.isPublic, a.isPublic); + Assert.assertEquals(orig.isLocked, a.isLocked); + Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); + Assert.assertEquals(orig.properties, a.properties); + + Assert.assertTrue(a instanceof LinkNode); + LinkNode link = (LinkNode) a; + Assert.assertEquals(orig.getTarget(), link.getTarget()); + + // these are set in put + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + + // update + Thread.sleep(10L); + orig.isPublic = false; + orig.isLocked = true; + orig.readOnlyGroup.clear(); + orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); + orig.readWriteGroup.clear(); + orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); + orig.properties.clear(); + orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + // don't change target + nodeDAO.put(orig); + Node updated = nodeDAO.get(orig.getID()); + Assert.assertNotNull(updated); + Assert.assertEquals(orig.getID(), updated.getID()); + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertTrue(a.getLastModified().before(updated.getLastModified())); + Assert.assertNotEquals(a.getMetaChecksum(), updated.getMetaChecksum()); + + Assert.assertNull(updated.parent); // get-node-by-id: comes pack without parent + Assert.assertEquals(orig.getName(), updated.getName()); + Assert.assertEquals(orig.ownerID, updated.ownerID); + Assert.assertEquals(orig.isPublic, updated.isPublic); + Assert.assertEquals(orig.isLocked, updated.isLocked); + Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); + Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); + Assert.assertEquals(orig.properties, updated.properties); + + Assert.assertTrue(updated instanceof LinkNode); + LinkNode ulink = (LinkNode) updated; + Assert.assertEquals(orig.getTarget(), ulink.getTarget()); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + + @Test + public void testPutGetDeleteContainerNodeChildren() throws IOException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + + ContainerNode orig = new ContainerNode("container-test", false); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; - Assert.assertEquals(n.getName(), a.getName()); + // these are set in put - Assert.assertEquals(n.getMetaChecksum(), a.getMetaChecksum()); - Assert.assertEquals(n.getLastModified(), a.getLastModified()); - ContainerNode ac = nodeDAO.getChildren(c); - Assert.assertNotNull(ac); - Assert.assertTrue(ac.nodes.isEmpty()); + Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); + Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + //ResourceIterator emptyIter = nodeDAO.childIterator(orig); + //Assert.assertNotNull(emptyIter); + //Assert.assertFalse(emptyIter.hasNext()); + //emptyIter.close(); // add children ContainerNode cont = new ContainerNode("container1", false); cont.parent = c; - DataNode data = new DataNode("data1"); + cont.ownerID = c.ownerID; + DataNode data = new DataNode("data1", URI.create("cadc:vault/" + UUID.randomUUID())); data.parent = c; + data.ownerID = c.ownerID; LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); link.parent = c; + link.ownerID = c.ownerID; + log.info("put child: " + cont + " of " + cont.parent); nodeDAO.put(cont); + log.info("put child: " + data + " of " + data.parent); nodeDAO.put(data); + log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); - ac = nodeDAO.getChildren(c); - Assert.assertNotNull(ac); - Assert.assertFalse(ac.nodes.isEmpty()); - Assert.assertEquals(3, ac.nodes.size()); - } - - @Test - public void testPutGetDeleteDataNode() { - log.info("TODO"); + ResourceIterator iter = nodeDAO.childIterator(orig); + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + Node c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + Node c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + Node c3 = iter.next(); + + // default order: alpha + Assert.assertEquals(cont.getID(), c1.getID()); + Assert.assertEquals(cont.getName(), c1.getName()); + + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + Assert.assertEquals(link.getID(), c3.getID()); + Assert.assertEquals(link.getName(), c3.getName()); + + // depth first delete required? + try { + nodeDAO.delete(orig.getID()); + Assert.fail("expected IllegalStateException but successfully deleted non-empty container"); + } catch (IllegalStateException expected) { + log.info("caught expected: " + expected); + } + + nodeDAO.delete(cont.getID()); + nodeDAO.delete(data.getID()); + nodeDAO.delete(link.getID()); + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); } - @Test - public void testPutGetDeleteLinkNode() { + //@Test + public void testPutGetDeleteNodeProperties() { log.info("TODO"); } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 7a83254c1..3216ad3bd 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -72,6 +72,8 @@ import ca.nrc.cadc.util.StringUtil; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Array; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -84,6 +86,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.StringTokenizer; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; @@ -99,8 +102,13 @@ import org.opencadc.inventory.StorageLocationEvent; import org.opencadc.inventory.StorageSite; import org.opencadc.persist.Entity; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; import org.opencadc.vospace.DeletedNodeEvent; +import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -229,7 +237,8 @@ protected void init() { "readOnlyGroups", "readWriteGroups", "properties", - "busyState", + "inheritPermissions", + "busy", "storageID", "target", "lastModified", @@ -263,7 +272,27 @@ public String getCurrentTimeSQL() { } public String getTable(Class c) { - return tableMap.get(c); + Class targetClass = c; + String ret = tableMap.get(targetClass); + if (ret == null) { + // enable finding a common table that stores subclass instances + targetClass = targetClass.getSuperclass(); + ret = tableMap.get(targetClass); + } + log.debug("table: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + ret); + return ret; + } + + private String[] getColumns(Class c) { + Class targetClass = c; + String[] ret = columnMap.get(targetClass); + if (ret == null) { + // enable finding a common table that stores subclass instances + targetClass = targetClass.getSuperclass(); + ret = columnMap.get(targetClass); + } + log.debug("columns: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + (ret == null ? null : ret.length)); + return ret; } public EntityGet getEntityGet(Class c) { @@ -278,7 +307,7 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new StorageSiteGet(forUpdate); } - if (Node.class.equals(c)) { + if (Node.class.equals(c) || Node.class.isInstance(c)) { return new NodeGet(forUpdate); } if (DeletedNodeEvent.class.equals(c)) { @@ -322,15 +351,6 @@ public EntityList getEntityList(Class c) { throw new UnsupportedOperationException("entity-list: " + c.getName()); } - /* - private EntityLock getEntityLock(Class c) { - if (Artifact.class.equals(c)) { - return new EntityLockImpl(c); - } - throw new UnsupportedOperationException("entity-list: " + c.getName()); - } - */ - public EntityGet getSkeletonEntityGet(Class c) { EntityGet ret = new SkeletonGet(c); return ret; @@ -358,7 +378,7 @@ public EntityPut getEntityPut(Class c, boolean update) { if (HarvestState.class.equals(c)) { return new HarvestStatePut(update); } - if (Node.class.equals(c)) { + if (Node.class.isAssignableFrom(c)) { return new NodePut(update); } if (DeletedNodeEvent.class.equals(c)) { @@ -467,7 +487,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String col = getKeyColumn(ObsoleteStorageLocation.class, true); sb.append(col).append(" = ?"); } else { - String[] cols = columnMap.get(ObsoleteStorageLocation.class); + String[] cols = getColumns(ObsoleteStorageLocation.class); sb.append(cols[0]).append(" = ?"); sb.append(" AND "); if (loc.storageBucket != null) { @@ -524,7 +544,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String col = getKeyColumn(HarvestState.class, true); sb.append(col).append(" = ?"); } else { - String[] cols = columnMap.get(HarvestState.class); + String[] cols = getColumns(HarvestState.class); sb.append(cols[0]).append(" = ?"); sb.append(" AND "); sb.append(cols[1]).append(" = ?"); @@ -824,6 +844,23 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } + private void safeSetBoolean(PreparedStatement prep, int col, Boolean value) throws SQLException { + log.debug("safeSetBoolean: " + col + " " + value); + if (value != null) { + prep.setBoolean(col, value); + } else { + prep.setNull(col, Types.BOOLEAN); + } + } + + private void safeSetString(PreparedStatement prep, int col, URI value) throws SQLException { + String v = null; + if (value != null) { + v = value.toASCIIString(); + } + safeSetString(prep, col, v); + } + private void safeSetString(PreparedStatement prep, int col, String value) throws SQLException { log.debug("safeSetString: " + col + " " + value); if (value != null) { @@ -851,6 +888,24 @@ private void safeSetTimestamp(PreparedStatement prep, int col, Timestamp value, } } + private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException { + + if (values != null && !values.isEmpty()) { + log.debug("safeSetArray: " + col + " " + values.size()); + String[] array1d = new String[values.size()]; + int i = 0; + for (URI u : values) { + array1d[i] = u.toASCIIString(); + i++; + } + java.sql.Array arr = prep.getConnection().createArrayOf("text", array1d); + prep.setObject(col, arr); + } else { + log.debug("safeSetArray: " + col + " null"); + prep.setNull(col, Types.ARRAY); + } + } + private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws SQLException { if (value != null) { @@ -863,6 +918,25 @@ private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws } } + private void safeSetProps(PreparedStatement prep, int col, Set values) throws SQLException { + + if (values != null && !values.isEmpty()) { + log.debug("safeSetProps: " + col + " " + values.size()); + String[][] array2d = new String[values.size()][2]; // TODO: w-h or h-w?? + int i = 0; + for (NodeProperty np : values) { + array2d[i][0] = np.getKey().toASCIIString(); + array2d[i][1] = np.getValue(); + i++; + } + java.sql.Array arr = prep.getConnection().createArrayOf("text", array2d); + prep.setObject(col, arr); + } else { + log.debug("safeSetProps: " + col + " = null"); + prep.setNull(col, Types.ARRAY); + } + } + private class ArtifactPut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; @@ -917,8 +991,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce safeSetString(prep, col++, value.storageLocation.getStorageID().toASCIIString()); safeSetString(prep, col++, value.storageLocation.storageBucket); } else { - safeSetString(prep, col++, null); // storageLocation.storageID - safeSetString(prep, col++, null); // storageLocation.storageBucket + safeSetString(prep, col++, (URI) null); // storageLocation.storageID + safeSetString(prep, col++, (URI) null); // storageLocation.storageBucket } safeSetTimestamp(prep, col++, new Timestamp(value.getLastModified().getTime()), utc); @@ -1013,7 +1087,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce if (value.getLocation().storageBucket != null) { safeSetString(prep, col++, value.getLocation().storageBucket); } else { - safeSetString(prep, col++, null); + safeSetString(prep, col++, (String) null); } prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); @@ -1107,17 +1181,54 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } else { sql = getInsertSQL(Node.class); } - log.debug("StorageSitePut: " + sql); + log.debug("NodePut: " + sql); PreparedStatement prep = conn.prepareStatement(sql); int col = 1; - throw new UnsupportedOperationException("TODO"); + if (value.parent == null) { + throw new RuntimeException("BUG: cannot put Node without a parent: " + value); + } + prep.setObject(col++, value.parent.getID()); + prep.setString(col++, value.getName()); + prep.setString(col++, value.getClass().getSimpleName().substring(0, 1)); // HACK + if (value.ownerID == null) { + throw new RuntimeException("BUG: cannot put Node without an ownerID: " + value); + } + prep.setString(col++, value.ownerID.toString()); + safeSetBoolean(prep, col++, value.isPublic); + safeSetBoolean(prep, col++, value.isLocked); + safeSetArray(prep, col++, value.readOnlyGroup); + safeSetArray(prep, col++, value.readWriteGroup); + safeSetProps(prep, col++, value.properties); + if (value instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) value; + safeSetBoolean(prep, col++, cn.inheritPermissions); + } else { + safeSetBoolean(prep, col++, null); + } + if (value instanceof DataNode) { + DataNode dn = (DataNode) value; + if (dn.getStorageID() == null) { + throw new RuntimeException("BUG: cannot put DataNode without a storageID: " + value); + } + safeSetBoolean(prep, col++, dn.busy); + safeSetString(prep, col++, dn.getStorageID()); + } else { + safeSetBoolean(prep, col++, null); + safeSetString(prep, col++, (URI) null); + } + if (value instanceof LinkNode) { + LinkNode ln = (LinkNode) value; + prep.setString(col++, ln.getTarget().toASCIIString()); + } else { + safeSetString(prep, col++, (URI) null); + } - //prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); - //prep.setString(col++, value.getMetaChecksum().toASCIIString()); - //prep.setObject(col++, value.getID()); + prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); + prep.setString(col++, value.getMetaChecksum().toASCIIString()); + prep.setObject(col++, value.getID()); - //return prep; + return prep; } } @@ -1190,11 +1301,13 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } private StringBuilder getSelectFromSQL(Class c, boolean entityCols) { - String tab = tableMap.get(c); - String[] cols = columnMap.get(c); + String tab = getTable(c); + Class targetClass = c; if (entityCols) { - cols = columnMap.get(Entity.class); + targetClass = Entity.class; } + String[] cols = getColumns(targetClass); + if (tab == null || cols == null) { throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName()); } @@ -1213,21 +1326,13 @@ private StringBuilder getSelectFromSQL(Class c, boolean entityCols) { return sb; } - private String getLockSQL(Class c) { - StringBuilder sb = new StringBuilder(); - String pk = getKeyColumn(c, true); - sb.append("UPDATE "); - sb.append(tableMap.get(c)); - sb.append(" SET ").append(pk).append(" = ? WHERE ").append(pk).append(" = ?"); - return sb.toString(); - } - private String getUpdateSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("UPDATE "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" SET "); - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); for (int i = 0; i < cols.length - 1; i++) { // PK is last if (i > 0) { sb.append(","); @@ -1244,10 +1349,11 @@ private String getUpdateSQL(Class c) { private String getInsertSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("INSERT INTO "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" ("); - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); for (int i = 0; i < cols.length; i++) { if (i > 0) { sb.append(","); @@ -1268,14 +1374,15 @@ private String getInsertSQL(Class c) { private String getDeleteSQL(Class c) { StringBuilder sb = new StringBuilder(); + String tab = getTable(c); sb.append("DELETE FROM "); - sb.append(tableMap.get(c)); + sb.append(tab); sb.append(" WHERE id = ?"); return sb.toString(); } private String getKeyColumn(Class c, boolean pk) { - String[] cols = columnMap.get(c); + String[] cols = getColumns(c); if (cols == null) { throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName()); } @@ -1583,8 +1690,73 @@ public Node extractData(ResultSet rs) throws SQLException, DataAccessException { return null; } int col = 1; - throw new UnsupportedOperationException("TODO"); + /* + "parentID", + "name", + "nodeType", + "ownerID", + "isPublic", + "isLocked", + "readOnlyGroups", + "readWriteGroups", + "properties", + + "inheritPermissions", + "busy", + "storageID", + "target", + + "lastModified", + "metaChecksum", + "id" // last column is always PK + */ + col++; //UUID unused = Util.getUUID(rs, col++); + final String name = rs.getString(col++); + final String nodeType = rs.getString(col++); + final String ownerID = rs.getString(col++); + final Boolean isPublic = Util.getBoolean(rs, col++); + final Boolean isLocked = Util.getBoolean(rs, col++); + final String rawROG = rs.getString(col++); + final String rawRWG = rs.getString(col++); + final String rawProps = rs.getString(col++); + final Boolean inheritPermissions = Util.getBoolean(rs, col++); + final Boolean busy = Util.getBoolean(rs, col++); + final URI storageID = Util.getURI(rs, col++); + final URI linkTarget = Util.getURI(rs, col++); + final Date lastModified = Util.getDate(rs, col++, utc); + final URI metaChecksum = Util.getURI(rs, col++); + final UUID id = Util.getUUID(rs, col++); + + Node ret; + if (nodeType.equals("C")) { + ret = new ContainerNode(id, name, inheritPermissions); + } else if (nodeType.equals("D")) { + ret = new DataNode(id, name, storageID); + } else if (nodeType.equals("L")) { + ret = new LinkNode(id, name, linkTarget); + } else { + throw new RuntimeException("BUG: unexpected node type code: " + nodeType); + } + ret.ownerID = ownerID; + ret.isPublic = isPublic; + ret.isLocked = isLocked; + + if (rawROG != null) { + Util.parseArrayURI(rawROG, ret.readOnlyGroup); + } + if (rawRWG != null) { + Util.parseArrayURI(rawRWG, ret.readWriteGroup); + } + if (rawProps != null) { + Util.parseArrayProps(rawProps, ret.properties); + } + + InventoryUtil.assignLastModified(ret, lastModified); + InventoryUtil.assignMetaChecksum(ret, metaChecksum); + + return ret; } + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java index 404841ff8..cd20020bf 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java @@ -75,10 +75,15 @@ import java.sql.Array; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.StringTokenizer; import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.vospace.NodeProperty; /** * @@ -378,39 +383,114 @@ public static byte[] getByteArray(ResultSet rs, int col) throw new UnsupportedOperationException("converting " + o.getClass().getName() + " " + o + " to byte[]"); } - /* - * public static int[] getIntArray(ResultSet rs, int col) - * throws SQLException - * { - * Object o = rs.getObject(col); - * return toIntArray(o); - * } - * - * static int[] toIntArray(Object o) - * throws SQLException - * { - * if (o == null) - * return null; - * if (o instanceof Array) - * { - * Array a = (Array) o; - * o = a.getArray(); - * } - * if (o instanceof int[]) - * return (int[]) o; - * if (o instanceof byte[]) - * return CaomUtil.decodeIntArray((byte[]) o); - * if (o instanceof Integer[]) - * { - * Integer[] ia = (Integer[]) o; - * int[] ret = new int[ia.length]; - * for (int i=0; i dest) { + // postgresql 1D array: {a,"b,c"} + if (val == null || val.isEmpty()) { + return; + } + // GroupURI names can contain alphanumeric,comma,dash,dot,underscore,~ + // PG quotes them if comma is present (eg in the group name) + char delim = '"'; + int i = 0; + int j = val.indexOf(delim); + while (j != -1) { + String token = val.substring(i, j); + //log.warn("token: " + i + "," + j + " " + token); + i = j + 1; + j = val.indexOf(delim, i); + + handleToken(token, dest); + } + String token = val.substring(i); + //log.warn("token: " + i + " " + token); + handleToken(token, dest); + } + + private static void handleToken(String token, Set dest) { + if (token.startsWith("ivo://")) { + dest.add(URI.create(token)); + } else { + StringTokenizer st = new StringTokenizer(token, "{,}"); + while (st.hasMoreTokens()) { + String s = st.nextToken(); + dest.add(URI.create(s)); + } + } + } + + public static void parseArrayProps(String val, Set dest) { + // postgresql 2D array: {{a,b},{c,d}} + if (val == null || val.isEmpty()) { + return; + } + char open = '{'; + char close = '}'; + char quote = '"'; + int i = val.indexOf(open); + int j = val.lastIndexOf(close); + if (j > i) { + val = val.substring(i + 1, j); + } + i = val.indexOf(open); + j = val.indexOf(close, i + 1); + int k = 0; + while (i != -1 && j != -1 && k++ < 20) { + String t1 = val.substring(i + 1, j); + //log.warn("\tt1: " + i + "," + j + " " + t1); + handleProp(t1, dest); + + if (i != -1 && j > 0) { + i = val.indexOf(open, j); + j = val.indexOf(close, i + 1); + // look ahead for quotes + int q = val.indexOf(quote, i); + //log.warn("i=" + i + " j=" + j + " q=" + q); + if (q != -1 && q < j) { + int cq = val.indexOf(quote, q + 1); + j = val.indexOf(close, cq); + //log.warn("\tcq=" + cq + " j=" + j); + } + } + } + } + private static void handleProp(String token, Set dest) { + int q = token.indexOf('"'); + int cq = -1; + if (q == -1) { + q = Integer.MAX_VALUE; + } else { + cq = token.indexOf('"', q + 1); + } + int c = token.indexOf(','); + + String key; + int split = c; + if (c < q) { + // key + key = token.substring(0, c); + } else { + // "key" + key = token.substring(q + 1, cq); + split = cq + 1; + } + //log.warn("\tkey: " + key); + + q = token.indexOf('"', split + 1); + cq = -1; + if (q == -1) { + q = Integer.MAX_VALUE; + } else { + cq = token.indexOf('"', q + 1); + } + String val; + if (token.length() < q) { + val = token.substring(split + 1); + } else { + val = token.substring(q + 1, cq); + } + //log.warn("\tval: " + val); + + dest.add(new NodeProperty(URI.create(key), val)); + } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 67c8fcee5..7c1155be7 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -67,11 +67,13 @@ package org.opencadc.vospace.db; +import ca.nrc.cadc.io.ResourceIterator; import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.db.AbstractDAO; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; +import org.opencadc.vospace.VOSURI; /** * @@ -84,11 +86,24 @@ public NodeDAO() { super(true); } + @Override + public void put(Node val) { + super.put(val); + } + public Node get(UUID id) { return super.get(Node.class, id); } - public ContainerNode getChildren(ContainerNode cn) { + public void delete(UUID id) { + super.delete(Node.class, id); + } + + public ResourceIterator childIterator(ContainerNode parent) { + return childIterator(parent, null, null); + } + + public ResourceIterator childIterator(ContainerNode parent, VOSURI start, Integer limit) { throw new UnsupportedOperationException(); } } diff --git a/cadc-inventory-db/src/main/resources/vos.Node.sql b/cadc-inventory-db/src/main/resources/vos.Node.sql index 7039b300a..e52a5ed8c 100644 --- a/cadc-inventory-db/src/main/resources/vos.Node.sql +++ b/cadc-inventory-db/src/main/resources/vos.Node.sql @@ -14,11 +14,14 @@ create table .Node ( -- store all props in a 2D array properties text[][], + -- ContainerNode + inheritPermissions boolean, + -- DataNode - busyState boolean, + busy boolean, storageID varchar(512), - -- linkNode + -- LinkNode target text, lastModified timestamp not null, From 7156929f4aaa6e672cb7bf35d0719460071e4906 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 6 Apr 2023 15:19:24 -0700 Subject: [PATCH 005/186] unit test code for postgresql array parsing --- .../org/opencadc/inventory/db/UtilTest.java | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java diff --git a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java new file mode 100644 index 000000000..adf255402 --- /dev/null +++ b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java @@ -0,0 +1,160 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.inventory.db; + +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import java.util.Set; +import java.util.TreeSet; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Test; +import org.opencadc.vospace.NodeProperty; + +/** + * + * @author pdowler + */ +public class UtilTest { + private static final Logger log = Logger.getLogger(UtilTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + } + + public UtilTest() { + } + + @Test + public void testParseArrayURI() throws Exception { + + String str = "{ivo://opencadc.org/gms?g3," + + "\"ivo://opencadc.org/gms?g4,g5\"," + + "ivo://opencadc.org/gms?g6-g7," + + "ivo://opencadc.org/gms?g6.g7," + + "ivo://opencadc.org/gms?g6_g7," + + "ivo://opencadc.org/gms?g6~g7}"; + + Set dest = new TreeSet<>(); + Util.parseArrayURI(str, dest); + for (URI u : dest) { + log.info("uri: " + u); + } + } + + @Test + public void testParseArrayNodeProperty() throws Exception { + + String str = ""; + Set dest = new TreeSet<>(); + + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,stuff}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"}," + + "{ivo://ivoa.net/vospace/core#type,text/plain}}"; + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + + str = "{{custom:prop,\"spaces in value\"}," + + "{ivo://ivoa.net/vospace/core#length,123}," + + "{ivo://ivoa.net/vospace/core#type,text/plain}," + + "{\"sketchy:a,b\",comma-in-uri}," + + "{sketchy:funny,\"value,with,{delims}\"}}"; + + dest.clear(); + log.info("raw:\n" + str + "\n"); + Util.parseArrayProps(str, dest); + for (NodeProperty p : dest) { + log.info("prop: " + p.getKey() + " = " + p.getValue()); + } + } +} From 2cdd2de31511dd82a9caad1dcb870f2c4ba6f427 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 13 Apr 2023 13:45:05 -0700 Subject: [PATCH 006/186] incomplete node iterator --- .../org/opencadc/vospace/db/NodeDAOTest.java | 86 ++++-- .../inventory/db/EntityIteratorQuery.java | 4 +- .../opencadc/inventory/db/SQLGenerator.java | 281 +++++++++++++----- .../java/org/opencadc/vospace/db/NodeDAO.java | 51 +++- 4 files changed, 325 insertions(+), 97 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 71ec77702..0f4a6a5fe 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -74,7 +74,6 @@ import ca.nrc.cadc.util.Log4jInit; import java.io.IOException; import java.net.URI; -import java.util.Date; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -116,7 +115,7 @@ public NodeDAOTest() throws Exception { ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest", cc); - Map config = new TreeMap(); + Map config = new TreeMap<>(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); config.put("database", TestUtil.DATABASE); @@ -149,12 +148,24 @@ public void init_cleanup() throws Exception { } @Test - public void testGetByID() { + public void testGetByID_NotFound() { UUID id = UUID.randomUUID(); Node a = nodeDAO.get(id); Assert.assertNull(a); } + @Test + public void testGetByPath_NotFound() { + ContainerNode parent = new ContainerNode("not-found", false); + Node a = nodeDAO.get(parent, "not-found"); + Assert.assertNull(a); + + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root", false); + a = nodeDAO.get(root, "not-found"); + Assert.assertNull(a); + } + @Test public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { UUID rootID = new UUID(0L, 0L); @@ -166,13 +177,22 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { orig.ownerID = "the-owner"; nodeDAO.put(orig); - // get + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -251,12 +271,22 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.properties.add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); nodeDAO.put(orig); + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -325,12 +355,22 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); nodeDAO.put(orig); + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -401,12 +441,22 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); nodeDAO.put(orig); + // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + // get-by-path + Node aa = nodeDAO.get(root, orig.getName()); + Assert.assertNotNull(aa); + log.info("found: " + aa.getID() + " aka " + aa); + Assert.assertEquals(orig.getID(), aa.getID()); + Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertNotNull(aa.parent); + Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); Assert.assertEquals(orig.ownerID, a.ownerID); @@ -478,26 +528,25 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { Assert.assertEquals(orig.getName(), a.getName()); Assert.assertTrue(a instanceof ContainerNode); - ContainerNode c = (ContainerNode) a; // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); - //ResourceIterator emptyIter = nodeDAO.childIterator(orig); - //Assert.assertNotNull(emptyIter); - //Assert.assertFalse(emptyIter.hasNext()); - //emptyIter.close(); + ResourceIterator emptyIter = nodeDAO.iterator(orig); + Assert.assertNotNull(emptyIter); + Assert.assertFalse(emptyIter.hasNext()); + emptyIter.close(); // add children ContainerNode cont = new ContainerNode("container1", false); - cont.parent = c; - cont.ownerID = c.ownerID; + cont.parent = orig; + cont.ownerID = orig.ownerID; DataNode data = new DataNode("data1", URI.create("cadc:vault/" + UUID.randomUUID())); - data.parent = c; - data.ownerID = c.ownerID; + data.parent = orig; + data.ownerID = orig.ownerID; LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); - link.parent = c; - link.ownerID = c.ownerID; + link.parent = orig; + link.ownerID = orig.ownerID; log.info("put child: " + cont + " of " + cont.parent); nodeDAO.put(cont); log.info("put child: " + data + " of " + data.parent); @@ -505,7 +554,7 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); - ResourceIterator iter = nodeDAO.childIterator(orig); + ResourceIterator iter = nodeDAO.iterator(orig); Assert.assertNotNull(iter); Assert.assertTrue(iter.hasNext()); Node c1 = iter.next(); @@ -539,9 +588,4 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { Node gone = nodeDAO.get(orig.getID()); Assert.assertNull(gone); } - - //@Test - public void testPutGetDeleteNodeProperties() { - log.info("TODO"); - } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java index 782de3129..04e065bb8 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java @@ -67,7 +67,7 @@ package org.opencadc.inventory.db; -import java.util.Iterator; +import ca.nrc.cadc.io.ResourceIterator; import javax.sql.DataSource; /** @@ -76,5 +76,5 @@ * @param entity subclass */ public interface EntityIteratorQuery { - Iterator query(DataSource ds); + ResourceIterator query(DataSource ds); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 3216ad3bd..d9c45bc93 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -83,6 +83,7 @@ import java.util.Calendar; import java.util.Comparator; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -341,7 +342,10 @@ public EntityIteratorQuery getEntityIteratorQuery(Class c) { if (Artifact.class.equals(c)) { return new ArtifactIteratorQuery(); } - throw new UnsupportedOperationException("entity-list: " + c.getName()); + if (Node.class.equals(c)) { + return new NodeIteratorQuery(); + } + throw new UnsupportedOperationException("entity-iterator: " + c.getName()); } public EntityList getEntityList(Class c) { @@ -805,8 +809,10 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } - private class NodeGet implements EntityGet { + public class NodeGet implements EntityGet { private UUID id; + private ContainerNode parent; + private String name; private final boolean forUpdate; public NodeGet(boolean forUpdate) { @@ -818,9 +824,14 @@ public void setID(UUID id) { this.id = id; } + public void setPath(ContainerNode parent, String name) { + this.parent = parent; + this.name = name; + } + @Override public Node execute(JdbcTemplate jdbc) { - return (Node) jdbc.query(this, new NodeExtractor()); + return (Node) jdbc.query(this, new NodeExtractor(parent)); } @Override @@ -830,6 +841,11 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce if (id != null) { String col = getKeyColumn(Node.class, true); sb.append(col).append(" = ?"); + } else if (parent != null && name != null) { + String pidCol = "parentID"; + String nameCol = "name"; + // TODO: better way to get column names? + sb.append(pidCol).append(" = ? and ").append(nameCol).append(" = ?"); } else { throw new IllegalStateException("primary key is null"); } @@ -839,11 +855,83 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce String sql = sb.toString(); log.debug("Node: " + sql); PreparedStatement prep = conn.prepareStatement(sql); - prep.setObject(1, id); + if (id != null) { + prep.setObject(1, id); + } else { + prep.setObject(1, parent.getID()); + prep.setObject(2, name); + } + return prep; } } + public class NodeIteratorQuery implements EntityIteratorQuery { + private ContainerNode parent; + private String start; + private Integer limit; + + public NodeIteratorQuery() { + } + + public void setParent(ContainerNode parent) { + this.parent = parent; + } + + public void setStart(String start) { + this.start = start; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + @Override + public ResourceIterator query(DataSource ds) { + if (parent == null) { + throw new RuntimeException("BUG: cannot query for children with parent==null"); + } + + StringBuilder sb = getSelectFromSQL(Node.class, false); + sb.append(" WHERE parentID = ?"); + if (start != null) { + sb.append(" AND ? <= name"); + } + sb.append(" ORDER BY name ASC"); + if (limit != null) { + sb.append(" LIMIT ?"); + } + + String sql = sb.toString(); + log.debug("sql: " + sql); + + try { + Connection con = ds.getConnection(); + log.debug("NodeIteratorQuery: setAutoCommit(false)"); + con.setAutoCommit(false); + // defaults for options: ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY + PreparedStatement ps = con.prepareStatement(sql); + ps.setFetchSize(1000); + ps.setFetchDirection(ResultSet.FETCH_FORWARD); + int col = 1; + + ps.setObject(1, parent.getID()); + if (start != null) { + ps.setString(col++, start); + } + if (limit != null) { + ps.setInt(col++, limit); + } + ResultSet rs = ps.executeQuery(); + + return new NodeResultSetIterator(con, rs, parent); + } catch (SQLException ex) { + throw new RuntimeException("BUG: artifact iterator query failed", ex); + } + + } + } + private void safeSetBoolean(PreparedStatement prep, int col, Boolean value) throws SQLException { log.debug("safeSetBoolean: " + col + " " + value); if (value != null) { @@ -1495,6 +1583,63 @@ public Artifact next() { } } + private class NodeResultSetIterator implements ResourceIterator { + final Calendar utc = Calendar.getInstance(DateUtil.UTC); + private final Connection con; + private final ResultSet rs; + boolean hasRow; + + ContainerNode parent; + + public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) { + this.con = con; + this.rs = rs; + this.parent = parent; + } + + @Override + public void close() throws IOException { + if (hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); + try { + con.setAutoCommit(true); + hasRow = false; + } catch (SQLException ex) { + throw new RuntimeException("BUG: node list query failed during close()", ex); + } + } + } + + @Override + public boolean hasNext() { + return hasRow; + } + + @Override + public Node next() { + try { + Node ret = mapRowToNode(rs, utc, parent); + hasRow = rs.next(); + if (!hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); + con.setAutoCommit(true); + } + return ret; + } catch (Exception ex) { + if (hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); + try { + close(); + hasRow = false; + } catch (IOException unexpected) { + log.debug("BUG: unexpected IOException from close", unexpected); + } + } + throw new RuntimeException("BUG: node list query failed while iterating", ex); + } + } + } + private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLException { int col = 1; final URI uri = Util.getURI(rs, col++); @@ -1528,6 +1673,62 @@ private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLExceptio return a; } + private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) throws SQLException { + int col = 1; + final UUID parentID = Util.getUUID(rs, col++); + final String name = rs.getString(col++); + final String nodeType = rs.getString(col++); + final String ownerID = rs.getString(col++); + final Boolean isPublic = Util.getBoolean(rs, col++); + final Boolean isLocked = Util.getBoolean(rs, col++); + final String rawROG = rs.getString(col++); + final String rawRWG = rs.getString(col++); + final String rawProps = rs.getString(col++); + final Boolean inheritPermissions = Util.getBoolean(rs, col++); + final Boolean busy = Util.getBoolean(rs, col++); + final URI storageID = Util.getURI(rs, col++); + final URI linkTarget = Util.getURI(rs, col++); + final Date lastModified = Util.getDate(rs, col++, utc); + final URI metaChecksum = Util.getURI(rs, col++); + final UUID id = Util.getUUID(rs, col++); + + Node ret; + if (nodeType.equals("C")) { + ret = new ContainerNode(id, name, inheritPermissions); + } else if (nodeType.equals("D")) { + ret = new DataNode(id, name, storageID); + } else if (nodeType.equals("L")) { + ret = new LinkNode(id, name, linkTarget); + } else { + throw new RuntimeException("BUG: unexpected node type code: " + nodeType); + } + ret.ownerID = ownerID; + ret.isPublic = isPublic; + ret.isLocked = isLocked; + + if (rawROG != null) { + Util.parseArrayURI(rawROG, ret.readOnlyGroup); + } + if (rawRWG != null) { + Util.parseArrayURI(rawRWG, ret.readWriteGroup); + } + if (rawProps != null) { + Util.parseArrayProps(rawProps, ret.properties); + } + + InventoryUtil.assignLastModified(ret, lastModified); + InventoryUtil.assignMetaChecksum(ret, metaChecksum); + + if (parent != null) { + if (!parent.getID().equals(parentID)) { + throw new RuntimeException("BUG: expected parentID=" + parent.getID() + " but got: " + parentID); + } + ret.parent = parent; + } + + return ret; + } + private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { final Calendar utc = Calendar.getInstance(DateUtil.UTC); @@ -1681,80 +1882,20 @@ public StorageLocationEvent extractData(ResultSet rs) throws SQLException, DataA } private class NodeExtractor implements ResultSetExtractor { - + private ContainerNode parent; final Calendar utc = Calendar.getInstance(DateUtil.UTC); + + public NodeExtractor(ContainerNode parent) { + this.parent = parent; // optional + } @Override public Node extractData(ResultSet rs) throws SQLException, DataAccessException { if (!rs.next()) { return null; } - int col = 1; - /* - "parentID", - "name", - "nodeType", - "ownerID", - "isPublic", - "isLocked", - "readOnlyGroups", - "readWriteGroups", - "properties", - - "inheritPermissions", - "busy", - "storageID", - "target", - - "lastModified", - "metaChecksum", - "id" // last column is always PK - */ - col++; //UUID unused = Util.getUUID(rs, col++); - final String name = rs.getString(col++); - final String nodeType = rs.getString(col++); - final String ownerID = rs.getString(col++); - final Boolean isPublic = Util.getBoolean(rs, col++); - final Boolean isLocked = Util.getBoolean(rs, col++); - final String rawROG = rs.getString(col++); - final String rawRWG = rs.getString(col++); - final String rawProps = rs.getString(col++); - final Boolean inheritPermissions = Util.getBoolean(rs, col++); - final Boolean busy = Util.getBoolean(rs, col++); - final URI storageID = Util.getURI(rs, col++); - final URI linkTarget = Util.getURI(rs, col++); - final Date lastModified = Util.getDate(rs, col++, utc); - final URI metaChecksum = Util.getURI(rs, col++); - final UUID id = Util.getUUID(rs, col++); - - Node ret; - if (nodeType.equals("C")) { - ret = new ContainerNode(id, name, inheritPermissions); - } else if (nodeType.equals("D")) { - ret = new DataNode(id, name, storageID); - } else if (nodeType.equals("L")) { - ret = new LinkNode(id, name, linkTarget); - } else { - throw new RuntimeException("BUG: unexpected node type code: " + nodeType); - } - ret.ownerID = ownerID; - ret.isPublic = isPublic; - ret.isLocked = isLocked; - - if (rawROG != null) { - Util.parseArrayURI(rawROG, ret.readOnlyGroup); - } - if (rawRWG != null) { - Util.parseArrayURI(rawRWG, ret.readWriteGroup); - } - if (rawProps != null) { - Util.parseArrayProps(rawProps, ret.properties); - } - InventoryUtil.assignLastModified(ret, lastModified); - InventoryUtil.assignMetaChecksum(ret, metaChecksum); - - return ret; + return mapRowToNode(rs, utc, parent); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 7c1155be7..756651843 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -70,10 +70,14 @@ import ca.nrc.cadc.io.ResourceIterator; import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; import org.opencadc.inventory.db.AbstractDAO; +import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.VOSURI; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.jdbc.core.JdbcTemplate; /** * @@ -95,15 +99,54 @@ public Node get(UUID id) { return super.get(Node.class, id); } + public Node get(ContainerNode parent, String name) { + checkInit(); + log.debug("GET: " + parent.getID() + " + " + name); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + SQLGenerator.NodeGet get = (SQLGenerator.NodeGet) gen.getEntityGet(Node.class); + get.setPath(parent, name); + return get.execute(jdbc); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("GET: " + parent.getID() + " + " + name + " " + dt + "ms"); + } + throw new RuntimeException("BUG: handleInternalFail did not throw"); + } + public void delete(UUID id) { super.delete(Node.class, id); } - public ResourceIterator childIterator(ContainerNode parent) { - return childIterator(parent, null, null); + public ResourceIterator iterator(ContainerNode parent) { + return NodeDAO.this.iterator(parent, null, null); } - public ResourceIterator childIterator(ContainerNode parent, VOSURI start, Integer limit) { - throw new UnsupportedOperationException(); + public ResourceIterator iterator(ContainerNode parent, String start, Integer limit) { + if (parent == null) { + throw new IllegalArgumentException("childIterator: parent cannot be null"); + } + log.debug("iterator: " + parent.getID()); + + checkInit(); + long t = System.currentTimeMillis(); + + try { + SQLGenerator.NodeIteratorQuery iter = (SQLGenerator.NodeIteratorQuery) gen.getEntityIteratorQuery(Node.class); + iter.setParent(parent); + iter.setStart(start); + iter.setLimit(limit); + return iter.query(dataSource); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("iterator: " + parent.getID() + " " + dt + "ms"); + } + throw new RuntimeException("BUG: should be unreachable"); } } From 7905cc0c88068a63709c48200479881be4d07a0e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 14 Apr 2023 13:05:05 -0700 Subject: [PATCH 007/186] working node child iterator --- .../org/opencadc/vospace/db/NodeDAOTest.java | 48 +++++++++++-------- .../opencadc/inventory/db/SQLGenerator.java | 11 ++++- .../java/org/opencadc/vospace/db/NodeDAO.java | 1 + 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 0f4a6a5fe..62374a3f8 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -512,7 +512,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { } @Test - public void testPutGetDeleteContainerNodeChildren() throws IOException { + public void testContainerNodeIterator() throws IOException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -532,10 +532,19 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); - ResourceIterator emptyIter = nodeDAO.iterator(orig); - Assert.assertNotNull(emptyIter); - Assert.assertFalse(emptyIter.hasNext()); - emptyIter.close(); + try (ResourceIterator emptyIter = nodeDAO.iterator(orig)) { + Assert.assertNotNull(emptyIter); + Assert.assertFalse(emptyIter.hasNext()); + } // auto-close + + Node top = null; + try (ResourceIterator rootIter = nodeDAO.iterator(root)) { + if (rootIter.hasNext()) { + top = rootIter.next(); + } + } + Assert.assertNotNull(top); + Assert.assertEquals(orig.getID(), top.getID()); // add children ContainerNode cont = new ContainerNode("container1", false); @@ -554,14 +563,18 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); - ResourceIterator iter = nodeDAO.iterator(orig); - Assert.assertNotNull(iter); - Assert.assertTrue(iter.hasNext()); - Node c1 = iter.next(); - Assert.assertTrue(iter.hasNext()); - Node c2 = iter.next(); - Assert.assertTrue(iter.hasNext()); - Node c3 = iter.next(); + Node c1; + Node c2; + Node c3; + try (ResourceIterator iter = nodeDAO.iterator(orig)) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c3 = iter.next(); + } // default order: alpha Assert.assertEquals(cont.getID(), c1.getID()); @@ -573,14 +586,7 @@ public void testPutGetDeleteContainerNodeChildren() throws IOException { Assert.assertEquals(link.getID(), c3.getID()); Assert.assertEquals(link.getName(), c3.getName()); - // depth first delete required? - try { - nodeDAO.delete(orig.getID()); - Assert.fail("expected IllegalStateException but successfully deleted non-empty container"); - } catch (IllegalStateException expected) { - log.info("caught expected: " + expected); - } - + // depth first delete required but not enforced by DAO nodeDAO.delete(cont.getID()); nodeDAO.delete(data.getID()); nodeDAO.delete(link.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index d9c45bc93..d8074515a 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -916,11 +916,14 @@ public ResourceIterator query(DataSource ds) { int col = 1; ps.setObject(1, parent.getID()); + log.debug("parentID = " + parent.getID()); if (start != null) { ps.setString(col++, start); + log.debug("start = " + start); } if (limit != null) { ps.setInt(col++, limit); + log.debug("limit = " + limit); } ResultSet rs = ps.executeQuery(); @@ -1591,10 +1594,16 @@ private class NodeResultSetIterator implements ResourceIterator { ContainerNode parent; - public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) { + public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) throws SQLException { this.con = con; this.rs = rs; this.parent = parent; + hasRow = rs.next(); + log.debug("NodeResultSetIterator: " + super.toString() + " ctor " + hasRow); + if (!hasRow) { + log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); + con.setAutoCommit(true); + } } @Override diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 756651843..b4d36d562 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -68,6 +68,7 @@ package org.opencadc.vospace.db; import ca.nrc.cadc.io.ResourceIterator; +import java.io.IOException; import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; From d47e89653b71f9821446b1880d1d0420cbcceb01 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 19 Apr 2023 12:56:21 -0700 Subject: [PATCH 008/186] fix and test use of Node.parentID add checksum verification to NodeDAO tests --- cadc-inventory-db/build.gradle | 1 + .../org/opencadc/vospace/db/NodeDAOTest.java | 53 ++++++++++++++----- .../opencadc/inventory/db/SQLGenerator.java | 19 +++---- .../java/org/opencadc/vospace/db/NodeDAO.java | 5 ++ 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 798804dc9..940b17d71 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -27,6 +27,7 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.9.5,2.0)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 62374a3f8..cbab0c4ba 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -74,6 +74,8 @@ import ca.nrc.cadc.util.Log4jInit; import java.io.IOException; import java.net.URI; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -167,7 +169,8 @@ public void testGetByPath_NotFound() { } @Test - public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { + public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -180,16 +183,18 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); - log.info("found: " + a.getID() + " aka " + a); + log.info("found by id: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); Assert.assertNotNull(aa); - log.info("found: " + aa.getID() + " aka " + aa); + log.info("found by path: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); + Assert.assertEquals(root.getID(), a.parentID); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); @@ -210,6 +215,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); @@ -244,7 +252,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException { } @Test - public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException { + public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -274,18 +283,20 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException // get-by-id Node a = nodeDAO.get(orig.getID()); Assert.assertNotNull(a); - log.info("found: " + a.getID() + " aka " + a); + log.info("found by id: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); Assert.assertNotNull(aa); - log.info("found: " + aa.getID() + " aka " + aa); + log.info("found by path: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -304,6 +315,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.isPublic = false; @@ -342,11 +356,12 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException } @Test - public void testPutGetUpdateDeleteDataNode() throws InterruptedException { + public void testPutGetUpdateDeleteDataNode() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); - DataNode orig = new DataNode("data-test", URI.create("cadc:vault/" + UUID.randomUUID())); + DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); orig.parent = root; orig.ownerID = "the-owner"; orig.isPublic = true; @@ -361,6 +376,8 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); @@ -370,6 +387,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -382,12 +400,15 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { Assert.assertTrue(a instanceof DataNode); DataNode dn = (DataNode) a; - Assert.assertEquals(orig.getStorageID(), dn.getStorageID()); + Assert.assertEquals(orig.storageID, dn.storageID); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.isPublic = false; @@ -419,7 +440,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { Assert.assertTrue(a instanceof DataNode); DataNode udn = (DataNode) updated; - Assert.assertEquals(orig.getStorageID(), udn.getStorageID()); + Assert.assertEquals(orig.storageID, udn.storageID); nodeDAO.delete(orig.getID()); Node gone = nodeDAO.get(orig.getID()); @@ -427,7 +448,8 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException { } @Test - public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { + public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, + NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root", false); @@ -447,6 +469,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { log.info("found: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); // get-by-path Node aa = nodeDAO.get(root, orig.getName()); @@ -456,6 +479,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertNotNull(aa.parent); Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -474,6 +498,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException { Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); + URI mcs = a.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("metaChecksum", a.getMetaChecksum(), mcs); + // update Thread.sleep(10L); orig.isPublic = false; @@ -550,7 +577,7 @@ public void testContainerNodeIterator() throws IOException { ContainerNode cont = new ContainerNode("container1", false); cont.parent = orig; cont.ownerID = orig.ownerID; - DataNode data = new DataNode("data1", URI.create("cadc:vault/" + UUID.randomUUID())); + DataNode data = new DataNode(UUID.randomUUID(), "data1", URI.create("cadc:vault/" + UUID.randomUUID())); data.parent = orig; data.ownerID = orig.ownerID; LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index d8074515a..7aa14c010 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -476,7 +476,7 @@ public void setLocation(StorageLocation loc) { @Override public ObsoleteStorageLocation execute(JdbcTemplate jdbc) { - return (ObsoleteStorageLocation) jdbc.query(this, new ObsoleteStorageLocationExtractor()); + return jdbc.query(this, new ObsoleteStorageLocationExtractor()); } @Override @@ -1276,10 +1276,10 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce PreparedStatement prep = conn.prepareStatement(sql); int col = 1; - if (value.parent == null) { - throw new RuntimeException("BUG: cannot put Node without a parent: " + value); + if (value.parentID == null) { + throw new RuntimeException("BUG: cannot put Node without a parentID: " + value); } - prep.setObject(col++, value.parent.getID()); + prep.setObject(col++, value.parentID); prep.setString(col++, value.getName()); prep.setString(col++, value.getClass().getSimpleName().substring(0, 1)); // HACK if (value.ownerID == null) { @@ -1299,11 +1299,11 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } if (value instanceof DataNode) { DataNode dn = (DataNode) value; - if (dn.getStorageID() == null) { + if (dn.storageID == null) { throw new RuntimeException("BUG: cannot put DataNode without a storageID: " + value); } safeSetBoolean(prep, col++, dn.busy); - safeSetString(prep, col++, dn.getStorageID()); + safeSetString(prep, col++, dn.storageID); } else { safeSetBoolean(prep, col++, null); safeSetString(prep, col++, (URI) null); @@ -1711,6 +1711,7 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro } else { throw new RuntimeException("BUG: unexpected node type code: " + nodeType); } + ret.parentID = parentID; ret.ownerID = ownerID; ret.isPublic = isPublic; ret.isLocked = isLocked; @@ -1738,12 +1739,12 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro return ret; } - private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { + private class ObsoleteStorageLocationExtractor implements ResultSetExtractor { final Calendar utc = Calendar.getInstance(DateUtil.UTC); @Override - public Object extractData(ResultSet rs) throws SQLException, DataAccessException { + public ObsoleteStorageLocation extractData(ResultSet rs) throws SQLException, DataAccessException { if (!rs.next()) { return null; } @@ -1756,7 +1757,7 @@ public Object extractData(ResultSet rs) throws SQLException, DataAccessException StorageLocation s = new StorageLocation(storLoc); s.storageBucket = storBucket; - Entity ret = new ObsoleteStorageLocation(id, s); + ObsoleteStorageLocation ret = new ObsoleteStorageLocation(id, s); InventoryUtil.assignLastModified(ret, lastModified); InventoryUtil.assignMetaChecksum(ret, metaChecksum); return ret; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index b4d36d562..1da5df287 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -93,6 +93,11 @@ public NodeDAO() { @Override public void put(Node val) { + // TBD: caller can assign parent or parentID before put, but here we need + // parentID and it must be assigned before metaChecksum compute in super.put() + if (val.parentID == null && val.parent != null) { + val.parentID = val.parent.getID(); + } super.put(val); } From 59e86b18f61e62b1dafbd789fd5b1e1270ba4add Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 12:26:03 -0700 Subject: [PATCH 009/186] vault: add init database, update README with additional config --- vault/README.md | 50 ++++- vault/build.gradle | 9 + .../opencadc/vault/NodePersistenceImpl.java | 182 ++++++++++++++++++ .../org/opencadc/vault/VaultInitAction.java | 49 +++-- 4 files changed, 271 insertions(+), 19 deletions(-) create mode 100644 vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java diff --git a/vault/README.md b/vault/README.md index 3dd77529a..20dad5cfa 100644 --- a/vault/README.md +++ b/vault/README.md @@ -1,8 +1,20 @@ -# Storage Inventory storage management service (vault) +# Storage Inventory VOSpace-2.1 service (vault) + +The `vault` servcie is an implementation of the IVOA VOSpace +specification designed to co-exist with other storage-inventory components. It provides a heirarchical data +organization laye on top of the storage management of storage-inventory. + +The simplest configuration would be to deploy `vault` with `minoc` with a single metadata database and single +back end storage system. Details: TBD. + +The other option would be to deploy `vault` with `raven` and `luskan` in a global inventory database and make +use of one or more of the network of known storage sites to store files. Details: TBD. ## configuration See the [cadc-tomcat](https://github.com/opencadc/docker-base/tree/master/cadc-tomcat) image docs -for expected deployment and general config requirements. The `vault` war file can be renamed +for expected deployment and general config requirements. + +The `vault` war file can be renamed at deployment time in order to support an alternate service name, including introducing additional path elements (see war-rename.conf). @@ -18,7 +30,7 @@ org.opencadc.vault.nodes.username={username for vospace pool} org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} ``` -The `nodes` account owns and manages (create, alter, drop) vault database objects and manages +The `nodes` account owns and manages (create, alter, drop) vospace database objects and manages all the content (insert, update, delete). The database is specified in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. @@ -27,25 +39,47 @@ VOSI-availability output. A vault.properties file in /config is required to run this service. The following keys are required: ``` # service identity -org.opencadc.vault.resourceID=ivo://{authority}/{name} +org.opencadc.vault.resourceID = ivo://{authority}/{name} # vault database settings -org.opencadc.vault.nodes.schema={schema name} +org.opencadc.vault.inventory.schema = {inventory schema name} +org.opencadc.vault.vospace.schema = {vospace schema name} + +# root container nodes +org.opencadc.vault.root.owner = {owner of root node} + +# storage namespace +org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. -The nodes _schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). +The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. + +The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must be in the same database and some operations may join tables +in the two schemas (probably just vospace.node join inventory.artifact). + +The root node owner has full read and write permission in the root container, so it can create and delete container +nodes at the root and to assign container node properties that are normally read-only to normal users: owner, quota, +etc. This is probably an X509 distingushed name of the user (to start). TBD. + +The _namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. This only +applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't +want to change this... prevent change? ### vault-availability.properties (optional) ``` -The vault-availability.properties file specifies which users have the authority to change the availability state of the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. +The vault-availability.properties file specifies which users have the authority to change the availability state of +the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical +user name. ``` Example: ``` users = {user identity} ``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities (X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. +`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities +(X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is not found or does not +list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. ## building it ``` diff --git a/vault/build.gradle b/vault/build.gradle index dc09ec290..94f622694 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -26,14 +26,23 @@ war { } dependencies { + compile 'org.opencadc:cadc-util:[1.9,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' + //compile 'org.opencadc:cadc-vos-server:[2.0,3.0)' + compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' + compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' testCompile 'junit:junit:[4.0,)' + runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' + runtime 'org.opencadc:cadc-gms:[1.0.4,)' + intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } configurations { + compile.exclude group: 'org.restlet.jee' runtime.exclude group: 'org.postgresql:postgresql' } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java new file mode 100644 index 000000000..3b770b479 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -0,0 +1,182 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.io.ResourceIterator; +import ca.nrc.cadc.net.TransientException; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.log4j.Logger; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeNotFoundException; +import org.opencadc.vospace.NodeNotSupportedException; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.db.NodeDAO; + +/** + * + * @author pdowler + */ +public class NodePersistenceImpl { //implements NodePersistence { + private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); + + private final Map nodeDaoConfig = new TreeMap<>(); + + public NodePersistenceImpl(String dataSourceName, String inventorySchema, String vospaceSchema) { + nodeDaoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); + nodeDaoConfig.put("jndiDataSourceName", dataSourceName); + nodeDaoConfig.put("schema", inventorySchema); + nodeDaoConfig.put("vosSchema", vospaceSchema); + + } + + private NodeDAO getDAO() { + NodeDAO instance = new NodeDAO(); + instance.setConfig(nodeDaoConfig); + return instance; + } + + /** + * Get a node by name. + * @param parent parent node, may be special root node but not null + * @param name relative name of the child node + * @return the child node or null if it does not exist + * @throws TransientException + */ + public Node get(ContainerNode parent, String name) throws TransientException { + throw new UnsupportedOperationException(); + } + + /** + * Get an iterator over the children of a node. The output can optionally be + * limited to a specific number of children and can optionally start at a + * specific child (usually the last one from a previous "batch") to resume + * listing at a known position. + * + * @param parent the container to iterate + * @param limit max number of nodes to return, may be null + * @param start first node in order to consider, may be null + * @return iterator of matching child nodes, may be empty + */ + public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { + throw new UnsupportedOperationException(); + } + + /** + * Load additional node properties for the specified node. Note: this may not be + * necessary and may be removed. TBD. + * + * @param node + * @throws TransientException + */ + public void getProperties(Node node) throws TransientException { + throw new UnsupportedOperationException(); + } + + /** + * Put the specified node. This can be an insert or update; to update, the argument + * node must have been retrieved from persistence so it has the right Entity.id + * value. This method may modify the Entity.metaChecksum and the Entity.lastModified + * values. + * + * @param node the node to insert or update + * @return the possibly modified node + * @throws NodeNotSupportedException + * @throws TransientException + */ + public Node put(Node node) throws NodeNotSupportedException, TransientException { + // TODO: assign node.parentID here and remove from NodeDAO? + // TODO: assign DataNode.storageID here -- only when null aka new DataNode? + throw new UnsupportedOperationException(); + } + + /** + * This can be done via put(Node) so probably obsolete. TBD. + * @param node + * @param list + * @return + * @throws TransientException + */ + public Node updateProperties(Node node, List list) throws TransientException { + throw new UnsupportedOperationException(); + } + + /** + * Delete the specified node. + * + * @param node the node to delete + * @throws TransientException + */ + public void delete(Node node) throws TransientException { + throw new UnsupportedOperationException(); + } + +} diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 2f6a34f71..e606d2ad0 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -77,6 +77,8 @@ import java.util.TreeMap; import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.inventory.Namespace; +import org.opencadc.vospace.db.InitDatabaseVOS; /** * @@ -90,10 +92,16 @@ public class VaultInitAction extends InitAction { // config keys private static final String VAULT_KEY = "org.opencadc.vault"; static final String RESOURCE_ID_KEY = VAULT_KEY + ".resourceID"; - static final String SCHEMA_KEY = VAULT_KEY + ".nodes.schema"; + static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; + static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; + + static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? + + static final String STORAGE_NAMESPACE_KEY = VAULT_KEY + ".storage.namespace"; MultiValuedProperties props; private URI resourceID; + private Namespace storageNamespace; private Map daoConfig; public VaultInitAction() { @@ -129,15 +137,33 @@ static MultiValuedProperties getConfig() { sb.append("OK"); } - String schema = mvp.getFirstPropertyValue(SCHEMA_KEY); - sb.append("\n\t").append(SCHEMA_KEY).append(": "); - if (schema == null) { + String invSchema = mvp.getFirstPropertyValue(INVENTORY_SCHEMA_KEY); + sb.append("\n\t").append(INVENTORY_SCHEMA_KEY).append(": "); + if (invSchema == null) { sb.append("MISSING"); ok = false; } else { sb.append("OK"); } - + + String vosSchema = mvp.getFirstPropertyValue(VOSPACE_SCHEMA_KEY); + sb.append("\n\t").append(VOSPACE_SCHEMA_KEY).append(": "); + if (vosSchema == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + + String ns = mvp.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); + sb.append("\n\t").append(STORAGE_NAMESPACE_KEY).append(": "); + if (ns == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + if (!ok) { throw new IllegalStateException(sb.toString()); } @@ -148,7 +174,8 @@ static MultiValuedProperties getConfig() { static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.SCHEMA_KEY)); + ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); + ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); return ret; } @@ -156,9 +183,10 @@ private void initConfig() { log.info("initConfig: START"); this.props = getConfig(); String rid = props.getFirstPropertyValue(RESOURCE_ID_KEY); - + String ns = props.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); try { this.resourceID = new URI(rid); + this.storageNamespace = new Namespace(ns); this.daoConfig = getDaoConfig(props); log.info("initConfig: OK"); } catch (URISyntaxException ex) { @@ -170,10 +198,9 @@ private void initDatabase() { log.info("initDatabase: START"); try { DataSource ds = DBUtil.findJNDIDataSource(JNDI_DATASOURCE); - String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); - //VaultInitDatabase init = new VaultInitDatabase(ds, database, schema); - //init.doInit(); + String schema = (String) daoConfig.get("vosSchema"); + InitDatabaseVOS init = new InitDatabaseVOS(ds, null, schema); + init.doInit(); log.info("initDatabase: " + JNDI_DATASOURCE + " " + schema + " OK"); } catch (Exception ex) { throw new IllegalStateException("check/init database failed", ex); From 34bfcd852f0d632e48c1826ecd1f3871e71cae70 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 13:56:53 -0700 Subject: [PATCH 010/186] vault: implement more NodePersistenceImpl methods add TODO --- vault/TODO | 6 + .../opencadc/vault/NodePersistenceImpl.java | 209 ++++++++++++++++-- 2 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 vault/TODO diff --git a/vault/TODO b/vault/TODO new file mode 100644 index 000000000..003ad444f --- /dev/null +++ b/vault/TODO @@ -0,0 +1,6 @@ + +NodePersistenceImpl: review for necessary transactions and locks + +NodePersistenceImpl: reconcile with NodePersistence API + + diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 3b770b479..afa32fc85 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -69,16 +69,24 @@ import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.util.MultiValuedProperties; +import java.io.IOException; +import java.net.URI; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; import org.opencadc.vospace.Node; -import org.opencadc.vospace.NodeNotFoundException; import org.opencadc.vospace.NodeNotSupportedException; import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; import org.opencadc.vospace.db.NodeDAO; /** @@ -89,13 +97,66 @@ public class NodePersistenceImpl { //implements NodePersistence { private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); private final Map nodeDaoConfig = new TreeMap<>(); + private final ContainerNode root; + private final ContainerNode trash; + private final Namespace storageNamespace; + private final Set nonWritableProps = new TreeSet<>(); + private final Set rootAdminProps = new TreeSet<>(); + private final Set directNodeProps = new TreeSet<>(); - public NodePersistenceImpl(String dataSourceName, String inventorySchema, String vospaceSchema) { + public NodePersistenceImpl() { + MultiValuedProperties config = VaultInitAction.getConfig(); + String dataSourceName = VaultInitAction.JNDI_DATASOURCE; + String inventorySchema = config.getFirstPropertyValue(VaultInitAction.INVENTORY_SCHEMA_KEY); + String vospaceSchema = config.getFirstPropertyValue(VaultInitAction.VOSPACE_SCHEMA_KEY); nodeDaoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); nodeDaoConfig.put("jndiDataSourceName", dataSourceName); nodeDaoConfig.put("schema", inventorySchema); nodeDaoConfig.put("vosSchema", vospaceSchema); + + UUID rootID = new UUID(0L, 0L); + this.root = new ContainerNode(rootID, "", false); + String owner = config.getFirstPropertyValue(VaultInitAction.ROOT_OWNER); + //root.owner = ?? // subject needed to authorize admin requests + //root.ownerID not needed + + // TODO: do this setup in a txn with a lock on something + NodeDAO dao = getDAO(); + ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); + if (tn == null) { + tn = new ContainerNode(".trash", false); + tn.ownerID = root.ownerID; + tn.owner = root.owner; + tn.isPublic = false; + dao.put(tn); + } + this.trash = tn; + + String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); + this.storageNamespace = new Namespace(ns); + + // node properties that match immutable Artifact fields + nonWritableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // data nodes + nonWritableProps.add(VOS.PROPERTY_URI_CONTENTMD5); // data nodes + nonWritableProps.add(VOS.PROPERTY_URI_CREATION_DATE); // no touch + + // computed properties + nonWritableProps.add(VOS.PROPERTY_URI_WRITABLE); // prediction for current caller + + // props only the root admin can modify + rootAdminProps.add(VOS.PROPERTY_URI_AVAILABLESPACE); + rootAdminProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes + rootAdminProps.add(VOS.PROPERTY_URI_QUOTA); + rootAdminProps.add(VOS.PROPERTY_URI_CREATOR); // owner + + // props that are stored as Node fields + directNodeProps.add(VOS.PROPERTY_URI_GROUPREAD); + directNodeProps.add(VOS.PROPERTY_URI_GROUPWRITE); + directNodeProps.add(VOS.PROPERTY_URI_INHERIT_PERMISSIONS); + directNodeProps.add(VOS.PROPERTY_URI_ISLOCKED); + directNodeProps.add(VOS.PROPERTY_URI_ISPUBLIC); + } private NodeDAO getDAO() { @@ -103,16 +164,41 @@ private NodeDAO getDAO() { instance.setConfig(nodeDaoConfig); return instance; } + + private URI generateStorageID() { + UUID id = UUID.randomUUID(); + URI ret = URI.create(storageNamespace.getNamespace() + id.toString()); + return ret; + } /** - * Get a node by name. + * Get the container node that represents the root of all other nodes. + * This container node is used to navigate a path (from the root) using + * get(ContainerNode parent, String name). + * + * @return the root container node + */ + public ContainerNode getRootNode() { + return root; + } + + /** + * Get a node by name. Concept: The caller uses this to navigate the path from the root + * node to the target, checking permissions and deciding what to do about + * LinkNode(s) along the way. + * * @param parent parent node, may be special root node but not null * @param name relative name of the child node * @return the child node or null if it does not exist * @throws TransientException */ public Node get(ContainerNode parent, String name) throws TransientException { - throw new UnsupportedOperationException(); + if (parent == null || name == null) { + throw new IllegalArgumentException("args cannot be null: parent, name"); + } + NodeDAO dao = getDAO(); + Node ret = dao.get(parent, name); + return ret; } /** @@ -127,7 +213,12 @@ public Node get(ContainerNode parent, String name) throws TransientException { * @return iterator of matching child nodes, may be empty */ public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { - throw new UnsupportedOperationException(); + if (parent == null) { + throw new IllegalArgumentException("arg cannot be null: parent"); + } + NodeDAO dao = getDAO(); + ResourceIterator ret = dao.iterator(parent, limit, start); + return ret; } /** @@ -153,20 +244,86 @@ public void getProperties(Node node) throws TransientException { * @throws TransientException */ public Node put(Node node) throws NodeNotSupportedException, TransientException { - // TODO: assign node.parentID here and remove from NodeDAO? - // TODO: assign DataNode.storageID here -- only when null aka new DataNode? - throw new UnsupportedOperationException(); + if (node == null) { + throw new IllegalArgumentException("arg cannot be null: node"); + } + if (node.parentID == null) { + if (node.parent == null) { + throw new RuntimeException("BUG: cannot persist node without parent: " + node); + } + node.parentID = node.parent.getID(); + } + if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + if (dn.storageID == null) { + // new data node? if lastModified is assigned, this looks sketchy + if (dn.getLastModified() != null) { + throw new RuntimeException( + "BUG: attempt to put a previously stored DataNode without persistent storageID: " + + dn.getID() + " aka " + dn); + } + // concept: use a persistent storageID in the node that resolves to a a file + // once someone puts the file to minoc, so Node.storageID == Artifact.uri + // but the artifact may or may not exist + dn.storageID = generateStorageID(); + } + } + NodeDAO dao = getDAO(); + dao.put(node); + return node; } /** - * This can be done via put(Node) so probably obsolete. TBD. - * @param node - * @param list - * @return + * Update properties of a node. This method is responsible for accepting/rejecting changes + * based on whether a node property is read-only or writable by the caller (implementation + * specific). + * + * @param node the node to update + * @param props new property values to set + * @return the modified node * @throws TransientException */ - public Node updateProperties(Node node, List list) throws TransientException { - throw new UnsupportedOperationException(); + public Node updateProperties(Node node, List props) throws TransientException { + if (node == null || props == null) { + throw new IllegalArgumentException("args cannot be null: node, props"); + } + + // merge props -> node and/or node.properties + for (NodeProperty np : props) { + if (nonWritableProps.contains(np.getKey())) { + log.debug("updateProperties: skip " + np.getKey()); + } else { + // some props only root admin can modify + if (rootAdminProps.contains(np.getKey())) { + throw new UnsupportedOperationException("TODO: allow admin to modify prop: " + np.getKey()); + } + + // some props are directly in the node + if (VOS.PROPERTY_URI_FORMAT.equals(np.getKey()) + || VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { + throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); + } + + // some props are directly in the artifact + if (directNodeProps.contains(np.getKey())) { + throw new UnsupportedOperationException("TODO: set mutable node prop: " + np.getKey()); + } + + // generic key-value props + if (node.properties.contains(np)) { + log.debug("updateProperties: remove previous " + np.getKey()); + node.properties.remove(np); + } + if (!np.isMarkedForDeletion()) { + node.properties.add(np); + } + } + } + + NodeDAO dao = getDAO(); + dao.put(node); + return node; + } /** @@ -176,7 +333,29 @@ public Node updateProperties(Node node, List list) throws Transien * @throws TransientException */ public void delete(Node node) throws TransientException { - throw new UnsupportedOperationException(); + if (node == null) { + throw new IllegalArgumentException("arg cannot be null: node"); + } + + NodeDAO dao = getDAO(); + boolean moveToTrash = false; + if (node instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) node; + try (ResourceIterator iter = dao.iterator(cn, 1, null)) { + moveToTrash = !iter.hasNext(); + } catch (IOException ex) { + throw new TransientException("database IO failure", ex); + } + } + + // TODO: DeletedNodeEvent + if (moveToTrash) { + node.parentID = trash.getID(); + dao.put(node); + } else { + dao.delete(node.getID()); + } + } } From 8082aab99651c47181977b1fc86c9e6d2e1c9891 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 13:58:12 -0700 Subject: [PATCH 011/186] NodeDAO: remove assign parentID in put --- .../java/org/opencadc/vospace/db/NodeDAOTest.java | 6 +++--- .../src/main/java/org/opencadc/vospace/db/NodeDAO.java | 9 +-------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index cbab0c4ba..145db59a7 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -559,13 +559,13 @@ public void testContainerNodeIterator() throws IOException { // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); Assert.assertEquals(orig.getLastModified(), a.getLastModified()); - try (ResourceIterator emptyIter = nodeDAO.iterator(orig)) { + try (ResourceIterator emptyIter = nodeDAO.iterator(orig, null, null)) { Assert.assertNotNull(emptyIter); Assert.assertFalse(emptyIter.hasNext()); } // auto-close Node top = null; - try (ResourceIterator rootIter = nodeDAO.iterator(root)) { + try (ResourceIterator rootIter = nodeDAO.iterator(root, null, null)) { if (rootIter.hasNext()) { top = rootIter.next(); } @@ -593,7 +593,7 @@ public void testContainerNodeIterator() throws IOException { Node c1; Node c2; Node c3; - try (ResourceIterator iter = nodeDAO.iterator(orig)) { + try (ResourceIterator iter = nodeDAO.iterator(orig, null, null)) { Assert.assertNotNull(iter); Assert.assertTrue(iter.hasNext()); c1 = iter.next(); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 1da5df287..6fc157862 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -68,15 +68,12 @@ package org.opencadc.vospace.db; import ca.nrc.cadc.io.ResourceIterator; -import java.io.IOException; import java.util.UUID; import org.apache.log4j.Logger; -import org.opencadc.inventory.Artifact; import org.opencadc.inventory.db.AbstractDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.Node; -import org.opencadc.vospace.VOSURI; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; @@ -128,11 +125,7 @@ public void delete(UUID id) { super.delete(Node.class, id); } - public ResourceIterator iterator(ContainerNode parent) { - return NodeDAO.this.iterator(parent, null, null); - } - - public ResourceIterator iterator(ContainerNode parent, String start, Integer limit) { + public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { if (parent == null) { throw new IllegalArgumentException("childIterator: parent cannot be null"); } From 87c68bbb34b80ff6f07375a0839d612b9044f8a3 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 13:59:15 -0700 Subject: [PATCH 012/186] cadc-inventory-db: nominally assign version as 0.15 --- cadc-inventory-db/build.gradle | 2 +- fenwick/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 940b17d71..2f1566ca2 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -17,7 +17,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.14.5' +version = '0.15.0' description = 'OpenCADC Storage Inventory database library' def git_url = 'https://github.com/opencadc/storage-inventory' diff --git a/fenwick/build.gradle b/fenwick/build.gradle index 15e17f012..6d9c2cdb8 100644 --- a/fenwick/build.gradle +++ b/fenwick/build.gradle @@ -16,8 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.9.6,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,2.0)' + compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' + compile 'org.opencadc:cadc-inventory-db:[0.14.5,1.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.14,1.2)' // 1.2 upper bound is correct #reasons From 41dc8dcdee3d5a206670b815939fc5183826be02 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 21 Apr 2023 14:12:48 -0700 Subject: [PATCH 013/186] update TODO --- vault/TODO | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vault/TODO b/vault/TODO index 003ad444f..e26e4199d 100644 --- a/vault/TODO +++ b/vault/TODO @@ -1,6 +1,15 @@ -NodePersistenceImpl: review for necessary transactions and locks +* NodePersistenceImpl: review for necessary transactions and locks -NodePersistenceImpl: reconcile with NodePersistence API +* NodePersistenceImpl: reconcile with NodePersistence API +* files endpoint: +- if coexist with minoc: generate pre-auth URL to it and redirect +- if coexist with raven: need raven ProtocolsGenerator + +* transfer negotiation: +- review cadc-vos-server and cavern implementations +- probably a complete TransferRunner; maybe separate sync and async runners +- if co-exist with minoc: generate pre-auth URL to it +- if co-exist with raven: need raven ProtocolsGenerator From 2512aa35c2686dd002449d9cc11bfb861c593e5e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 28 Apr 2023 09:09:34 -0700 Subject: [PATCH 014/186] vault: TODO updates --- vault/TODO | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vault/TODO b/vault/TODO index e26e4199d..1481088d1 100644 --- a/vault/TODO +++ b/vault/TODO @@ -1,7 +1,10 @@ * NodePersistenceImpl: review for necessary transactions and locks -* NodePersistenceImpl: reconcile with NodePersistence API +* NodePersistenceImpl: reconcile with NodePersistence API and assign responsibilities +- property update checking +- permission checking +- link node resolution * files endpoint: - if coexist with minoc: generate pre-auth URL to it and redirect @@ -12,4 +15,8 @@ - probably a complete TransferRunner; maybe separate sync and async runners - if co-exist with minoc: generate pre-auth URL to it - if co-exist with raven: need raven ProtocolsGenerator +- figure out if/how vault can have it's own uws tables in db or share with inventory (luskan) +* pre-auth URL keys -- what to support? recommend? +- vault has it's own key pair && minoc(s) have multiple pub keys? +- vault and raven share private key? From 39f51eaf398aa0237af45d549c88a0eb305f7a94 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 May 2023 10:29:44 -0700 Subject: [PATCH 015/186] vault: update README --- vault/README.md | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/vault/README.md b/vault/README.md index 20dad5cfa..67d202847 100644 --- a/vault/README.md +++ b/vault/README.md @@ -10,19 +10,30 @@ back end storage system. Details: TBD. The other option would be to deploy `vault` with `raven` and `luskan` in a global inventory database and make use of one or more of the network of known storage sites to store files. Details: TBD. -## configuration -See the [cadc-tomcat](https://github.com/opencadc/docker-base/tree/master/cadc-tomcat) image docs -for expected deployment and general config requirements. +## deployment -The `vault` war file can be renamed -at deployment time in order to support an alternate service name, including introducing -additional path elements (see war-rename.conf). +The `vault` war file can be renamed at deployment time in order to support an alternate service name, +including introducing additional path elements. See +cadc-tomcat (war-rename.conf). -Runtime configuration must be made available via the `/config` directory. +## configuration +The following runtime configuration must be made available via the `/config` directory. ### catalina.properties -When running vault.war in tomcat, parameters of the connection pool in META-INF/context.xml need -to be configured in catalina.properties: +This file contains java system properties to configure the tomcat server and some of the java libraries used in the service. + +See cadc-tomcat +for system properties related to the deployment environment. + +See cadc-util +for common system properties. + +`minoc` includes multiple IdentityManager implementations to support authenticated access: +- See cadc-access-control-identity for CADC access-control system support. + +- See cadc-gms for OIDC token support. + +`vault` requires a connection pool to the local database: ``` # database connection pools org.opencadc.vault.nodes.maxActive={max connections for vospace pool} @@ -53,25 +64,25 @@ org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. -The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. +The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due to configuration +limitations in luskan. The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must be in the same database and some operations may join tables in the two schemas (probably just vospace.node join inventory.artifact). The root node owner has full read and write permission in the root container, so it can create and delete container -nodes at the root and to assign container node properties that are normally read-only to normal users: owner, quota, -etc. This is probably an X509 distingushed name of the user (to start). TBD. +nodes at the root and assign container node properties that are normally read-only to normal users: owner, quota, +etc. This is probably an X509 distingushed name of the user (to start). **not fully implemented** TBD. The _namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. This only applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't -want to change this... prevent change? +want to change this... prevent change? TBD. ### vault-availability.properties (optional) -``` + The vault-availability.properties file specifies which users have the authority to change the availability state of the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. -``` Example: ``` From 83381d9192504d7b31688aa2a99d82c89ca2c633 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 May 2023 10:32:57 -0700 Subject: [PATCH 016/186] add vault to repo README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2838d9b05..1b2fbf4da 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ local artifact selection policy. This is used if the new fenwick policy excludes This is an implementation of the **file-validate** process that compares the inventory database against the back end storage at a storage site. +## vault +UNDER DEVELOPMENT: This is an implementation of an IVOA VOSpace +service that uses storage-inventory as the back end storage mechanism. + ## cadc-* These are libraries used in multiple services and applications. From c542400a1604c27ed42664a596a9f7983eaeb2df Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 31 May 2023 14:05:00 -0700 Subject: [PATCH 017/186] update for Node API changes --- .../org/opencadc/vospace/db/NodeDAOTest.java | 122 +++++++++--------- .../opencadc/inventory/db/SQLGenerator.java | 12 +- .../java/org/opencadc/inventory/db/Util.java | 4 +- 3 files changed, 70 insertions(+), 68 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 145db59a7..b3aa5d1ef 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -203,9 +203,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; @@ -220,9 +220,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, // update Thread.sleep(10L); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.isPublic = true; nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -237,9 +237,9 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(updated instanceof ContainerNode); ContainerNode uc = (ContainerNode) updated; @@ -265,19 +265,19 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.isPublic = true; orig.isLocked = false; orig.inheritPermissions = false; - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g2")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g4,g5")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6-g7")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6.g7")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6_g7")); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g6~g7")); - - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); - orig.properties.add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); - orig.properties.add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); - orig.properties.add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g2")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g4,g5")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6-g7")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6.g7")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6_g7")); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6~g7")); + + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getProperties().add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); + orig.getProperties().add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); + orig.getProperties().add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); nodeDAO.put(orig); // get-by-id @@ -303,9 +303,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; @@ -322,12 +322,12 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Thread.sleep(10L); orig.isPublic = false; orig.isLocked = true; - orig.readOnlyGroup.clear(); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.clear(); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.clear(); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.inheritPermissions = true; nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -342,9 +342,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(updated instanceof ContainerNode); ContainerNode uc = (ContainerNode) updated; @@ -366,8 +366,8 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, orig.ownerID = "the-owner"; orig.isPublic = true; orig.isLocked = false; - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "this is the good stuff(tm)")); nodeDAO.put(orig); // get-by-id @@ -394,9 +394,9 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof DataNode); DataNode dn = (DataNode) a; @@ -413,12 +413,12 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, Thread.sleep(10L); orig.isPublic = false; orig.isLocked = true; - orig.readOnlyGroup.clear(); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.clear(); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.clear(); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change storageID nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -433,9 +433,9 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(a instanceof DataNode); @@ -460,7 +460,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, orig.ownerID = "the-owner"; orig.isPublic = true; orig.isLocked = false; - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DESCRIPTION, "link to the good stuff(tm)")); nodeDAO.put(orig); // get-by-id @@ -486,9 +486,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, a.ownerID); Assert.assertEquals(orig.isPublic, a.isPublic); Assert.assertEquals(orig.isLocked, a.isLocked); - Assert.assertEquals(orig.readOnlyGroup, a.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, a.readWriteGroup); - Assert.assertEquals(orig.properties, a.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), a.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), a.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), a.getProperties()); Assert.assertTrue(a instanceof LinkNode); LinkNode link = (LinkNode) a; @@ -505,12 +505,12 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Thread.sleep(10L); orig.isPublic = false; orig.isLocked = true; - orig.readOnlyGroup.clear(); - orig.readOnlyGroup.add(URI.create("ivo://opencadc.org/gms?g1")); - orig.readWriteGroup.clear(); - orig.readWriteGroup.add(URI.create("ivo://opencadc.org/gms?g3")); - orig.properties.clear(); - orig.properties.add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); + orig.getReadOnlyGroup().clear(); + orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadWriteGroup().clear(); + orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getProperties().clear(); + orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change target nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -525,9 +525,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Assert.assertEquals(orig.ownerID, updated.ownerID); Assert.assertEquals(orig.isPublic, updated.isPublic); Assert.assertEquals(orig.isLocked, updated.isLocked); - Assert.assertEquals(orig.readOnlyGroup, updated.readOnlyGroup); - Assert.assertEquals(orig.readWriteGroup, updated.readWriteGroup); - Assert.assertEquals(orig.properties, updated.properties); + Assert.assertEquals(orig.getReadOnlyGroup(), updated.getReadOnlyGroup()); + Assert.assertEquals(orig.getReadWriteGroup(), updated.getReadWriteGroup()); + Assert.assertEquals(orig.getProperties(), updated.getProperties()); Assert.assertTrue(updated instanceof LinkNode); LinkNode ulink = (LinkNode) updated; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 7aa14c010..97d27e985 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1288,9 +1288,9 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce prep.setString(col++, value.ownerID.toString()); safeSetBoolean(prep, col++, value.isPublic); safeSetBoolean(prep, col++, value.isLocked); - safeSetArray(prep, col++, value.readOnlyGroup); - safeSetArray(prep, col++, value.readWriteGroup); - safeSetProps(prep, col++, value.properties); + safeSetArray(prep, col++, value.getReadOnlyGroup()); + safeSetArray(prep, col++, value.getReadWriteGroup()); + safeSetProps(prep, col++, value.getProperties()); if (value instanceof ContainerNode) { ContainerNode cn = (ContainerNode) value; safeSetBoolean(prep, col++, cn.inheritPermissions); @@ -1717,13 +1717,13 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro ret.isLocked = isLocked; if (rawROG != null) { - Util.parseArrayURI(rawROG, ret.readOnlyGroup); + Util.parseArrayURI(rawROG, ret.getReadOnlyGroup()); } if (rawRWG != null) { - Util.parseArrayURI(rawRWG, ret.readWriteGroup); + Util.parseArrayURI(rawRWG, ret.getReadWriteGroup()); } if (rawProps != null) { - Util.parseArrayProps(rawProps, ret.properties); + Util.parseArrayProps(rawProps, ret.getProperties()); } InventoryUtil.assignLastModified(ret, lastModified); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java index cd20020bf..2e7e197b8 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java @@ -383,6 +383,7 @@ public static byte[] getByteArray(ResultSet rs, int col) throw new UnsupportedOperationException("converting " + o.getClass().getName() + " " + o + " to byte[]"); } + // fills the dest set public static void parseArrayURI(String val, Set dest) { // postgresql 1D array: {a,"b,c"} if (val == null || val.isEmpty()) { @@ -417,7 +418,8 @@ private static void handleToken(String token, Set dest) { } } } - + + // fills the dest set public static void parseArrayProps(String val, Set dest) { // postgresql 2D array: {{a,b},{c,d}} if (val == null || val.isEmpty()) { From c1e02401a4a8294f057d97c08f6ac60e2d3bda54 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 1 Jun 2023 13:24:09 -0700 Subject: [PATCH 018/186] vault: NodePersistenceImpl implements extracted interface --- .../opencadc/vault/NodePersistenceImpl.java | 299 +++++++++++++----- 1 file changed, 217 insertions(+), 82 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index afa32fc85..706499811 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -67,22 +67,37 @@ package org.opencadc.vault; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.auth.PrincipalExtractor; +import ca.nrc.cadc.auth.X509CertificateChain; +import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; import java.net.URI; +import java.security.Principal; +import java.text.DateFormat; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; +import javax.security.auth.Subject; +import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.DeletedNodeEvent; +import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotSupportedException; import org.opencadc.vospace.NodeProperty; @@ -93,16 +108,18 @@ * * @author pdowler */ -public class NodePersistenceImpl { //implements NodePersistence { +public class NodePersistenceImpl implements NodePersistence { private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); private final Map nodeDaoConfig = new TreeMap<>(); private final ContainerNode root; private final ContainerNode trash; private final Namespace storageNamespace; - private final Set nonWritableProps = new TreeSet<>(); + private final Set immutableProps = new TreeSet<>(); + + private final Set artifactProps = new TreeSet<>(); + private final Set rootAdminContainerProps = new TreeSet<>(); private final Set rootAdminProps = new TreeSet<>(); - private final Set directNodeProps = new TreeSet<>(); public NodePersistenceImpl() { MultiValuedProperties config = VaultInitAction.getConfig(); @@ -115,48 +132,65 @@ public NodePersistenceImpl() { nodeDaoConfig.put("vosSchema", vospaceSchema); + final String owner = config.getFirstPropertyValue(VaultInitAction.ROOT_OWNER); + if (owner == null) { + throw new InvalidConfigException(VaultInitAction.ROOT_OWNER + " cannot be null"); + } + IdentityManager im = AuthenticationUtil.getIdentityManager(); + Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { + @Override + public Set getPrincipals() { + Set ret = new TreeSet<>(); + ret.add(new X500Principal(owner)); + return ret; + } + + @Override + public X509CertificateChain getCertificateChain() { + return null; + } + }); + + // root node UUID rootID = new UUID(0L, 0L); this.root = new ContainerNode(rootID, "", false); - String owner = config.getFirstPropertyValue(VaultInitAction.ROOT_OWNER); - //root.owner = ?? // subject needed to authorize admin requests - //root.ownerID not needed + root.owner = im.augment(rawOwnerSubject); + root.ownerID = im.toDisplayString(root.owner); + // trash node // TODO: do this setup in a txn with a lock on something NodeDAO dao = getDAO(); ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); if (tn == null) { tn = new ContainerNode(".trash", false); - tn.ownerID = root.ownerID; - tn.owner = root.owner; - tn.isPublic = false; - dao.put(tn); } + // always reset props to current config + tn.ownerID = root.ownerID; + tn.owner = root.owner; + tn.isPublic = false; + dao.put(tn); this.trash = tn; String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); this.storageNamespace = new Namespace(ns); - // node properties that match immutable Artifact fields - nonWritableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // data nodes - nonWritableProps.add(VOS.PROPERTY_URI_CONTENTMD5); // data nodes - nonWritableProps.add(VOS.PROPERTY_URI_CREATION_DATE); // no touch - // computed properties - nonWritableProps.add(VOS.PROPERTY_URI_WRITABLE); // prediction for current caller + // VOS.PROPERTY_URI_AVAILABLESPACE // container nodes + // VOS.PROPERTY_URI_WRITABLE // prediction for current caller - // props only the root admin can modify - rootAdminProps.add(VOS.PROPERTY_URI_AVAILABLESPACE); - rootAdminProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes - rootAdminProps.add(VOS.PROPERTY_URI_QUOTA); + // props only the admin (root owner) can modify rootAdminProps.add(VOS.PROPERTY_URI_CREATOR); // owner + rootAdminContainerProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes + rootAdminContainerProps.add(VOS.PROPERTY_URI_QUOTA); // container nodes - // props that are stored as Node fields - directNodeProps.add(VOS.PROPERTY_URI_GROUPREAD); - directNodeProps.add(VOS.PROPERTY_URI_GROUPWRITE); - directNodeProps.add(VOS.PROPERTY_URI_INHERIT_PERMISSIONS); - directNodeProps.add(VOS.PROPERTY_URI_ISLOCKED); - directNodeProps.add(VOS.PROPERTY_URI_ISPUBLIC); + // node properties that match immutable Artifact fields + immutableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // immutable + immutableProps.add(VOS.PROPERTY_URI_CONTENTMD5); // immutable + immutableProps.add(VOS.PROPERTY_URI_CREATION_DATE); // immutable + artifactProps.addAll(immutableProps); + artifactProps.add(VOS.PROPERTY_URI_CONTENTENCODING); // mutable + artifactProps.add(VOS.PROPERTY_URI_TYPE); // mutable } private NodeDAO getDAO() { @@ -165,6 +199,12 @@ private NodeDAO getDAO() { return instance; } + private ArtifactDAO getArtifactDAO() { + ArtifactDAO instance = new ArtifactDAO(true); // origin==true? + instance.setConfig(nodeDaoConfig); + return instance; + } + private URI generateStorageID() { UUID id = UUID.randomUUID(); URI ret = URI.create(storageNamespace.getNamespace() + id.toString()); @@ -178,26 +218,51 @@ private URI generateStorageID() { * * @return the root container node */ + @Override public ContainerNode getRootNode() { return root; } - + + @Override + public Set getImmutableProps() { + return immutableProps; + } + /** - * Get a node by name. Concept: The caller uses this to navigate the path from the root - * node to the target, checking permissions and deciding what to do about - * LinkNode(s) along the way. + * Get a node by name. Concept: The caller uses this to navigate the path + * from the root node to the target, checking permissions and deciding what + * to do about LinkNode(s) along the way. * * @param parent parent node, may be special root node but not null * @param name relative name of the child node * @return the child node or null if it does not exist * @throws TransientException */ + @Override public Node get(ContainerNode parent, String name) throws TransientException { if (parent == null || name == null) { throw new IllegalArgumentException("args cannot be null: parent, name"); } NodeDAO dao = getDAO(); Node ret = dao.get(parent, name); + if (ret instanceof DataNode) { + DataNode dn = (DataNode) ret; + ArtifactDAO artifactDAO = getArtifactDAO(); + Artifact a = artifactDAO.get(dn.storageID); + if (a != null) { + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, a.getContentLength().toString())); + // assume MD5 + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTMD5, a.getContentChecksum().getSchemeSpecificPart())); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(a.getContentLastModified()))); + if (a.contentEncoding != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTENCODING, a.contentEncoding)); + } + if (a.contentType != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); + } + } + } return ret; } @@ -212,6 +277,7 @@ public Node get(ContainerNode parent, String name) throws TransientException { * @param start first node in order to consider, may be null * @return iterator of matching child nodes, may be empty */ + @Override public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { if (parent == null) { throw new IllegalArgumentException("arg cannot be null: parent"); @@ -228,8 +294,9 @@ public ResourceIterator iterator(ContainerNode parent, Integer limit, Stri * @param node * @throws TransientException */ + @Override public void getProperties(Node node) throws TransientException { - throw new UnsupportedOperationException(); + // no-op } /** @@ -243,6 +310,7 @@ public void getProperties(Node node) throws TransientException { * @throws NodeNotSupportedException * @throws TransientException */ + @Override public Node put(Node node) throws NodeNotSupportedException, TransientException { if (node == null) { throw new IllegalArgumentException("arg cannot be null: node"); @@ -266,56 +334,46 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException // once someone puts the file to minoc, so Node.storageID == Artifact.uri // but the artifact may or may not exist dn.storageID = generateStorageID(); - } - } - NodeDAO dao = getDAO(); - dao.put(node); - return node; - } - - /** - * Update properties of a node. This method is responsible for accepting/rejecting changes - * based on whether a node property is read-only or writable by the caller (implementation - * specific). - * - * @param node the node to update - * @param props new property values to set - * @return the modified node - * @throws TransientException - */ - public Node updateProperties(Node node, List props) throws TransientException { - if (node == null || props == null) { - throw new IllegalArgumentException("args cannot be null: node, props"); - } - - // merge props -> node and/or node.properties - for (NodeProperty np : props) { - if (nonWritableProps.contains(np.getKey())) { - log.debug("updateProperties: skip " + np.getKey()); } else { - // some props only root admin can modify - if (rootAdminProps.contains(np.getKey())) { - throw new UnsupportedOperationException("TODO: allow admin to modify prop: " + np.getKey()); - } - - // some props are directly in the node - if (VOS.PROPERTY_URI_FORMAT.equals(np.getKey()) - || VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { - throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); - } + // update existing data node + // need to remove all artifact props from the node.getProperties() + // and use artifactDAO to set the mutable ones + NodeProperty contentType = null; + NodeProperty contentEncoding = null; - // some props are directly in the artifact - if (directNodeProps.contains(np.getKey())) { - throw new UnsupportedOperationException("TODO: set mutable node prop: " + np.getKey()); - } - - // generic key-value props - if (node.properties.contains(np)) { - log.debug("updateProperties: remove previous " + np.getKey()); - node.properties.remove(np); + Iterator i = dn.getProperties().iterator(); + while (i.hasNext()) { + NodeProperty np = i.next(); + if (VOS.PROPERTY_URI_TYPE.equals(np.getKey())) { + contentType = np; + } else if (VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { + contentEncoding = np; + } + + if (artifactProps.contains(np.getKey())) { + i.remove(); + } } - if (!np.isMarkedForDeletion()) { - node.properties.add(np); + if (contentType != null || contentEncoding != null) { // optimization + ArtifactDAO artifactDAO = getArtifactDAO(); + Artifact a = artifactDAO.get(dn.storageID); + if (a != null) { + if (contentType != null) { + if (contentType.isMarkedForDeletion()) { + a.contentType = null; + } else { + a.contentType = contentType.getValue(); + } + } + if (contentEncoding != null) { + if (contentEncoding.isMarkedForDeletion()) { + a.contentEncoding = null; + } else { + a.contentEncoding = contentEncoding.getValue(); + } + } + artifactDAO.put(a); + } } } } @@ -323,7 +381,6 @@ public Node updateProperties(Node node, List props) throws Transie NodeDAO dao = getDAO(); dao.put(node); return node; - } /** @@ -332,23 +389,47 @@ public Node updateProperties(Node node, List props) throws Transie * @param node the node to delete * @throws TransientException */ + @Override public void delete(Node node) throws TransientException { if (node == null) { throw new IllegalArgumentException("arg cannot be null: node"); } NodeDAO dao = getDAO(); - boolean moveToTrash = false; + + // TODO: do the following in a transaction, acquire lock on target node + + boolean moveToTrash = true; // default + URI storageID = null; if (node instanceof ContainerNode) { ContainerNode cn = (ContainerNode) node; try (ResourceIterator iter = dao.iterator(cn, 1, null)) { - moveToTrash = !iter.hasNext(); + moveToTrash = iter.hasNext(); // empty } catch (IOException ex) { throw new TransientException("database IO failure", ex); } + } else if (node instanceof LinkNode) { + moveToTrash = false; + } else if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); + if (len == null) { + // artifact does not exist + moveToTrash = false; + } else { + storageID = dn.storageID; + } } - // TODO: DeletedNodeEvent + // TODO: create DeletedNodeEvent + // what about DNE for all child nodes, which also got deleted? or would sync of a + // DNE involve calling NodePersistence.delete(DeletedNodeEvent) to replay this same + // deletion logic in a mirror?? TBD + // persisting the DNE means that recovery from trash has to re-assign IDs + + DeletedNodeEvent dne = new DeletedNodeEvent(node.getID(), node.getClass(), storageID); + // TODO: need DeletedNodeDAO + if (moveToTrash) { node.parentID = trash.getID(); dao.put(node); @@ -358,4 +439,58 @@ public void delete(Node node) throws TransientException { } + + // this code is incomplete but shows the logic of merging the requested property changes + // into the node before put(node)... keeping it here for reference + public Node updateProperties(Node node, List props) + throws TransientException, NodeNotSupportedException { + if (node == null || props == null) { + throw new IllegalArgumentException("args cannot be null: node, props"); + } + + // merge props -> node and/or node.properties and/or mutable artifact + for (NodeProperty np : props) { + boolean writable = (node instanceof ContainerNode && this.rootAdminContainerProps.contains(np.getKey())); + writable = writable || rootAdminProps.contains(np.getKey()); + if (writable) { + // some props are directly in the node + if (artifactProps.contains(np.getKey())) { + throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); + } + + // some props are directly in the node + if (VOS.PROPERTY_URI_GROUPREAD.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_GROUPWRITE.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_INHERIT_PERMISSIONS.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_ISLOCKED.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } else if (VOS.PROPERTY_URI_ISPUBLIC.equals(np.getKey())) { + String raw = np.getValue(); + throw new UnsupportedOperationException(); + } + + // generic key-value props + if (node.getProperties().contains(np)) { + // remove previous; covers mark for deletion case + node.getProperties().remove(np); + } + + if (!np.isMarkedForDeletion()) { + // add new + node.getProperties().add(np); + } + } else { + log.debug("updateProperties: skip non-writable " + np.getKey()); + } + } + + return put(node); + } } From dc1e726de11a502612f8092fe3c9c6c43313e808 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 16 Jun 2023 10:40:52 +0800 Subject: [PATCH 019/186] Checkpoint --- vault/Dockerfile | 5 +- vault/build.gradle | 25 +++- .../opencadc/vault/NodePersistenceImpl.java | 6 +- vault/src/main/webapp/WEB-INF/web.xml | 117 +++++++++++------- vault/src/main/webapp/capabilities.xml | 11 ++ 5 files changed, 108 insertions(+), 56 deletions(-) diff --git a/vault/Dockerfile b/vault/Dockerfile index f3bc94674..4f35f9ff5 100644 --- a/vault/Dockerfile +++ b/vault/Dockerfile @@ -1,3 +1,4 @@ -FROM cadc-tomcat:1 +FROM images.opencadc.org/library/cadc-tomcat:1 + +COPY build/libs/vault.war /usr/share/tomcat/webapps/vault.war -COPY build/libs/vault.war /usr/share/tomcat/webapps/ diff --git a/vault/build.gradle b/vault/build.gradle index 94f622694..a07901381 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -24,21 +24,36 @@ war { include 'VERSION' } } +description = 'OpenCADC VOSpace server' +def git_url = 'https://github.com/opencadc/vos' dependencies { - compile 'org.opencadc:cadc-util:[1.9,2.0)' - compile 'org.opencadc:cadc-log:[1.1.6,2.0)' - compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' - compile 'org.opencadc:cadc-vos:[2.0,3.0)' - //compile 'org.opencadc:cadc-vos-server:[2.0,3.0)' + compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' + + compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-gms:[1.0.0,)' + compile 'org.opencadc:cadc-rest:[1.3.16,)' + compile 'org.opencadc:cadc-vos:[2.0,)' + compile 'org.opencadc:cadc-vos-server-2.0:[2.0,)' + compile 'org.opencadc:cadc-vosi:[1.3.2,)' + compile 'org.opencadc:cadc-uws:[1.0,)' + compile 'org.opencadc:cadc-uws-server:[1.2.12,)' + compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' + compile 'org.opencadc:cadc-cdp:[1.2.3,)' + compile 'org.opencadc:cadc-registry:[1.5.15,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' testCompile 'junit:junit:[4.0,)' + testCompile 'org.easymock:easymock:3.6' + testCompile 'org.skyscreamer:jsonassert:[1.0,)' runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' runtime 'org.opencadc:cadc-gms:[1.0.4,)' + // JDBC drivers + intTestRuntime 'org.postgresql:postgresql:[42.2.8,)' + intTestRuntime 'net.sourceforge.jtds:jtds:[1.0,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index afa32fc85..885ab980c 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -310,12 +310,12 @@ public Node updateProperties(Node node, List props) throws Transie } // generic key-value props - if (node.properties.contains(np)) { + if (node.getProperties().contains(np)) { log.debug("updateProperties: remove previous " + np.getKey()); - node.properties.remove(np); + node.getProperties().remove(np); } if (!np.isMarkedForDeletion()) { - node.properties.add(np); + node.getProperties().add(np); } } } diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 2d519aa70..d28360085 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -5,7 +5,6 @@ "http://java.sun.com/j2ee/dtds/web-app_2_3.dtd"> - vault @@ -20,6 +19,7 @@ org.opencadc.vault ca.nrc.cadc.db + org.opencadc.vospace ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi @@ -34,51 +34,71 @@ NodesServlet - ca.nrc.cadc.rest.RestServlet - - init - org.opencadc.vault.VaultInitAction - - 2 - + ca.nrc.cadc.rest.RestServlet + + augmentSubject + false + + + init + org.opencadc.vault.VaultInitAction + + + get + org.opencadc.vospace.server.web.actions.GetNodeAction + + + put + org.opencadc.vospace.server.web.actions.CreateNodeAction + + + post + org.opencadc.vospace.server.web.actions.UpdateNodeAction + + + delete + org.opencadc.vospace.server.web.actions.DeleteNodeAction + + 2 + + + + + CapabilitiesServlet + ca.nrc.cadc.rest.RestServlet + + init + ca.nrc.cadc.vosi.CapInitAction + + + head + ca.nrc.cadc.vosi.CapHeadAction + + + get + ca.nrc.cadc.vosi.CapGetAction + + + input + /capabilities.xml + + 3 + - - - CapabilitiesServlet - ca.nrc.cadc.rest.RestServlet - - init - ca.nrc.cadc.vosi.CapInitAction - - - head - ca.nrc.cadc.vosi.CapHeadAction - - - get - ca.nrc.cadc.vosi.CapGetAction - - - input - /capabilities.xml - - 3 - - - - - AvailabilityServlet - ca.nrc.cadc.vosi.AvailabilityServlet - - ca.nrc.cadc.vosi.AvailabilityPlugin - org.opencadc.vault.ServiceAvailability - - - availabilityProperties - vault-availability.properties - - 4 - + + + AvailabilityServlet + ca.nrc.cadc.vosi.AvailabilityServlet + + ca.nrc.cadc.vosi.AvailabilityPlugin + org.opencadc.vospace.ServiceAvailability + + + availabilityProperties + vault-availability.properties + + 4 + @@ -86,7 +106,7 @@ NodesServlet /nodes/* - + AvailabilityServlet @@ -105,4 +125,9 @@ /logControl + + index.html + + + diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index 0a6205003..9663ecec2 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -4,6 +4,16 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:vs="http://www.ivoa.net/xml/VODataService/v1.1"> + + + https://replace.com/vault/nodes + + + + + + + https://replace.com/vault/capabilities @@ -26,3 +36,4 @@ + From 6a8e3542d22f44b3519a66e8bb8f60c50c8f4444 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 29 Jun 2023 11:18:58 -0700 Subject: [PATCH 020/186] vault: add cadc-registry link to README --- vault/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vault/README.md b/vault/README.md index 67d202847..c4615fc99 100644 --- a/vault/README.md +++ b/vault/README.md @@ -28,9 +28,8 @@ for system properties related to the deployment environment. See cadc-util for common system properties. -`minoc` includes multiple IdentityManager implementations to support authenticated access: +`vault` includes multiple IdentityManager implementations to support authenticated access: - See cadc-access-control-identity for CADC access-control system support. - - See cadc-gms for OIDC token support. `vault` requires a connection pool to the local database: @@ -41,11 +40,15 @@ org.opencadc.vault.nodes.username={username for vospace pool} org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} ``` -The `nodes` account owns and manages (create, alter, drop) vospace database objects and manages +The _nodes_ account owns and manages (create, alter, drop) vospace database objects and manages all the content (insert, update, delete). The database is specified in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. +### cadc-registry.properties + +See cadc-registry. + ### vault.properties A vault.properties file in /config is required to run this service. The following keys are required: ``` From 5403a9a13d6302a6f2a05e4695e34a9eeb87169a Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 10 Jul 2023 16:30:45 -0700 Subject: [PATCH 021/186] cadc-inventory-db: node iterator bug fix --- .../org/opencadc/vospace/db/NodeDAOTest.java | 42 ++++++++++++++++++- .../opencadc/inventory/db/SQLGenerator.java | 2 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index b3aa5d1ef..ae9938908 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -601,8 +601,8 @@ public void testContainerNodeIterator() throws IOException { c2 = iter.next(); Assert.assertTrue(iter.hasNext()); c3 = iter.next(); + Assert.assertFalse(iter.hasNext()); } - // default order: alpha Assert.assertEquals(cont.getID(), c1.getID()); Assert.assertEquals(cont.getName(), c1.getName()); @@ -613,6 +613,46 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(link.getID(), c3.getID()); Assert.assertEquals(link.getName(), c3.getName()); + // iterate with limit + try (ResourceIterator iter = nodeDAO.iterator(orig, 2, null)) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c1 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(cont.getID(), c1.getID()); + Assert.assertEquals(cont.getName(), c1.getName()); + + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + // iterate with start + try (ResourceIterator iter = nodeDAO.iterator(orig, null, c2.getName())) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertTrue(iter.hasNext()); + c3 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + + Assert.assertEquals(link.getID(), c3.getID()); + Assert.assertEquals(link.getName(), c3.getName()); + + // iteratoe with limit and start + try (ResourceIterator iter = nodeDAO.iterator(orig, 1, c2.getName())) { + Assert.assertNotNull(iter); + Assert.assertTrue(iter.hasNext()); + c2 = iter.next(); + Assert.assertFalse(iter.hasNext()); + } + Assert.assertEquals(data.getID(), c2.getID()); + Assert.assertEquals(data.getName(), c2.getName()); + // depth first delete required but not enforced by DAO nodeDAO.delete(cont.getID()); nodeDAO.delete(data.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 97d27e985..131cb274b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -915,7 +915,7 @@ public ResourceIterator query(DataSource ds) { ps.setFetchDirection(ResultSet.FETCH_FORWARD); int col = 1; - ps.setObject(1, parent.getID()); + ps.setObject(col++, parent.getID()); log.debug("parentID = " + parent.getID()); if (start != null) { ps.setString(col++, start); From 5092f91c40fa72ce233b6c3c0ad12bd1a0db0fe1 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 10 Jul 2023 16:32:56 -0700 Subject: [PATCH 022/186] typo --- .../src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index ae9938908..1c41e5da4 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -643,7 +643,7 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(link.getID(), c3.getID()); Assert.assertEquals(link.getName(), c3.getName()); - // iteratoe with limit and start + // iterate with limit and start try (ResourceIterator iter = nodeDAO.iterator(orig, 1, c2.getName())) { Assert.assertNotNull(iter); Assert.assertTrue(iter.hasNext()); From a57647969af2e49ba89a8a9259ba14a7c967f6f5 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 11 Jul 2023 07:59:42 -0700 Subject: [PATCH 023/186] cadc-inventory-db: improve txn close in catch to avoid leak --- .../org/opencadc/vospace/db/NodeDAOTest.java | 4 ++-- .../org/opencadc/inventory/db/SQLGenerator.java | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 1c41e5da4..d079ba329 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -103,10 +103,10 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); - Log4jInit.setLevel("org.opencadc.inventory.db", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); - Log4jInit.setLevel("org.opencadc.vospace.db", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); } NodeDAO nodeDAO; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 131cb274b..40b35fd55 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -222,7 +222,7 @@ protected void init() { this.columnMap.put(HarvestState.class, cols); // optional vospace - log.warn("vosSchema: " + vosSchema); + log.debug("vosSchema: " + vosSchema); if (vosSchema != null) { pref = vosSchema + "."; tableMap.put(Node.class, pref + Node.class.getSimpleName()); @@ -1575,10 +1575,10 @@ public Artifact next() { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - close(); + con.setAutoCommit(false); hasRow = false; - } catch (IOException unexpected) { - log.debug("BUG: unexpected IOException from close", unexpected); + } catch (SQLException unexpected) { + log.error("unexpected SQLException trying to close txn", unexpected); } } throw new RuntimeException("BUG: artifact list query failed while iterating", ex); @@ -1609,7 +1609,7 @@ public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) @Override public void close() throws IOException { if (hasRow) { - log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); + log.debug("NodeResultSetIterator: " + super.toString() + " close - setAutoCommit(true)"); try { con.setAutoCommit(true); hasRow = false; @@ -1638,10 +1638,10 @@ public Node next() { if (hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - close(); + con.setAutoCommit(true); hasRow = false; - } catch (IOException unexpected) { - log.debug("BUG: unexpected IOException from close", unexpected); + } catch (SQLException unexpected) { + log.error("unexpected SQLException trying to close txn", unexpected); } } throw new RuntimeException("BUG: node list query failed while iterating", ex); From fa011e01440865beed7c7b7a02e9da61dec132d5 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 12 Jul 2023 18:11:58 +0300 Subject: [PATCH 024/186] Fixed small issues --- vault/src/main/java/org/opencadc/vault/VaultInitAction.java | 3 +-- vault/src/main/webapp/WEB-INF/web.xml | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index e65aa14c6..a6c2195ad 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -135,7 +135,6 @@ static MultiValuedProperties getConfig() { sb.append("incomplete config: "); boolean ok = true; - System.out.println("looking for " + RESOURCE_ID_KEY); String rid = mvp.getFirstPropertyValue(RESOURCE_ID_KEY); sb.append("\n\t" + RESOURCE_ID_KEY + ": "); if (rid == null) { @@ -223,7 +222,7 @@ protected void initNodePersistence() { try { ctx.unbind(jndiNodePersistence); } catch (NamingException ignore) { - log.debug("unbind previous JobManager failed... ignoring"); + log.debug("unbind previous JNDI key (" + jndiNodePersistence + ") failed... ignoring"); } NodePersistence npi = new NodePersistenceImpl(resourceID); ctx.bind(jndiNodePersistence, npi); diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 0759ae033..5d371d0b4 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -12,7 +12,7 @@ ca.nrc.cadc.log.LogControlServlet logLevel - debug + info logLevelPackages @@ -20,9 +20,7 @@ org.opencadc.vault ca.nrc.cadc.db org.opencadc.vospace - + ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi From 3fb92010039239d24f9453f654771ed497b9bdd6 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 12 Jul 2023 11:23:32 -0700 Subject: [PATCH 025/186] cadc-inventory-db: add connection.close() calls in NodeIterator to return con to pool also added to ArtifactIterator for completness, but disabled by default because they are currently used in a non-pool way (critwall, ratik, tantar) --- .../org/opencadc/vospace/db/NodeDAOTest.java | 12 ++-- .../opencadc/inventory/db/SQLGenerator.java | 67 ++++++++++++++----- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index d079ba329..ba8cc20d6 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -76,6 +76,7 @@ import java.net.URI; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.sql.Connection; import java.util.Map; import java.util.TreeMap; import java.util.UUID; @@ -104,7 +105,7 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); - Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.DEBUG); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); } @@ -115,11 +116,12 @@ public NodeDAOTest() throws Exception { try { DBConfig dbrc = new DBConfig(); ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); - DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest", cc); + DBUtil.PoolConfig pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); + DBUtil.createJNDIDataSource("jdbc/NodeDAOTest", pool); Map config = new TreeMap<>(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); - config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); + config.put("jndiDataSourceName", "jdbc/NodeDAOTest"); config.put("database", TestUtil.DATABASE); config.put("schema", TestUtil.SCHEMA); config.put("vosSchema", TestUtil.VOS_SCHEMA); @@ -145,7 +147,9 @@ public void init_cleanup() throws Exception { DataSource ds = nodeDAO.getDataSource(); String sql = "delete from " + gen.getTable(ContainerNode.class); log.info("pre-test cleanup: " + sql); - ds.getConnection().createStatement().execute(sql); + Connection con = ds.getConnection(); + con.createStatement().execute(sql); + con.close(); log.info("clearing old content... OK"); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 40b35fd55..8a22ddb8a 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1531,6 +1531,7 @@ private class ArtifactResultSetIterator implements ResourceIterator { private final Connection con; private final ResultSet rs; boolean hasRow; + boolean closeWhenDone = false; // not a pooled connection ArtifactResultSetIterator(Connection con, ResultSet rs) throws SQLException { this.con = con; @@ -1539,7 +1540,14 @@ private class ArtifactResultSetIterator implements ResourceIterator { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor " + hasRow); if (!hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); + } } } @@ -1548,10 +1556,13 @@ public void close() throws IOException { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); try { - con.setAutoCommit(true); - hasRow = false; - } catch (SQLException ex) { - throw new RuntimeException("BUG: artifact list query failed during close()", ex); + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + hasRow = false; + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); } } } @@ -1568,17 +1579,27 @@ public Artifact next() { hasRow = rs.next(); if (!hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } + } catch (SQLException unexpected) { + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); + } } return ret; } catch (Exception ex) { if (hasRow) { log.debug("ArtifactResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - con.setAutoCommit(false); + con.setAutoCommit(true); // commit txn + if (closeWhenDone) { + con.close(); // return to pool + } hasRow = false; } catch (SQLException unexpected) { - log.error("unexpected SQLException trying to close txn", unexpected); + log.error("Connection.setAutoCommit(true) & close() failed", unexpected); } } throw new RuntimeException("BUG: artifact list query failed while iterating", ex); @@ -1602,7 +1623,13 @@ public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) log.debug("NodeResultSetIterator: " + super.toString() + " ctor " + hasRow); if (!hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)"); - con.setAutoCommit(true); + + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } } } @@ -1611,10 +1638,11 @@ public void close() throws IOException { if (hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " close - setAutoCommit(true)"); try { - con.setAutoCommit(true); + con.setAutoCommit(true); // commit txn + con.close(); // return to pool hasRow = false; - } catch (SQLException ex) { - throw new RuntimeException("BUG: node list query failed during close()", ex); + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); } } } @@ -1631,17 +1659,24 @@ public Node next() { hasRow = rs.next(); if (!hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)"); - con.setAutoCommit(true); + try { + con.setAutoCommit(true); // commit txn + con.close(); // return to pool + hasRow = false; + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); + } } return ret; } catch (Exception ex) { if (hasRow) { log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)"); try { - con.setAutoCommit(true); + con.setAutoCommit(true); // commit txn + con.close(); // return to pool hasRow = false; - } catch (SQLException unexpected) { - log.error("unexpected SQLException trying to close txn", unexpected); + } catch (SQLException ignore) { + log.error("Connection.setAutoCommit(true) & close() failed", ignore); } } throw new RuntimeException("BUG: node list query failed while iterating", ex); From ed3fa618ad63669b0897ac879e6d7cf6128267a4 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 12 Jul 2023 11:43:29 -0700 Subject: [PATCH 026/186] debug->info in test code --- .../src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index ba8cc20d6..e5fedfb54 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -105,7 +105,7 @@ public class NodeDAOTest { static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); - Log4jInit.setLevel("ca.nrc.cadc.db", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); } From bf26dde87f5f85691fce7512fda06ae2d9138364 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 12 Jul 2023 11:47:19 -0700 Subject: [PATCH 027/186] raven: limit cadc-inventory-db version to avoid cadc-vos-2.0 until ready to adapt --- raven/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/raven/build.gradle b/raven/build.gradle index 1026ab9ed..6d2cbba4e 100644 --- a/raven/build.gradle +++ b/raven/build.gradle @@ -34,7 +34,8 @@ dependencies { compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-gms:[1.0.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,1.0)' + // temporarily limit db lib + compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15.0)' compile 'org.opencadc:cadc-inventory-server:[0.2.1,)' compile 'org.opencadc:cadc-permissions:[0.3.1,)' compile 'org.opencadc:cadc-permissions-client:[0.3,)' From ce0898f62fe9272d7e116e0895b54f78214c0d16 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 12 Jul 2023 23:04:42 +0300 Subject: [PATCH 028/186] Re-factored the library --- vault/src/main/webapp/WEB-INF/web.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 5d371d0b4..a671c0448 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -12,7 +12,7 @@ ca.nrc.cadc.log.LogControlServlet logLevel - info + debug logLevelPackages @@ -45,19 +45,19 @@ get - org.opencadc.vospace.server.web.actions.GetNodeAction + org.opencadc.vospace.server.actions.GetNodeAction put - org.opencadc.vospace.server.web.actions.CreateNodeAction + org.opencadc.vospace.server.actions.CreateNodeAction post - org.opencadc.vospace.server.web.actions.UpdateNodeAction + org.opencadc.vospace.server.actions.UpdateNodeAction delete - org.opencadc.vospace.server.web.actions.DeleteNodeAction + org.opencadc.vospace.server.actions.DeleteNodeAction 2 From 5b4e2727b08e042552f413e0c2eb567f1e8ea798 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 21:10:36 +0300 Subject: [PATCH 029/186] Small fixes --- vault/build.gradle | 4 ---- .../src/main/java/org/opencadc/vault/NodePersistenceImpl.java | 3 +-- vault/src/main/webapp/WEB-INF/web.xml | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index bf307a578..96ecd7f1c 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -46,15 +46,11 @@ dependencies { compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' testCompile 'junit:junit:[4.0,)' - testCompile 'org.easymock:easymock:3.6' - testCompile 'org.skyscreamer:jsonassert:[1.0,)' runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' runtime 'org.opencadc:cadc-gms:[1.0.4,)' // JDBC drivers - intTestRuntime 'org.postgresql:postgresql:[42.2.8,)' - intTestRuntime 'net.sourceforge.jtds:jtds:[1.0,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 80e809d9d..d36965c0b 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -81,7 +81,6 @@ import java.net.URI; import java.security.Principal; import java.text.DateFormat; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -147,7 +146,7 @@ public NodePersistenceImpl(URI resourceID) { Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { @Override public Set getPrincipals() { - Set ret = new HashSet<>(); + Set ret = new TreeSet<>(); ret.add(new X500Principal(owner)); return ret; } diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index a671c0448..af0394c10 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -12,7 +12,7 @@ ca.nrc.cadc.log.LogControlServlet logLevel - debug + info logLevelPackages From 1c885d7dc96810c39e7191701b5a27469f1da9e6 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 21:11:41 +0300 Subject: [PATCH 030/186] Small fixes --- vault/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/vault/build.gradle b/vault/build.gradle index 96ecd7f1c..787a86b9b 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -50,7 +50,6 @@ dependencies { runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' runtime 'org.opencadc:cadc-gms:[1.0.4,)' - // JDBC drivers intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' } From a2c63e8f83b36693d4d667e635af2126945e4b41 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 22:23:17 +0300 Subject: [PATCH 031/186] Renamed cadc-vos-server dependency --- vault/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault/build.gradle b/vault/build.gradle index 787a86b9b..eb52ef101 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -35,7 +35,7 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' - compile 'org.opencadc:cadc-vos-server-2.0:[2.0,)' + compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' compile 'org.opencadc:cadc-uws-server:[1.2.12,)' From d364005888ec288e653c32c856765a031e9ea62d Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 13 Jul 2023 23:41:37 +0300 Subject: [PATCH 032/186] Fixed bug --- .../src/main/java/org/opencadc/vault/NodePersistenceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index d36965c0b..80e809d9d 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -81,6 +81,7 @@ import java.net.URI; import java.security.Principal; import java.text.DateFormat; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -146,7 +147,7 @@ public NodePersistenceImpl(URI resourceID) { Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { @Override public Set getPrincipals() { - Set ret = new TreeSet<>(); + Set ret = new HashSet<>(); ret.add(new X500Principal(owner)); return ret; } From e6ca6ec93970a5cf63342c95f792a852e255fda9 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:16:45 -0700 Subject: [PATCH 033/186] vault: added NodesTest integration test using cadc-test-vos-2.0 --- vault/build.gradle | 1 + .../java/org/opencadc/vault/NodesTest.java | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 vault/src/intTest/java/org/opencadc/vault/NodesTest.java diff --git a/vault/build.gradle b/vault/build.gradle index eb52ef101..ca56af16c 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -51,6 +51,7 @@ dependencies { runtime 'org.opencadc:cadc-gms:[1.0.4,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' + intTestCompile 'org.opencadc:cadc-test-vos:[2.0,3.0)' } configurations { diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java new file mode 100644 index 000000000..a8afb4078 --- /dev/null +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -0,0 +1,82 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import org.apache.log4j.Logger; + +/** + * Test the nodes endpoint. + * + * @author pdowler + */ +public class NodesTest extends org.opencadc.conformance.vos.NodesTest { + private static final Logger log = Logger.getLogger(NodesTest.class); + + public NodesTest() { + } +} From 2b45eaa068f05e10a3cd513bd80372ad18739937 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:17:22 -0700 Subject: [PATCH 034/186] vault: comment out unimplemented endpoints from capabilities --- vault/src/main/webapp/capabilities.xml | 50 ++++++++++---------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index f36f7a0f1..fb145630e 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -23,11 +23,13 @@ + @@ -37,12 +39,9 @@ - - https://replace.me.com/vault/auth/nodes - - + + + + + + + + From 3efead89371a5415b9edfe0c875976527f390522 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:18:09 -0700 Subject: [PATCH 035/186] vault: tweak root and trash node settings in NodePersistenceImpl --- .../main/java/org/opencadc/vault/NodePersistenceImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 80e809d9d..d15725966 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -162,7 +162,11 @@ public X509CertificateChain getCertificateChain() { UUID rootID = new UUID(0L, 0L); this.root = new ContainerNode(rootID, "", false); root.owner = im.augment(rawOwnerSubject); - root.ownerID = im.toOwner(root.owner); + log.warn("ROOT owner: " + root.owner); + root.ownerID = im.toOwner(rawOwnerSubject); + log.warn("ROOT ownerID: " + root.ownerID + " rtype: " + root.ownerID.getClass().getName()); + root.isPublic = true; + root.inheritPermissions = false; // trash node // TODO: do this setup in a txn with a lock on something @@ -175,6 +179,7 @@ public X509CertificateChain getCertificateChain() { tn.ownerID = root.ownerID; tn.owner = root.owner; tn.isPublic = false; + tn.inheritPermissions = false; tn.parentID = rootID; dao.put(tn); this.trash = tn; From 60651cb494b204578d52a42c151b54ab35a0086c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 12:25:10 -0700 Subject: [PATCH 036/186] vault: update dependencies --- vault/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index ca56af16c..86b736b0c 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -32,7 +32,7 @@ dependencies { compile 'org.opencadc:cadc-util:[1.9.5,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' - compile 'org.opencadc:cadc-gms:[1.0.0,)' + compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' @@ -47,8 +47,8 @@ dependencies { testCompile 'junit:junit:[4.0,)' - runtime 'org.opencadc:cadc-access-control-identity:[1.2.0,)' - runtime 'org.opencadc:cadc-gms:[1.0.4,)' + runtime 'org.opencadc:cadc-access-control-identity:[1.2.1,)' + runtime 'org.opencadc:cadc-gms:[1.0.5,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' intTestCompile 'org.opencadc:cadc-test-vos:[2.0,3.0)' From 0531edc6618e1afa566b30668fc758a320e4d577 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 4 Aug 2023 14:26:17 -0700 Subject: [PATCH 037/186] vault: cleanup in NodePersistenceImpl --- .../opencadc/vault/NodePersistenceImpl.java | 81 +------------------ 1 file changed, 4 insertions(+), 77 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index d15725966..0d63898a8 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -119,10 +119,7 @@ public class NodePersistenceImpl implements NodePersistence { private final ContainerNode trash; private final Namespace storageNamespace; private final Set immutableProps = new TreeSet<>(); - private final Set artifactProps = new TreeSet<>(); - private final Set rootAdminContainerProps = new TreeSet<>(); - private final Set rootAdminProps = new TreeSet<>(); private URI resourceID; public NodePersistenceImpl(URI resourceID) { @@ -191,10 +188,10 @@ public X509CertificateChain getCertificateChain() { // VOS.PROPERTY_URI_AVAILABLESPACE // container nodes // VOS.PROPERTY_URI_WRITABLE // prediction for current caller - // props only the admin (root owner) can modify - rootAdminProps.add(VOS.PROPERTY_URI_CREATOR); // owner - rootAdminContainerProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // container nodes - rootAdminContainerProps.add(VOS.PROPERTY_URI_QUOTA); // container nodes + // props only the admin (root owner) can modify? + // VOS.PROPERTY_URI_CREATOR // owner + // VOS.PROPERTY_URI_CONTENTLENGTH // container nodes + // VOS.PROPERTY_URI_QUOTA // container nodes // node properties that match immutable Artifact fields immutableProps.add(VOS.PROPERTY_URI_CONTENTLENGTH); // immutable @@ -457,74 +454,4 @@ public void delete(Node node) throws TransientException { } } - - - // this code is incomplete but shows the logic of merging the requested property changes - // into the node before put(node)... keeping it here for reference - public Node updateProperties(Node node, List props) - throws TransientException, NodeNotSupportedException { - if (node == null || props == null) { - throw new IllegalArgumentException("args cannot be null: node, props"); - } - - // merge props -> node and/or node.properties and/or mutable artifact - for (NodeProperty np : props) { - boolean writable = (node instanceof ContainerNode && this.rootAdminContainerProps.contains(np.getKey())); - writable = writable || rootAdminProps.contains(np.getKey()); - if (writable) { - // some props are directly in the node - if (artifactProps.contains(np.getKey())) { - throw new UnsupportedOperationException("TODO: set mutable artifact prop: " + np.getKey()); - } - - // some props are directly in the node - if (VOS.PROPERTY_URI_GROUPREAD.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_GROUPWRITE.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_INHERIT_PERMISSIONS.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_ISLOCKED.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } else if (VOS.PROPERTY_URI_ISPUBLIC.equals(np.getKey())) { - String raw = np.getValue(); - throw new UnsupportedOperationException(); - } - - // generic key-value props - if (node.getProperties().contains(np)) { - // remove previous; covers mark for deletion case - node.getProperties().remove(np); - } - - if (!np.isMarkedForDeletion()) { - // add new - node.getProperties().add(np); - } - } else { - log.debug("updateProperties: skip non-writable " + np.getKey()); - } - } - - return put(node); - } - - @Override - public void setFileMetadata(DataNode dataNode, FileMetadata fileMetadata, boolean b) throws TransientException { - - } - - @Override - public void move(Node node, ContainerNode containerNode) throws TransientException { - - } - - @Override - public void copy(Node node, ContainerNode containerNode) throws TransientException { - - } } From a79673b6aa1340c393246162f62c96b72865e349 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 8 Aug 2023 16:38:28 -0700 Subject: [PATCH 038/186] move assign of owner and parent details to NodePersistenceImpl --- .../org/opencadc/vospace/db/NodeDAOTest.java | 14 +++-- .../opencadc/inventory/db/SQLGenerator.java | 7 --- .../opencadc/vault/NodePersistenceImpl.java | 53 +++++++++++++++++-- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index e5fedfb54..24291b403 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -199,8 +199,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); Assert.assertEquals(root.getID(), a.parentID); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); + Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent Assert.assertEquals(orig.getName(), a.getName()); @@ -290,6 +290,7 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException log.info("found by id: " + a.getID() + " aka " + a); Assert.assertEquals(orig.getID(), a.getID()); Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertNotNull(a.parentID); Assert.assertEquals(root.getID(), a.parentID); // get-by-path @@ -298,8 +299,7 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException log.info("found by path: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent @@ -389,8 +389,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, log.info("found: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent @@ -481,8 +480,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, log.info("found: " + aa.getID() + " aka " + aa); Assert.assertEquals(orig.getID(), aa.getID()); Assert.assertEquals(orig.getName(), aa.getName()); - Assert.assertNotNull(aa.parent); - Assert.assertEquals(root.getID(), aa.parent.getID()); + Assert.assertNotNull(aa.parentID); Assert.assertEquals(root.getID(), aa.parentID); Assert.assertNull(a.parent); // get-node-by-id: comes pack without parent diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 8a22ddb8a..bd5b2eb42 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1764,13 +1764,6 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro InventoryUtil.assignLastModified(ret, lastModified); InventoryUtil.assignMetaChecksum(ret, metaChecksum); - if (parent != null) { - if (!parent.getID().equals(parentID)) { - throw new RuntimeException("BUG: expected parentID=" + parent.getID() + " but got: " + parentID); - } - ret.parent = parent; - } - return ret; } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0d63898a8..ab960c6fc 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -67,6 +67,7 @@ package org.opencadc.vault; +import sun.rmi.rmic.IndentingWriter; import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.IdentityManager; import ca.nrc.cadc.auth.PrincipalExtractor; @@ -122,6 +123,8 @@ public class NodePersistenceImpl implements NodePersistence { private final Set artifactProps = new TreeSet<>(); private URI resourceID; + private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + public NodePersistenceImpl(URI resourceID) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); @@ -140,7 +143,6 @@ public NodePersistenceImpl(URI resourceID) { if (owner == null) { throw new InvalidConfigException(VaultInitAction.ROOT_OWNER + " cannot be null"); } - IdentityManager im = AuthenticationUtil.getIdentityManager(); Subject rawOwnerSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { @Override public Set getPrincipals() { @@ -158,9 +160,10 @@ public X509CertificateChain getCertificateChain() { // root node UUID rootID = new UUID(0L, 0L); this.root = new ContainerNode(rootID, "", false); - root.owner = im.augment(rawOwnerSubject); + root.owner = identityManager.augment(rawOwnerSubject); + root.ownerDisplay = identityManager.toDisplayString(root.owner); log.warn("ROOT owner: " + root.owner); - root.ownerID = im.toOwner(rawOwnerSubject); + root.ownerID = identityManager.toOwner(rawOwnerSubject); log.warn("ROOT ownerID: " + root.ownerID + " rtype: " + root.ownerID.getClass().getName()); root.isPublic = true; root.inheritPermissions = false; @@ -260,6 +263,10 @@ public Node get(ContainerNode parent, String name) throws TransientException { } NodeDAO dao = getDAO(); Node ret = dao.get(parent, name); + ret.parent = parent; + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + if (ret instanceof DataNode) { DataNode dn = (DataNode) ret; ArtifactDAO artifactDAO = getArtifactDAO(); @@ -299,7 +306,38 @@ public ResourceIterator iterator(ContainerNode parent, Integer limit, Stri } NodeDAO dao = getDAO(); ResourceIterator ret = dao.iterator(parent, limit, start); - return ret; + return new IdentWrapper(parent, ret); + } + + private class IdentWrapper implements ResourceIterator { + + private final ContainerNode parent; + private final ResourceIterator childIter; + + IdentWrapper(ContainerNode parent, ResourceIterator childIter) { + this.parent = parent; + this.childIter = childIter; + } + + @Override + public boolean hasNext() { + return childIter.hasNext(); + } + + @Override + public Node next() { + Node ret = childIter.next(); + ret.parent = parent; + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + return ret; + } + + @Override + public void close() throws IOException { + childIter.close(); + } + } /** @@ -336,6 +374,13 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException } node.parentID = node.parent.getID(); } + if (node.ownerID == null) { + if (node.owner == null) { + throw new RuntimeException("BUG: cannot persist node without owner: " + node); + } + node.ownerID = identityManager.toOwner(node.owner); + } + if (node instanceof DataNode) { DataNode dn = (DataNode) node; if (dn.storageID == null) { From 4f455a8ad816e7a08582ab29b3a5cfa9370f038f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 9 Aug 2023 15:58:06 -0700 Subject: [PATCH 039/186] vault: formatting --- .../org/opencadc/vault/VaultInitAction.java | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index a6c2195ad..bc9dc89f5 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -63,7 +63,7 @@ * . * ************************************************************************ -*/ + */ package org.opencadc.vault; @@ -89,8 +89,9 @@ * @author pdowler */ public class VaultInitAction extends InitAction { + private static final Logger log = Logger.getLogger(VaultInitAction.class); - + static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml // config keys @@ -98,15 +99,15 @@ public class VaultInitAction extends InitAction { static final String RESOURCE_ID_KEY = VAULT_KEY + ".resourceID"; static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; - + static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? - + static final String STORAGE_NAMESPACE_KEY = VAULT_KEY + ".storage.namespace"; - + MultiValuedProperties props; private URI resourceID; private Namespace storageNamespace; - private Map daoConfig; + private Map daoConfig; private String jndiNodePersistence; @@ -120,10 +121,10 @@ public void doInit() { initDatabase(); initNodePersistence(); } - + /** * Read config file and verify that all required entries are present. - * + * * @return MultiValuedProperties containing the application config * @throws IllegalStateException if required config items are missing */ @@ -152,7 +153,7 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } - + String vosSchema = mvp.getFirstPropertyValue(VOSPACE_SCHEMA_KEY); sb.append("\n\t").append(VOSPACE_SCHEMA_KEY).append(": "); if (vosSchema == null) { @@ -161,7 +162,7 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } - + String ns = mvp.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); sb.append("\n\t").append(STORAGE_NAMESPACE_KEY).append(": "); if (ns == null) { @@ -170,22 +171,22 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } - + if (!ok) { throw new IllegalStateException(sb.toString()); } return mvp; } - - static Map getDaoConfig(MultiValuedProperties props) { + + static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); return ret; } - + private void initConfig() { log.info("initConfig: START"); this.props = getConfig(); @@ -200,7 +201,7 @@ private void initConfig() { throw new IllegalStateException("invalid config: " + RESOURCE_ID_KEY + " must be a valid URI"); } } - + private void initDatabase() { log.info("initDatabase: START"); try { @@ -216,8 +217,7 @@ private void initDatabase() { protected void initNodePersistence() { jndiNodePersistence = componentID + ".nodePersistence"; - try - { + try { Context ctx = new InitialContext(); try { ctx.unbind(jndiNodePersistence); @@ -227,15 +227,14 @@ protected void initNodePersistence() { NodePersistence npi = new NodePersistenceImpl(resourceID); ctx.bind(jndiNodePersistence, npi); - log.info("created JNDI key" + jndiNodePersistence + " "); - } catch(Exception ex) { + log.info("created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); + } catch (Exception ex) { log.error("Failed to create JNDI Key " + jndiNodePersistence, ex); } } @Override - public void doShutdown() - { + public void doShutdown() { try { Context ctx = new InitialContext(); ctx.unbind(jndiNodePersistence); From 679b3ee488abfce249f0c010e6d4835d8ff8c197 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 11 Aug 2023 16:02:50 -0700 Subject: [PATCH 040/186] cadc-inventory-db: adapt to ContainerNode api change --- .../org/opencadc/vospace/db/NodeDAOTest.java | 23 ++++++++++--------- .../opencadc/inventory/db/SQLGenerator.java | 4 +++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 24291b403..230c70392 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -162,12 +162,12 @@ public void testGetByID_NotFound() { @Test public void testGetByPath_NotFound() { - ContainerNode parent = new ContainerNode("not-found", false); + ContainerNode parent = new ContainerNode("not-found"); Node a = nodeDAO.get(parent, "not-found"); Assert.assertNull(a); UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); a = nodeDAO.get(root, "not-found"); Assert.assertNull(a); } @@ -176,10 +176,10 @@ public void testGetByPath_NotFound() { public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); // put - ContainerNode orig = new ContainerNode("container-test", false); + ContainerNode orig = new ContainerNode("container-test"); orig.parent = root; orig.ownerID = "the-owner"; nodeDAO.put(orig); @@ -228,6 +228,7 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.isPublic = true; + orig.inheritPermissions = true; nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); Assert.assertNotNull(updated); @@ -259,11 +260,11 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); // TODO: use get-by-path to find and remove the test node - ContainerNode orig = new ContainerNode("container-test", false); + ContainerNode orig = new ContainerNode("container-test"); orig.parent = root; orig.ownerID = "the-owner"; orig.isPublic = true; @@ -363,7 +364,7 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException public void testPutGetUpdateDeleteDataNode() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); orig.parent = root; @@ -454,7 +455,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, NoSuchAlgorithmException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); // TODO: use get-by-path to find and remove the test node @@ -543,9 +544,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, @Test public void testContainerNodeIterator() throws IOException { UUID rootID = new UUID(0L, 0L); - ContainerNode root = new ContainerNode(rootID, "root", false); + ContainerNode root = new ContainerNode(rootID, "root"); - ContainerNode orig = new ContainerNode("container-test", false); + ContainerNode orig = new ContainerNode("container-test"); orig.parent = root; orig.ownerID = "the-owner"; nodeDAO.put(orig); @@ -576,7 +577,7 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(orig.getID(), top.getID()); // add children - ContainerNode cont = new ContainerNode("container1", false); + ContainerNode cont = new ContainerNode("container1"); cont.parent = orig; cont.ownerID = orig.ownerID; DataNode data = new DataNode(UUID.randomUUID(), "data1", URI.create("cadc:vault/" + UUID.randomUUID())); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index bd5b2eb42..76baf96de 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -1738,7 +1738,9 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro Node ret; if (nodeType.equals("C")) { - ret = new ContainerNode(id, name, inheritPermissions); + ContainerNode cn = new ContainerNode(id, name); + cn.inheritPermissions = inheritPermissions; + ret = cn; } else if (nodeType.equals("D")) { ret = new DataNode(id, name, storageID); } else if (nodeType.equals("L")) { From 62645da74dc390200826c3b6f486420a79bcd298 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 11 Aug 2023 16:03:17 -0700 Subject: [PATCH 041/186] vault: quick hack fix for delete(Node) to allow repeated create/delete to move to trash --- .../java/org/opencadc/vault/NodesTest.java | 11 ++++++++++- .../org/opencadc/vault/NodePersistenceImpl.java | 17 ++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java index a8afb4078..71bb1178b 100644 --- a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -67,6 +67,9 @@ package org.opencadc.vault; +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import org.apache.log4j.Level; import org.apache.log4j.Logger; /** @@ -77,6 +80,12 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest { private static final Logger log = Logger.getLogger(NodesTest.class); - public NodesTest() { + static { + Log4jInit.setLevel("org.opencadc.conformance.vos", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); + } + + public NodesTest() { + super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem"); } } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index ab960c6fc..07cfde569 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -159,12 +159,11 @@ public X509CertificateChain getCertificateChain() { // root node UUID rootID = new UUID(0L, 0L); - this.root = new ContainerNode(rootID, "", false); + this.root = new ContainerNode(rootID, ""); root.owner = identityManager.augment(rawOwnerSubject); root.ownerDisplay = identityManager.toDisplayString(root.owner); log.warn("ROOT owner: " + root.owner); root.ownerID = identityManager.toOwner(rawOwnerSubject); - log.warn("ROOT ownerID: " + root.ownerID + " rtype: " + root.ownerID.getClass().getName()); root.isPublic = true; root.inheritPermissions = false; @@ -173,7 +172,7 @@ public X509CertificateChain getCertificateChain() { NodeDAO dao = getDAO(); ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); if (tn == null) { - tn = new ContainerNode(".trash", false); + tn = new ContainerNode(".trash"); } // always reset props to current config tn.ownerID = root.ownerID; @@ -263,6 +262,9 @@ public Node get(ContainerNode parent, String name) throws TransientException { } NodeDAO dao = getDAO(); Node ret = dao.get(parent, name); + if (ret == null) { + return null; + } ret.parent = parent; ret.owner = identityManager.toSubject(ret.ownerID); ret.ownerDisplay = identityManager.toDisplayString(ret.owner); @@ -482,17 +484,18 @@ public void delete(Node node) throws TransientException { } } - // TODO: create DeletedNodeEvent + // TODO: create DeletedNodeEvent? + // need DeletedNodeDAO // what about DNE for all child nodes, which also got deleted? or would sync of a // DNE involve calling NodePersistence.delete(DeletedNodeEvent) to replay this same // deletion logic in a mirror?? TBD // persisting the DNE means that recovery from trash has to re-assign IDs - - DeletedNodeEvent dne = new DeletedNodeEvent(node.getID(), node.getClass(), storageID); - // TODO: need DeletedNodeDAO + + // TODO: if DataNode (storageID != null): delete artifact and create DeletedArtifactEvent? if (moveToTrash) { node.parentID = trash.getID(); + node.setName(node.getName() + "-" + UUID.randomUUID().toString()); dao.put(node); } else { dao.delete(node.getID()); From 8962bde9fe15a66b8316ea4d227d5f43efa288a0 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 15 Aug 2023 11:19:16 -0700 Subject: [PATCH 042/186] vault: fix imports --- .../src/main/java/org/opencadc/vault/NodePersistenceImpl.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 07cfde569..f89d04c57 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -67,7 +67,6 @@ package org.opencadc.vault; -import sun.rmi.rmic.IndentingWriter; import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.IdentityManager; import ca.nrc.cadc.auth.PrincipalExtractor; @@ -75,7 +74,6 @@ import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; -import ca.nrc.cadc.util.FileMetadata; import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; @@ -84,7 +82,6 @@ import java.text.DateFormat; import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -99,7 +96,6 @@ import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.DeletedNodeEvent; import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotSupportedException; From 0420a6b718511513bafa0ee8fc9ef2f8c372eacd Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 23 Aug 2023 11:18:10 -0700 Subject: [PATCH 043/186] vault: added indentity cache to child iterator --- .../org/opencadc/vault/NodePersistenceImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index f89d04c57..bfdb7ba04 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -311,10 +311,15 @@ private class IdentWrapper implements ResourceIterator { private final ContainerNode parent; private final ResourceIterator childIter; + private Map identCache = new TreeMap<>(); IdentWrapper(ContainerNode parent, ResourceIterator childIter) { this.parent = parent; this.childIter = childIter; + // prime cache with caller + Subject caller = AuthenticationUtil.getCurrentSubject(); + Object ownerID = identityManager.toOwner(caller); + identCache.put(ownerID, caller); } @Override @@ -326,7 +331,12 @@ public boolean hasNext() { public Node next() { Node ret = childIter.next(); ret.parent = parent; - ret.owner = identityManager.toSubject(ret.ownerID); + Subject s = identCache.get(ret.ownerID); + if (s == null) { + s = identityManager.toSubject(ret.ownerID); + identCache.put(ret.ownerID, s); + } + ret.owner = s; ret.ownerDisplay = identityManager.toDisplayString(ret.owner); return ret; } @@ -334,6 +344,7 @@ public Node next() { @Override public void close() throws IOException { childIter.close(); + identCache.clear(); } } From 1c269619c44065be7195c36aac48a780b6cb5c4e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 23 Aug 2023 13:17:22 -0700 Subject: [PATCH 044/186] vault: fix class cast issue with ident cache in iterator delete(Node) now rejects deleting non-empty container untested: delete(node) also delete Artifact and generate DeletedArtifactEvent --- .../opencadc/vault/NodePersistenceImpl.java | 64 +++++++++++-------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index bfdb7ba04..5484d14dd 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -91,8 +91,10 @@ import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; @@ -119,8 +121,6 @@ public class NodePersistenceImpl implements NodePersistence { private final Set artifactProps = new TreeSet<>(); private URI resourceID; - private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); - public NodePersistenceImpl(URI resourceID) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); @@ -152,6 +152,7 @@ public X509CertificateChain getCertificateChain() { return null; } }); + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); // root node UUID rootID = new UUID(0L, 0L); @@ -262,9 +263,12 @@ public Node get(ContainerNode parent, String name) throws TransientException { return null; } ret.parent = parent; + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); ret.owner = identityManager.toSubject(ret.ownerID); ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + // in principle we could have queried vospace.Node join inventory.Artifact above + // and avoid this query.... simplicity for now if (ret instanceof DataNode) { DataNode dn = (DataNode) ret; ArtifactDAO artifactDAO = getArtifactDAO(); @@ -311,6 +315,8 @@ private class IdentWrapper implements ResourceIterator { private final ContainerNode parent; private final ResourceIterator childIter; + + private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); private Map identCache = new TreeMap<>(); IdentWrapper(ContainerNode parent, ResourceIterator childIter) { @@ -318,8 +324,14 @@ private class IdentWrapper implements ResourceIterator { this.childIter = childIter; // prime cache with caller Subject caller = AuthenticationUtil.getCurrentSubject(); - Object ownerID = identityManager.toOwner(caller); - identCache.put(ownerID, caller); + if (caller != null) { + Object ownerID = identityManager.toOwner(caller); + if (ownerID != null) { + // HACK: NodeDAO returns ownerID as String and relies on the IM + // to convert to a number (eg) + identCache.put(ownerID.toString(), caller); + } + } } @Override @@ -387,6 +399,7 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException if (node.owner == null) { throw new RuntimeException("BUG: cannot persist node without owner: " + node); } + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); node.ownerID = identityManager.toOwner(node.owner); } @@ -465,48 +478,43 @@ public void delete(Node node) throws TransientException { throw new IllegalArgumentException("arg cannot be null: node"); } - NodeDAO dao = getDAO(); + final NodeDAO dao = getDAO(); + final ArtifactDAO artifactDAO = getArtifactDAO(); // TODO: do the following in a transaction, acquire lock on target node - boolean moveToTrash = true; // default URI storageID = null; if (node instanceof ContainerNode) { ContainerNode cn = (ContainerNode) node; try (ResourceIterator iter = dao.iterator(cn, 1, null)) { - moveToTrash = iter.hasNext(); // empty + if (iter.hasNext()) { + throw new IllegalArgumentException("container node '" + node.getName() + "' is not empty"); + } } catch (IOException ex) { throw new TransientException("database IO failure", ex); } - } else if (node instanceof LinkNode) { - moveToTrash = false; } else if (node instanceof DataNode) { DataNode dn = (DataNode) node; NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - if (len == null) { - // artifact does not exist - moveToTrash = false; - } else { + if (len != null) { + // artifact exists storageID = dn.storageID; } - } + } // else: LinkNode can always be deleted - // TODO: create DeletedNodeEvent? - // need DeletedNodeDAO - // what about DNE for all child nodes, which also got deleted? or would sync of a - // DNE involve calling NodePersistence.delete(DeletedNodeEvent) to replay this same - // deletion logic in a mirror?? TBD - // persisting the DNE means that recovery from trash has to re-assign IDs - - // TODO: if DataNode (storageID != null): delete artifact and create DeletedArtifactEvent? + // TODO: need DeletedNodeDAO to create DeletedNodeEvent - if (moveToTrash) { - node.parentID = trash.getID(); - node.setName(node.getName() + "-" + UUID.randomUUID().toString()); - dao.put(node); - } else { - dao.delete(node.getID()); + if (storageID != null) { + Artifact a = artifactDAO.get(storageID); + if (a != null) { + DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); + DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); + daeDAO.put(dae); + artifactDAO.delete(a.getID()); + } } + dao.delete(node.getID()); + // TODO: commit transaction } } From e6ef466abc65630e8bc56635ae58c813cbe45538 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Wed, 6 Sep 2023 14:10:22 -0700 Subject: [PATCH 045/186] Changed type of group sets in Node to GroupURI --- cadc-inventory-db/build.gradle | 1 + .../org/opencadc/vospace/db/NodeDAOTest.java | 32 +++++++++---------- .../opencadc/inventory/db/SQLGenerator.java | 16 ++++------ .../java/org/opencadc/inventory/db/Util.java | 11 +++---- .../org/opencadc/inventory/db/UtilTest.java | 13 ++++---- 5 files changed, 34 insertions(+), 39 deletions(-) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 2f1566ca2..d96867abb 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -26,6 +26,7 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' compile 'org.opencadc:cadc-vos:[2.0,3.0)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 230c70392..10c8e9e46 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -86,6 +86,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.opencadc.gms.GroupURI; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.TestUtil; import org.opencadc.vospace.ContainerNode; @@ -224,8 +225,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, // update Thread.sleep(10L); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.isPublic = true; orig.inheritPermissions = true; @@ -270,14 +271,13 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.isPublic = true; orig.isLocked = false; orig.inheritPermissions = false; - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g2")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g4,g5")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6-g7")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6.g7")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6_g7")); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g6~g7")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g2"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6-g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6.g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6_g7"))); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6~g7"))); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.getProperties().add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); @@ -328,9 +328,9 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.isPublic = false; orig.isLocked = true; orig.getReadOnlyGroup().clear(); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); orig.getReadWriteGroup().clear(); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.inheritPermissions = true; @@ -418,9 +418,9 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, orig.isPublic = false; orig.isLocked = true; orig.getReadOnlyGroup().clear(); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); orig.getReadWriteGroup().clear(); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change storageID @@ -509,9 +509,9 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, orig.isPublic = false; orig.isLocked = true; orig.getReadOnlyGroup().clear(); - orig.getReadOnlyGroup().add(URI.create("ivo://opencadc.org/gms?g1")); + orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); orig.getReadWriteGroup().clear(); - orig.getReadWriteGroup().add(URI.create("ivo://opencadc.org/gms?g3")); + orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change target diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 76baf96de..32a65631b 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -72,8 +72,6 @@ import ca.nrc.cadc.util.StringUtil; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; -import java.sql.Array; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -83,16 +81,15 @@ import java.util.Calendar; import java.util.Comparator; import java.util.Date; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.StringTokenizer; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; @@ -109,7 +106,6 @@ import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeProperty; -import org.opencadc.vospace.VOS; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -979,14 +975,14 @@ private void safeSetTimestamp(PreparedStatement prep, int col, Timestamp value, } } - private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException { + private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException { if (values != null && !values.isEmpty()) { log.debug("safeSetArray: " + col + " " + values.size()); String[] array1d = new String[values.size()]; int i = 0; - for (URI u : values) { - array1d[i] = u.toASCIIString(); + for (GroupURI u : values) { + array1d[i] = u.getURI().toASCIIString(); i++; } java.sql.Array arr = prep.getConnection().createArrayOf("text", array1d); @@ -1754,10 +1750,10 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro ret.isLocked = isLocked; if (rawROG != null) { - Util.parseArrayURI(rawROG, ret.getReadOnlyGroup()); + Util.parseArrayGroupURI(rawROG, ret.getReadOnlyGroup()); } if (rawRWG != null) { - Util.parseArrayURI(rawRWG, ret.getReadWriteGroup()); + Util.parseArrayGroupURI(rawRWG, ret.getReadWriteGroup()); } if (rawProps != null) { Util.parseArrayProps(rawProps, ret.getProperties()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java index 2e7e197b8..782cb8a6f 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java @@ -75,14 +75,13 @@ import java.sql.Array; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Calendar; import java.util.Date; -import java.util.List; import java.util.Set; import java.util.StringTokenizer; import java.util.UUID; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; import org.opencadc.vospace.NodeProperty; /** @@ -384,7 +383,7 @@ public static byte[] getByteArray(ResultSet rs, int col) } // fills the dest set - public static void parseArrayURI(String val, Set dest) { + public static void parseArrayGroupURI(String val, Set dest) { // postgresql 1D array: {a,"b,c"} if (val == null || val.isEmpty()) { return; @@ -407,14 +406,14 @@ public static void parseArrayURI(String val, Set dest) { handleToken(token, dest); } - private static void handleToken(String token, Set dest) { + private static void handleToken(String token, Set dest) { if (token.startsWith("ivo://")) { - dest.add(URI.create(token)); + dest.add(new GroupURI(URI.create(token))); } else { StringTokenizer st = new StringTokenizer(token, "{,}"); while (st.hasMoreTokens()) { String s = st.nextToken(); - dest.add(URI.create(s)); + dest.add(new GroupURI(URI.create(s))); } } } diff --git a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java index adf255402..c26cf3f26 100644 --- a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java +++ b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java @@ -68,12 +68,12 @@ package org.opencadc.inventory.db; import ca.nrc.cadc.util.Log4jInit; -import java.net.URI; import java.util.Set; import java.util.TreeSet; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Test; +import org.opencadc.gms.GroupURI; import org.opencadc.vospace.NodeProperty; /** @@ -91,19 +91,18 @@ public UtilTest() { } @Test - public void testParseArrayURI() throws Exception { + public void testParseArrayGroupURI() throws Exception { String str = "{ivo://opencadc.org/gms?g3," - + "\"ivo://opencadc.org/gms?g4,g5\"," + "ivo://opencadc.org/gms?g6-g7," + "ivo://opencadc.org/gms?g6.g7," + "ivo://opencadc.org/gms?g6_g7," + "ivo://opencadc.org/gms?g6~g7}"; - Set dest = new TreeSet<>(); - Util.parseArrayURI(str, dest); - for (URI u : dest) { - log.info("uri: " + u); + Set dest = new TreeSet<>(); + Util.parseArrayGroupURI(str, dest); + for (GroupURI u : dest) { + log.info("uri: " + u.getURI()); } } From ddde9d923a62e980b39b2d6ec92135233fcae79e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 7 Sep 2023 12:17:50 -0700 Subject: [PATCH 046/186] cadc-inventory-db: fix NodeDAO.lock for subclass support add NodeDAO.isEmpty(ContainerNode) to support check in delete code --- .../org/opencadc/vospace/db/NodeDAOTest.java | 34 +++++++++++++++++++ .../opencadc/inventory/db/SQLGenerator.java | 34 ++++++++++++++++--- .../java/org/opencadc/vospace/db/NodeDAO.java | 29 ++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 10c8e9e46..c24341c6e 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -541,6 +541,35 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Assert.assertNull(gone); } + @Test + public void testGetWithLock() { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + // put + ContainerNode orig = new ContainerNode("container-test"); + orig.parent = root; + orig.ownerID = "the-owner"; + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found by id: " + a.getID() + " aka " + a); + Assert.assertEquals(orig.getID(), a.getID()); + Assert.assertEquals(orig.getName(), a.getName()); + Assert.assertEquals(root.getID(), a.parentID); + + // get with lock + Node locked = nodeDAO.lock(a); + Assert.assertNotNull(locked); + log.info("locked: " + a.getID() + " aka " + a); + + nodeDAO.delete(orig.getID()); + Node gone = nodeDAO.get(orig.getID()); + Assert.assertNull(gone); + } + @Test public void testContainerNodeIterator() throws IOException { UUID rootID = new UUID(0L, 0L); @@ -558,6 +587,8 @@ public void testContainerNodeIterator() throws IOException { Assert.assertEquals(orig.getName(), a.getName()); Assert.assertTrue(a instanceof ContainerNode); + ContainerNode cn = (ContainerNode) a; + Assert.assertTrue(nodeDAO.isEmpty(cn)); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); @@ -588,10 +619,13 @@ public void testContainerNodeIterator() throws IOException { link.ownerID = orig.ownerID; log.info("put child: " + cont + " of " + cont.parent); nodeDAO.put(cont); + Assert.assertFalse(nodeDAO.isEmpty(cn)); log.info("put child: " + data + " of " + data.parent); nodeDAO.put(data); + Assert.assertFalse(nodeDAO.isEmpty(cn)); log.info("put child: " + link + " of " + link.parent); nodeDAO.put(link); + Assert.assertFalse(nodeDAO.isEmpty(cn)); Node c1; Node c2; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 32a65631b..6ac3dcbb0 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -108,6 +108,7 @@ import org.opencadc.vospace.NodeProperty; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementCreator; import org.springframework.jdbc.core.ResultSetExtractor; import org.springframework.jdbc.core.RowMapper; @@ -304,17 +305,15 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new StorageSiteGet(forUpdate); } - if (Node.class.equals(c) || Node.class.isInstance(c)) { + if (Node.class.equals(c)) { return new NodeGet(forUpdate); } - if (DeletedNodeEvent.class.equals(c)) { - //return new DeletedNodeGet(); - } if (forUpdate) { throw new UnsupportedOperationException("entity-get + forUpdate: " + c.getSimpleName()); } + // raw events are never locked for update if (DeletedArtifactEvent.class.equals(c)) { return new DeletedArtifactEventGet(); } @@ -327,6 +326,11 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { if (ObsoleteStorageLocation.class.equals(c)) { return new ObsoleteStorageLocationGet(); } + + if (DeletedNodeEvent.class.equals(c)) { + //return new DeletedNodeGet(); + } + if (HarvestState.class.equals(c)) { return new HarvestStateGet(); } @@ -334,6 +338,28 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { throw new UnsupportedOperationException("entity-get: " + c.getName()); } + public NodeCount getNodeCount() { + return new NodeCount(); + } + + public class NodeCount { + private UUID id; + + public void setID(UUID id) { + this.id = id; + } + + public int execute(JdbcTemplate jdbc) { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT count(*) FROM ").append(getTable(Node.class)); + sb.append(" WHERE parentID = '").append(id.toString()).append("'"); + String sql = sb.toString(); + log.debug("NodeCount: " + sql); + int ret = jdbc.queryForObject(sql, Integer.class); + return ret; + } + } + public EntityIteratorQuery getEntityIteratorQuery(Class c) { if (Artifact.class.equals(c)) { return new ArtifactIteratorQuery(); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 6fc157862..acf81b961 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -98,6 +98,15 @@ public void put(Node val) { super.put(val); } + @Override + public Node lock(Node n) { + if (n == null) { + throw new IllegalArgumentException("entity cannot be null"); + } + // override because Node has subclasses: force base class here + return super.lock(Node.class, n.getID()); + } + public Node get(UUID id) { return super.get(Node.class, id); } @@ -121,6 +130,26 @@ public Node get(ContainerNode parent, String name) { throw new RuntimeException("BUG: handleInternalFail did not throw"); } + public boolean isEmpty(ContainerNode parent) { + checkInit(); + log.debug("isEmpty: " + parent.getID()); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + SQLGenerator.NodeCount count = (SQLGenerator.NodeCount) gen.getNodeCount(); + count.setID(parent.getID()); + int num = count.execute(jdbc); + return (num == 0); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("isEmpty: " + parent.getID() + " " + dt + "ms"); + } + throw new RuntimeException("BUG: handleInternalFail did not throw"); + } + public void delete(UUID id) { super.delete(Node.class, id); } From b3f51aa38c8ac0e82cbb51b3b8adbaab3b5daf0c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 7 Sep 2023 12:18:53 -0700 Subject: [PATCH 047/186] vault: reject deletion of non-empty containers, perform delete actions in txn --- .../opencadc/vault/NodePersistenceImpl.java | 86 ++++++++++++------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 5484d14dd..63bf5ec96 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -72,6 +72,7 @@ import ca.nrc.cadc.auth.PrincipalExtractor; import ca.nrc.cadc.auth.X509CertificateChain; import ca.nrc.cadc.date.DateUtil; +import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; import ca.nrc.cadc.util.InvalidConfigException; @@ -480,41 +481,66 @@ public void delete(Node node) throws TransientException { final NodeDAO dao = getDAO(); final ArtifactDAO artifactDAO = getArtifactDAO(); + TransactionManager txn = dao.getTransactionManager(); - // TODO: do the following in a transaction, acquire lock on target node - - URI storageID = null; - if (node instanceof ContainerNode) { - ContainerNode cn = (ContainerNode) node; - try (ResourceIterator iter = dao.iterator(cn, 1, null)) { - if (iter.hasNext()) { - throw new IllegalArgumentException("container node '" + node.getName() + "' is not empty"); + try { + log.debug("starting transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + Node locked = dao.lock(node); + if (locked != null) { + node = locked; // safer than having two vars and accidentally using the wrong one + URI storageID = null; + if (node instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) node; + boolean empty = dao.isEmpty(cn); + if (!empty) { + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + throw new IllegalArgumentException("container node '" + node.getName() + "' is not empty"); + } + } else if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); + if (len != null) { + // artifact exists + storageID = dn.storageID; + } + } // else: LinkNode can always be deleted + + if (storageID != null) { + Artifact a = artifactDAO.get(storageID); + if (a != null) { + DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); + DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); + daeDAO.put(dae); + artifactDAO.delete(a.getID()); + } } - } catch (IOException ex) { - throw new TransientException("database IO failure", ex); + // TODO: need DeletedNodeDAO to create DeletedNodeEvent + dao.delete(node.getID()); + } else { + log.debug("failed to lock node " + node.getID() + " - assume deleted by another process"); } - } else if (node instanceof DataNode) { - DataNode dn = (DataNode) node; - NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - if (len != null) { - // artifact exists - storageID = dn.storageID; + + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to delete " + node.getID() + " aka " + node.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); } - } // else: LinkNode can always be deleted - - // TODO: need DeletedNodeDAO to create DeletedNodeEvent - - if (storageID != null) { - Artifact a = artifactDAO.get(storageID); - if (a != null) { - DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); - DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); - daeDAO.put(dae); - artifactDAO.delete(a.getID()); + throw ex; + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); } } - dao.delete(node.getID()); - - // TODO: commit transaction } } From 2911e7b2cc98cb4ca071c82523027088b466e5a0 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 8 Sep 2023 12:55:41 -0700 Subject: [PATCH 048/186] vault-quota: README and design doc --- vault-quota/Design.md | 66 +++++++++++++++++++++++++++++++++++++++++++ vault-quota/README.md | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 vault-quota/Design.md create mode 100644 vault-quota/README.md diff --git a/vault-quota/Design.md b/vault-quota/Design.md new file mode 100644 index 000000000..435360fce --- /dev/null +++ b/vault-quota/Design.md @@ -0,0 +1,66 @@ +# vault quota design/algorithms + +The definitive source of content-length (file size) of a DataNode coems from the +`inventory.Artifact` table and it not known until a PUT to storage is completed. +In the case of a `vault` service co-located with a single storage site (`minoc`), +the new Artifact is visible in the database as soon as the PUT to `minoc` is +completed. In the case of a `vault` service co-located with a global SI, the new +Artifact is visible in the database once it is synced from the site of the PUT to +`minoc` to the global database by `fenwick` (or worst case: `ratik`). + +## TODO +The design below only takes into account incremental propagation of space used +by stored files. It is not complete/verified until we also come up with a validation +algorithm that can detect and fix discrepancies in a live `vault`. + +## Event watcher algorithm: +``` +track progress using HarvestState (name: `Artifact`, source: `db:{bucket range}`) +incremental query for new artifacts in lastModified order +for each new Artifact: + query for DataNode (storageID = artifact.uri) + if Artifact.contentLength != Node.size: + start txn + lock datanode + compute delta + lock parent + apply delta to parent.delta + set dataNode.size + update HarvestState + commit txn +``` +The above sequence does the first step of propagation from DataNode to parent ContainerNode. +This can be done in parallel by using bucket ranges (smaller than 0-f). + +## Container size propagation algorithm: +``` +query for ContainerNode with non-zero delta +for each ContainerNode: + start txn + lock containernode + re-check delta + lock parent + apply delta to parent.delta + apply delta containernode.size, set containernode.delta=0 + commit txn +``` +The above sequence finds candidate propagations, locks (order: child-then-parent as above), +and applies the propagation. This moves the outstanding delta up the tree one level. If the +sequence acts on multiple child containers before the parent, the delta(s) naturally +_merge_ and there are fewer larger delta propagations in the upper part of the tree. It would +be optimal to do propagations depth-first but it doesn't seem practical to forcibly accomplish +that ordering. + +Container size propagation will be implemented as a single sequence (thread). We could add +something to the vospace.Node table to support subdividing work and enable multiple threads, +but there is nothing there right now. + +## database changes required +note: all field and column names TBD +* add `size` and `delta` fields to ContainerNode (transient) +* add `size` field to DataNode (transient) +* add `size` to the `vospace.Node` table +* add `delta` to the `vospace.Node` table +* incremental sync query/iterator (ArtifactDAO?) +* lookup DataNode by storageID (ArtifactDAO?) + diff --git a/vault-quota/README.md b/vault-quota/README.md new file mode 100644 index 000000000..c0f2db0eb --- /dev/null +++ b/vault-quota/README.md @@ -0,0 +1,59 @@ +# Storage Inventory VOSpace quota support process (vault-quota) + +Process to maintain container node sizes so that quota limits can be enforced by the +main `vault` service. This process runs in incremental mode (single process running +continuously) to update a local vospace database. + +`vault-quota` is an optional process that is only needed if `vault` is configured to +enforce quotas, although it could be used to maintain container node sizes without +quota enforcement. + +## configuration +See the [cadc-java](https://github.com/opencadc/docker-base/tree/master/cadc-java) image +docs for general config requirements. + +Runtime configuration must be made available via the `/config` directory. + +### vault-quota.properties +``` +org.opencadc.vault.quota.logging = {info|debug} + +# inventory database settings +org.opencadc.inventory.db.SQLGenerator=org.opencadc.inventory.db.SQLGenerator +org.opencadc.vault.quota.nodes.schema={schema for inventory database objects} +org.opencadc.vault.quota.nodes.username={username for inventory admin} +org.opencadc.vault.quota.nodes.password={password for inventory admin} +org.opencadc.vault.quota.nodes.url=jdbc:postgresql://{server}/{database} + +org.opencadc.vault.quota.threads={number of threads to watch for artifact events} + +# storage namespace +org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} +``` +The _nodes_ account owns and manages (create, alter, drop) vospace database objects and updates +content in the vospace schema. The database is specified in the JDBC URL. Failure to connect or +initialize the database will show up in logs. + +The _threads_ key configures the number of threads that watch for new Artifact events and initiate +the propagation of sizes to parent containers. These threads each monitor a subset of artifacts using +`Artifact.uriBucket` filtering; for simplicity, the following values are allowed: 1, 2, 4, 8, 16. + +In addition to the above threads, there is one additional thread that propagates size changes up +the tree of container nodes to the container node(s) where quotas are specified. + +## building it +``` +gradle clean build +docker build -t vault-quota -f Dockerfile . +``` + +## checking it +``` +docker run -it vault-quota:latest /bin/bash +``` + +## running it +``` +docker run --user opencadc:opencadc -v /path/to/external/config:/config:ro --name vault-quota vault-quota:latest +``` + From e15cbd0f4205a863e32f95c37f86eeb3a83ab993 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Sep 2023 09:37:23 -0700 Subject: [PATCH 049/186] Update Design.md --- vault-quota/Design.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index 435360fce..af2e188eb 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -1,6 +1,6 @@ # vault quota design/algorithms -The definitive source of content-length (file size) of a DataNode coems from the +The definitive source of content-length (file size) of a DataNode comes from the `inventory.Artifact` table and it not known until a PUT to storage is completed. In the case of a `vault` service co-located with a single storage site (`minoc`), the new Artifact is visible in the database as soon as the PUT to `minoc` is @@ -63,4 +63,5 @@ note: all field and column names TBD * add `delta` to the `vospace.Node` table * incremental sync query/iterator (ArtifactDAO?) * lookup DataNode by storageID (ArtifactDAO?) +* indices to support new queries From 07d759f59d81e8e39184c60a34accc5e30b1f175 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Sep 2023 12:13:30 -0700 Subject: [PATCH 050/186] Update Design.md --- vault-quota/Design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index af2e188eb..f6d6b137d 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -15,7 +15,7 @@ algorithm that can detect and fix discrepancies in a live `vault`. ## Event watcher algorithm: ``` -track progress using HarvestState (name: `Artifact`, source: `db:{bucket range}`) +track progress using HarvestState (source: `db:{bucket range}`, name: TBD) incremental query for new artifacts in lastModified order for each new Artifact: query for DataNode (storageID = artifact.uri) From 53363891932112e3dadf7dbb8f2b4b95d8b23d58 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 12 Sep 2023 13:16:52 -0700 Subject: [PATCH 051/186] vault-quota: update design doc with some validate ideas --- vault-quota/Design.md | 50 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index f6d6b137d..b2b43b920 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -13,7 +13,9 @@ The design below only takes into account incremental propagation of space used by stored files. It is not complete/verified until we also come up with a validation algorithm that can detect and fix discrepancies in a live `vault`. -## Event watcher algorithm: +## DataNode size algorithm: +This is an event watcher that gets Artifact events (after a PUT) and intiates the +propagation of sizes (space used). ``` track progress using HarvestState (source: `db:{bucket range}`, name: TBD) incremental query for new artifacts in lastModified order @@ -29,10 +31,12 @@ for each new Artifact: update HarvestState commit txn ``` -The above sequence does the first step of propagation from DataNode to parent ContainerNode. -This can be done in parallel by using bucket ranges (smaller than 0-f). +Optimization: The above sequence does the first step of propagation from DataNode to +parent ContainerNode so the maximum work can be done in parallel using bucket ranges +(smaller than 0-f). It also means the propagation below only has to consider +ContainerNode.delta since DataNode(s) never have a delta. -## Container size propagation algorithm: +## ContainerNode size propagation algorithm: ``` query for ContainerNode with non-zero delta for each ContainerNode: @@ -55,12 +59,50 @@ Container size propagation will be implemented as a single sequence (thread). We something to the vospace.Node table to support subdividing work and enable multiple threads, but there is nothing there right now. +## validation + +### DataNode vs Artifact discrepancies +These can be validated in parallel by multiple threads, subdivide work by bucket. + +``` +discrepancy 1: Artifact exists but DataNode does not +explanation: DataNode created, transfer negotiated, DataNode removed, transfer executed +evidence: check for DeletedNodeEvent +action: remove artifact, create DeletedArtifactEvent +else: ?? + +discrepancy 2: DataNode exists but Artifact does not +explanation: DataNode created, Artifact never (successfully) put +evidence: dataNode.size == 0 +action: none + +discrepancy 3: DataNode exists but Artifact does not +explanation: deleted or lost Artifact +evidence: DataNode.size != 0 (deleted vs lost: DeletedArtifactEvent exists) +action: fix DataNode.size + +discrepancy 4: DataNode.size != Artifact.contentLength +explanation: pending/missed Artifact event +action: fix DataNode and propagate delta to parent ContainerNode (same as incremental) +``` + +This could be accomplished with a single query on on inventory.Artifact full outer join +vospace.Node to get all the pairs. The more generic approach would be to do a merge join +of two iterators: + +Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); +Iterator niter = nodeDAO.iterator(vaultNamespace, bucket); + +The more generic dual iterator approach could be made to work if the inventory and vospace +content are in different PG database or server - TBD. + ## database changes required note: all field and column names TBD * add `size` and `delta` fields to ContainerNode (transient) * add `size` field to DataNode (transient) * add `size` to the `vospace.Node` table * add `delta` to the `vospace.Node` table +* add `storageBucket` to `vospace.Node` table (validation) * incremental sync query/iterator (ArtifactDAO?) * lookup DataNode by storageID (ArtifactDAO?) * indices to support new queries From dec5ed8b358195a1576a65d69ea66116cb3870af Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 6 Oct 2023 16:06:05 -0700 Subject: [PATCH 052/186] Support for recursive deletes --- vault/build.gradle | 6 ++-- .../java/org/opencadc/vault/NodesTest.java | 2 ++ .../org/opencadc/vault/VaultInitAction.java | 17 +++++++++++ vault/src/main/webapp/META-INF/context.xml | 13 ++++++++ vault/src/main/webapp/WEB-INF/web.xml | 30 +++++++++++++++++++ vault/src/main/webapp/capabilities.xml | 10 +++++++ 6 files changed, 75 insertions(+), 3 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index 86b736b0c..7fd069259 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -30,7 +30,7 @@ def git_url = 'https://github.com/opencadc/vos' dependencies { compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' - compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-util:[1.9.10,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' @@ -38,10 +38,10 @@ dependencies { compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' - compile 'org.opencadc:cadc-uws-server:[1.2.12,)' + compile 'org.opencadc:cadc-uws-server:[1.2.19,)' compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' compile 'org.opencadc:cadc-cdp:[1.2.3,)' - compile 'org.opencadc:cadc-registry:[1.5.15,)' + compile 'org.opencadc:cadc-registry:[1.7.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java index 71bb1178b..2a4a317da 100644 --- a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -71,6 +71,7 @@ import java.net.URI; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; /** * Test the nodes endpoint. @@ -86,6 +87,7 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest { } public NodesTest() { + //super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem", new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?CADC_TEST_GROUP2")), "vault-test-auth.pem"); super(URI.create("ivo://opencadc.org/vault"), "vault-test.pem"); } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index bc9dc89f5..c22083be8 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -71,6 +71,7 @@ import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -93,6 +94,7 @@ public class VaultInitAction extends InitAction { private static final Logger log = Logger.getLogger(VaultInitAction.class); static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml + static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml // config keys private static final String VAULT_KEY = "org.opencadc.vault"; @@ -119,6 +121,7 @@ public VaultInitAction() { public void doInit() { initConfig(); initDatabase(); + initUWSDatabase(); initNodePersistence(); } @@ -215,6 +218,20 @@ private void initDatabase() { } } + private void initUWSDatabase() { + log.info("initUWSDatabase: START"); + try { + // Init UWS database + DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); + InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws"); + uwsi.doInit(); + log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws OK"); + + } catch (Exception ex) { + throw new RuntimeException("check/init uws database failed", ex); + } + } + protected void initNodePersistence() { jndiNodePersistence = componentID + ".nodePersistence"; try { diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index cbc72f2e0..e5775723b 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -15,4 +15,17 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> + + + diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index af0394c10..90476792e 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -23,6 +23,7 @@ ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi + ca.nrc.cadc.uws @@ -62,6 +63,30 @@ 2 + + + RecursiveDeleteNodeServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.RecursiveDeleteNodeJobManager + + 3 + + + CapabilitiesServlet @@ -107,6 +132,11 @@ /nodes/* + + RecursiveDeleteNodeServlet + /recursiveDelete/* + + AvailabilityServlet diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index fb145630e..d24f1a220 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -41,6 +41,16 @@ + + + https://replace.me.com/vault/recursiveDelete + + + + + + + org.opencadc.vospace - ca.nrc.cadc.rest + @@ -63,7 +63,7 @@ 2 - + RecursiveDeleteNodeServlet ca.nrc.cadc.uws.server.JobServlet @@ -86,6 +86,33 @@ 3 + + + RecursiveNodePropsServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.RecursiveNodePropsJobManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.vospace.server.async.InlineNodeJobHandler + + 3 + + @@ -134,7 +161,12 @@ RecursiveDeleteNodeServlet - /recursiveDelete/* + /async-delete/* + + + + RecursiveNodePropsServlet + /async-setprops/* diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index d24f1a220..28cfb09bc 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -43,7 +43,17 @@ - https://replace.me.com/vault/recursiveDelete + https://replace.me.com/vault/async-delete + + + + + + + + + + https://replace.me.com/vault/async-setprops From 82de71bcdcc0e56240849313ffa47fa6bd8eee95 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 28 Nov 2023 08:50:49 -0800 Subject: [PATCH 063/186] vault: added code to restrict to local groups only, currently disabled --- .../opencadc/vault/NodePersistenceImpl.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index e072b4e5b..1fb0905d7 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -76,6 +76,8 @@ import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.LocalAuthority; import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; @@ -87,6 +89,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -94,6 +97,7 @@ import javax.security.auth.Subject; import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.Namespace; @@ -154,6 +158,7 @@ public class NodePersistenceImpl implements NodePersistence { private final ContainerNode trash; private final Namespace storageNamespace; + private final boolean localGroupsOnly; private URI resourceID; public NodePersistenceImpl(URI resourceID) { @@ -199,6 +204,8 @@ public NodePersistenceImpl(URI resourceID) { String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); this.storageNamespace = new Namespace(ns); + + this.localGroupsOnly = false; } private Subject getRootOwner(MultiValuedProperties mvp, IdentityManager im) { @@ -427,6 +434,33 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException node.ownerID = identityManager.toOwner(node.owner); } + if (localGroupsOnly) { + if (!node.getReadOnlyGroup().isEmpty() || !node.getReadWriteGroup().isEmpty()) { + LocalAuthority loc = new LocalAuthority(); + try { + URI localGMS = loc.getServiceURI(Standards.GMS_SEARCH_10.toASCIIString()); + StringBuilder serr = new StringBuilder("non-local groups:"); + int len = serr.length(); + for (GroupURI g : node.getReadOnlyGroup()) { + if (!localGMS.equals(g.getServiceID())) { + serr.append(" ").append(g.getURI().toASCIIString()); + } + } + for (GroupURI g : node.getReadWriteGroup()) { + if (!localGMS.equals(g.getServiceID())) { + serr.append(" ").append(g.getURI().toASCIIString()); + } + } + String err = serr.toString(); + if (err.length() > len) { + throw new IllegalArgumentException(err); + } + } catch (NoSuchElementException ex) { + throw new RuntimeException("CONFIG: localGroupOnly policy && local GMS service not configured"); + } + } + } + if (node instanceof DataNode) { DataNode dn = (DataNode) node; if (dn.storageID == null) { From f1d4a635c9e6d836d28ed2e0f329269a8c3210cb Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Mon, 11 Dec 2023 06:38:12 -0800 Subject: [PATCH 064/186] Small updates --- .../java/org/opencadc/vault/VosiCapabilitiesTest.java | 4 ++-- vault/src/main/webapp/WEB-INF/web.xml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vault/src/intTest/java/org/opencadc/vault/VosiCapabilitiesTest.java b/vault/src/intTest/java/org/opencadc/vault/VosiCapabilitiesTest.java index fb88ef76e..af49d8548 100644 --- a/vault/src/intTest/java/org/opencadc/vault/VosiCapabilitiesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/VosiCapabilitiesTest.java @@ -85,8 +85,8 @@ public class VosiCapabilitiesTest extends CapabilitiesTest { private static final Logger log = Logger.getLogger(VosiCapabilitiesTest.class); static { - Log4jInit.setLevel("ca.nrc.cadc.vosi", Level.DEBUG); - Log4jInit.setLevel("org.opencadc.vault", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.vosi", Level.INFO); + Log4jInit.setLevel("org.opencadc.vault", Level.INFO); } public static final URI VAULT_SERVICE_ID = URI.create("ivo://opencadc.org/vault"); diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 8dc696d97..c034a2f70 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -18,12 +18,12 @@ logLevelPackages org.opencadc.vault - + ca.nrc.cadc.db org.opencadc.vospace - + ca.nrc.cadc.uws From 0debf14be629547dfa4e86e8bc3b63f11c388679 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Mon, 11 Dec 2023 06:47:24 -0800 Subject: [PATCH 065/186] Updated dependencies --- vault/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index 2952fd263..6a637f96c 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -35,7 +35,7 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' - compile 'org.opencadc:cadc-vos-server:[2.0.1,)' + compile 'org.opencadc:cadc-vos-server:[2.0.3,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' compile 'org.opencadc:cadc-uws-server:[1.2.19,)' @@ -51,7 +51,7 @@ dependencies { runtime 'org.opencadc:cadc-gms:[1.0.5,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' - intTestCompile 'org.opencadc:cadc-test-vos:[2.0,3.0)' + intTestCompile 'org.opencadc:cadc-test-vos:[2.1,)' } configurations { From f858acad0a4559c64e13a7dfe8a2e6e417f78c3b Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Mon, 11 Dec 2023 10:08:00 -0800 Subject: [PATCH 066/186] Rework after code review --- .../vault/RecursiveDeleteNodeJobManager.java | 16 +-- .../vault/RecursiveNodePropsJobManager.java | 16 +-- .../org/opencadc/vault/VaultJobManager.java | 97 +++++++++++++++++++ 3 files changed, 103 insertions(+), 26 deletions(-) create mode 100644 vault/src/main/java/org/opencadc/vault/VaultJobManager.java diff --git a/vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java b/vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java index b438dbeaa..87619dd2a 100644 --- a/vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java +++ b/vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java @@ -69,13 +69,10 @@ package org.opencadc.vault; -import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.uws.server.JobExecutor; import ca.nrc.cadc.uws.server.JobPersistence; import ca.nrc.cadc.uws.server.JobUpdater; -import ca.nrc.cadc.uws.server.SimpleJobManager; import ca.nrc.cadc.uws.server.ThreadPoolExecutor; -import ca.nrc.cadc.uws.server.impl.PostgresJobPersistence; import org.apache.log4j.Logger; import org.opencadc.vospace.server.async.RecursiveDeleteNodeRunner; @@ -83,24 +80,17 @@ * * @author pdowler, majorb, yeunga, adriand */ -public class RecursiveDeleteNodeJobManager extends SimpleJobManager { +public class RecursiveDeleteNodeJobManager extends VaultJobManager { private static final Logger log = Logger.getLogger(RecursiveDeleteNodeJobManager.class); private static final Long MAX_EXEC_DURATION = Long.valueOf(12 * 7200L); // 24 hours? private static final Long MAX_DESTRUCTION = Long.valueOf(7 * 24 * 3600L); // 1 week private static final Long MAX_QUOTE = Long.valueOf(12 * 7200L); // same as exec - protected static JobPersistence jp; - - static { - log.info("Creating shared (postgres) job manager"); - jp = new PostgresJobPersistence(AuthenticationUtil.getIdentityManager()); - } - public RecursiveDeleteNodeJobManager() { super(); - // jp is instantiated in parent org.opencadc.cavern.JobManager - JobUpdater ju = jp; + JobPersistence jp = createJobPersistence(); + JobUpdater ju = (JobUpdater) jp; super.setJobPersistence(jp); JobExecutor jobExec = new ThreadPoolExecutor(ju, RecursiveDeleteNodeRunner.class, 3); diff --git a/vault/src/main/java/org/opencadc/vault/RecursiveNodePropsJobManager.java b/vault/src/main/java/org/opencadc/vault/RecursiveNodePropsJobManager.java index 1964d0bef..6117aa67c 100644 --- a/vault/src/main/java/org/opencadc/vault/RecursiveNodePropsJobManager.java +++ b/vault/src/main/java/org/opencadc/vault/RecursiveNodePropsJobManager.java @@ -69,13 +69,10 @@ package org.opencadc.vault; -import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.uws.server.JobExecutor; import ca.nrc.cadc.uws.server.JobPersistence; import ca.nrc.cadc.uws.server.JobUpdater; -import ca.nrc.cadc.uws.server.SimpleJobManager; import ca.nrc.cadc.uws.server.ThreadPoolExecutor; -import ca.nrc.cadc.uws.server.impl.PostgresJobPersistence; import org.apache.log4j.Logger; import org.opencadc.vospace.server.async.RecursiveNodePropsRunner; @@ -83,24 +80,17 @@ * * @author pdowler, majorb, yeunga, adriand */ -public class RecursiveNodePropsJobManager extends SimpleJobManager { +public class RecursiveNodePropsJobManager extends VaultJobManager { private static final Logger log = Logger.getLogger(RecursiveNodePropsJobManager.class); private static final Long MAX_EXEC_DURATION = Long.valueOf(12 * 7200L); // 24 hours? private static final Long MAX_DESTRUCTION = Long.valueOf(7 * 24 * 3600L); // 1 week private static final Long MAX_QUOTE = Long.valueOf(12 * 7200L); // same as exec - protected static JobPersistence jp; - - static { - log.info("Creating shared (postgres) job manager"); - jp = new PostgresJobPersistence(AuthenticationUtil.getIdentityManager()); - } - public RecursiveNodePropsJobManager() { super(); - // jp is instantiated in parent org.opencadc.cavern.JobManager - JobUpdater ju = jp; + JobPersistence jp = createJobPersistence(); + JobUpdater ju = (JobUpdater) jp; super.setJobPersistence(jp); JobExecutor jobExec = new ThreadPoolExecutor(ju, RecursiveNodePropsRunner.class, 3); diff --git a/vault/src/main/java/org/opencadc/vault/VaultJobManager.java b/vault/src/main/java/org/opencadc/vault/VaultJobManager.java new file mode 100644 index 000000000..7968e7f52 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/VaultJobManager.java @@ -0,0 +1,97 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 4 $ +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.uws.server.JobPersistence; +import ca.nrc.cadc.uws.server.SimpleJobManager; +import ca.nrc.cadc.uws.server.impl.PostgresJobPersistence; +import org.apache.log4j.Logger; + +/** + * + * @author adriand + */ +public class VaultJobManager extends SimpleJobManager { + private static final Logger log = Logger.getLogger(VaultJobManager.class); + + protected VaultJobManager() { + super(); + } + + protected final JobPersistence createJobPersistence() { + return new PostgresJobPersistence(AuthenticationUtil.getIdentityManager()); + } + + @Override + public void terminate() throws InterruptedException { + super.terminate(); + } +} From 4b3e2842b8e8780e4c275ce891b6ae340ed8502b Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Dec 2023 12:59:59 -0800 Subject: [PATCH 067/186] move transfer negotiation code from raven to cadc-inventory-server port to cadc-vos-2.0 API --- cadc-inventory-server/build.gradle | 6 +- .../transfer}/ProtocolsGenerator.java | 128 +++++++++--------- .../inventory/transfer}/StorageSiteRule.java | 5 +- .../transfer}/ProtocolsGeneratorTest.java | 4 +- raven/build.gradle | 2 +- .../java/org/opencadc/raven/FilesTest.java | 11 +- .../org/opencadc/raven/NegotiationTest.java | 13 +- .../java/org/opencadc/raven/RavenTest.java | 11 +- .../org/opencadc/raven/ArtifactAction.java | 7 +- .../org/opencadc/raven/GetFilesAction.java | 13 +- .../org/opencadc/raven/HeadFilesAction.java | 11 +- .../java/org/opencadc/raven/PostAction.java | 11 +- .../org/opencadc/raven/RavenInitAction.java | 6 +- vault/build.gradle | 2 +- 14 files changed, 119 insertions(+), 111 deletions(-) rename {raven/src/main/java/org/opencadc/raven => cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer}/ProtocolsGenerator.java (98%) rename {raven/src/main/java/org/opencadc/raven => cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer}/StorageSiteRule.java (97%) rename {raven/src/test/java/org/opencadc/raven => cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer}/ProtocolsGeneratorTest.java (99%) diff --git a/cadc-inventory-server/build.gradle b/cadc-inventory-server/build.gradle index 2c6e23ba6..814e1f972 100644 --- a/cadc-inventory-server/build.gradle +++ b/cadc-inventory-server/build.gradle @@ -12,20 +12,22 @@ repositories { sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.2.2' +version = '0.3.0' description = 'OpenCADC Storage Inventory server utility library' def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { compile 'org.opencadc:cadc-inventory:[0.7,1.0)' - compile 'org.opencadc:cadc-inventory-db:[0.9,)' + compile 'org.opencadc:cadc-inventory-db:[0.15,)' compile 'org.opencadc:cadc-util:[1.9,2.0)' compile 'org.opencadc:cadc-rest:[1.3.14,)' compile 'org.opencadc:cadc-gms:[1.0.4,)' compile 'org.opencadc:cadc-cdp:[1.3,2.0)' compile 'org.opencadc:cadc-permissions:[0.2,)' compile 'org.opencadc:cadc-permissions-client:[0.3,)' + compile 'org.opencadc:cadc-vos:[2.0,3.0)' + compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' testCompile 'junit:junit:[4.0,)' } diff --git a/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java similarity index 98% rename from raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java rename to cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index 2c4aa7e98..cfbce5070 100644 --- a/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -65,7 +65,7 @@ ************************************************************************ */ -package org.opencadc.raven; +package org.opencadc.inventory.transfer; import ca.nrc.cadc.cred.client.CredUtil; import ca.nrc.cadc.net.HttpGet; @@ -76,10 +76,6 @@ import ca.nrc.cadc.reg.Interface; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.reg.client.RegistryClient; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Protocol; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.VOS; import ca.nrc.cadc.vosi.Availability; import java.io.File; import java.io.IOException; @@ -109,6 +105,10 @@ import org.opencadc.permissions.ReadGrant; import org.opencadc.permissions.TokenTool; import org.opencadc.permissions.WriteGrant; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; /** * Class for generating protocol lists corresponding to transfer requests. @@ -149,7 +149,11 @@ public ProtocolsGenerator(ArtifactDAO artifactDAO, File publicKeyFile, File priv this.storageResolver = storageResolver; } - List getProtocols(Transfer transfer) throws ResourceNotFoundException, IOException { + public boolean getStorageResolverAdded() { + return storageResolverAdded; + } + + public List getProtocols(Transfer transfer) throws ResourceNotFoundException, IOException { String authToken = null; URI artifactURI = transfer.getTargets().get(0); // see PostAction line ~127 if (publicKeyFile != null && privateKeyFile != null) { @@ -171,60 +175,7 @@ List getProtocols(Transfer transfer) throws ResourceNotFoundException, return protos; } - static void prioritizePullFromSites(List storageSites) { - // contains the algorithm for prioritizing storage sites to pull from. - - // was: prefer read/write sites to put less load on a read-only "seeder" site during migration - //storageSites.sort((site1, site2) -> Boolean.compare(!site1.getAllowWrite(), !site2.getAllowWrite())); - - // random - Collections.shuffle(storageSites); - } - - Artifact getRemoteArtifact(URL location, URI artifactURI) { - try { - HttpGet head = new HttpGet(location, true); - head.setHeadOnly(true); - head.setReadTimeout(10000); - head.run(); - if (head.getResponseCode() != 200) { - // caught at the end of the method - throw new RuntimeException("Unsuccessful HEAD request: " + head.getResponseCode()); - } - UUID id = UUID.fromString(head.getResponseHeader(ARTIFACT_ID_HDR)); - Artifact result = new - Artifact(id, artifactURI, head.getDigest(), head.getLastModified(), head.getContentLength()); - result.contentType = head.getContentType(); - result.contentEncoding = head.getContentEncoding(); - return result; - } catch (Throwable t) { - log.debug("Could not retrieve artifact " + artifactURI.toASCIIString() + " from " + location, t); - return null; - } - } - - private Capability getFilesCapability(StorageSite storageSite) { - if (!isAvailable(storageSite.getResourceID())) { - log.warn("storage site is offline: " + storageSite.getResourceID()); - return null; - } - Capability filesCap = null; - try { - RegistryClient regClient = new RegistryClient(); - Capabilities caps = regClient.getCapabilities(storageSite.getResourceID()); - filesCap = caps.findCapability(Standards.SI_FILES); - if (filesCap == null) { - log.warn("service: " + storageSite.getResourceID() + " does not provide " + Standards.SI_FILES); - } - } catch (ResourceNotFoundException ex) { - log.warn("storage site not found: " + storageSite.getResourceID()); - } catch (Exception ex) { - log.warn("storage site not responding (capabilities): " + storageSite.getResourceID(), ex); - } - return filesCap; - } - - Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set storageSites, String authToken) { + public Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set storageSites, String authToken) { Artifact result = null; for (StorageSite storageSite : storageSites) { // check if site is currently offline @@ -296,6 +247,61 @@ Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set storageSites) { + // contains the algorithm for prioritizing storage sites to pull from. + + // was: prefer read/write sites to put less load on a read-only "seeder" site during migration + //storageSites.sort((site1, site2) -> Boolean.compare(!site1.getAllowWrite(), !site2.getAllowWrite())); + + // random + Collections.shuffle(storageSites); + } + + Artifact getRemoteArtifact(URL location, URI artifactURI) { + try { + HttpGet head = new HttpGet(location, true); + head.setHeadOnly(true); + head.setReadTimeout(10000); + head.run(); + if (head.getResponseCode() != 200) { + // caught at the end of the method + throw new RuntimeException("Unsuccessful HEAD request: " + head.getResponseCode()); + } + UUID id = UUID.fromString(head.getResponseHeader(ARTIFACT_ID_HDR)); + Artifact result = new + Artifact(id, artifactURI, head.getDigest(), head.getLastModified(), head.getContentLength()); + result.contentType = head.getContentType(); + result.contentEncoding = head.getContentEncoding(); + return result; + } catch (Throwable t) { + log.debug("Could not retrieve artifact " + artifactURI.toASCIIString() + " from " + location, t); + return null; + } + } + + private Capability getFilesCapability(StorageSite storageSite) { + if (!isAvailable(storageSite.getResourceID())) { + log.warn("storage site is offline: " + storageSite.getResourceID()); + return null; + } + Capability filesCap = null; + try { + RegistryClient regClient = new RegistryClient(); + Capabilities caps = regClient.getCapabilities(storageSite.getResourceID()); + filesCap = caps.findCapability(Standards.SI_FILES); + if (filesCap == null) { + log.warn("service: " + storageSite.getResourceID() + " does not provide " + Standards.SI_FILES); + } + } catch (ResourceNotFoundException ex) { + log.warn("storage site not found: " + storageSite.getResourceID()); + } catch (Exception ex) { + log.warn("storage site not responding (capabilities): " + storageSite.getResourceID(), ex); + } + return filesCap; + } + + List doPullFrom(URI artifactURI, Transfer transfer, String authToken) throws ResourceNotFoundException, IOException { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO); @@ -327,7 +333,7 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) if (storageSite.getAllowRead()) { // less generic request for service that implements an API // HACK: this is filesCap specific in here - if (proto.getUri().equals(filesCap.getStandardID().toASCIIString())) { + if (proto.getUri().equals(filesCap.getStandardID())) { Protocol p = new Protocol(proto.getUri()); p.setEndpoint(storageSite.getResourceID().toASCIIString()); protos.add(p); diff --git a/raven/src/main/java/org/opencadc/raven/StorageSiteRule.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteRule.java similarity index 97% rename from raven/src/main/java/org/opencadc/raven/StorageSiteRule.java rename to cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteRule.java index 5e2d73146..fbdba6bab 100644 --- a/raven/src/main/java/org/opencadc/raven/StorageSiteRule.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteRule.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2021. (c) 2021. + * (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -67,10 +67,9 @@ ************************************************************************ */ -package org.opencadc.raven; +package org.opencadc.inventory.transfer; import java.util.List; - import org.opencadc.inventory.Namespace; public class StorageSiteRule { diff --git a/raven/src/test/java/org/opencadc/raven/ProtocolsGeneratorTest.java b/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java similarity index 99% rename from raven/src/test/java/org/opencadc/raven/ProtocolsGeneratorTest.java rename to cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java index 0c42f6ddc..2cc21f3f9 100644 --- a/raven/src/test/java/org/opencadc/raven/ProtocolsGeneratorTest.java +++ b/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java @@ -66,10 +66,9 @@ */ -package org.opencadc.raven; +package org.opencadc.inventory.transfer; import ca.nrc.cadc.util.Log4jInit; - import java.net.InetAddress; import java.net.URI; import java.util.ArrayList; @@ -79,7 +78,6 @@ import java.util.Random; import java.util.SortedSet; import java.util.TreeSet; - import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; diff --git a/raven/build.gradle b/raven/build.gradle index c5069bfb4..108723926 100644 --- a/raven/build.gradle +++ b/raven/build.gradle @@ -35,7 +35,7 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15)' - compile 'org.opencadc:cadc-inventory-server:[0.2.1,)' + compile 'org.opencadc:cadc-inventory-server:[0.3,)' compile 'org.opencadc:cadc-permissions:[0.3.1,)' compile 'org.opencadc:cadc-permissions-client:[0.3,)' compile 'org.opencadc:cadc-vos:[1.2,2.0)' diff --git a/raven/src/intTest/java/org/opencadc/raven/FilesTest.java b/raven/src/intTest/java/org/opencadc/raven/FilesTest.java index b0658db85..3b213b051 100644 --- a/raven/src/intTest/java/org/opencadc/raven/FilesTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/FilesTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2022. (c) 2022. + * (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -76,10 +76,6 @@ import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.Log4jInit; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Protocol; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.VOS; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.InputStream; @@ -105,6 +101,11 @@ import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.transfer.ProtocolsGenerator; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; /** * Test files endpoint. diff --git a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java index d50c70d15..67bb77fef 100644 --- a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2022. (c) 2022. + * (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -76,10 +76,6 @@ import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.Log4jInit; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Protocol; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.VOS; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.URI; @@ -104,6 +100,11 @@ import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.transfer.ProtocolsGenerator; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; /** * Test transfer negotiation. @@ -749,7 +750,7 @@ public void testGetSites() throws Exception { // can deliver the file List requested = new ArrayList<>(); - Protocol files = new Protocol(Standards.SI_FILES.toASCIIString()); + Protocol files = new Protocol(Standards.SI_FILES); requested.add(files); try { diff --git a/raven/src/intTest/java/org/opencadc/raven/RavenTest.java b/raven/src/intTest/java/org/opencadc/raven/RavenTest.java index 587ca7370..86239f968 100644 --- a/raven/src/intTest/java/org/opencadc/raven/RavenTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/RavenTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2019. (c) 2019. + * (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -81,10 +81,6 @@ import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.Log4jInit; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.TransferParsingException; -import ca.nrc.cadc.vos.TransferReader; -import ca.nrc.cadc.vos.TransferWriter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -96,8 +92,11 @@ import javax.security.auth.Subject; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.junit.Assert; import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.vospace.transfer.Transfer; +import org.opencadc.vospace.transfer.TransferParsingException; +import org.opencadc.vospace.transfer.TransferReader; +import org.opencadc.vospace.transfer.TransferWriter; /** * Abstract integration test class with general setup and test support. diff --git a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java index 6e6483b7b..4776db0a9 100644 --- a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java +++ b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -73,8 +73,6 @@ import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.rest.Version; import ca.nrc.cadc.util.MultiValuedProperties; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Transfer; import ca.nrc.cadc.vosi.Availability; import java.io.File; import java.net.URI; @@ -88,9 +86,12 @@ import javax.naming.NamingException; import org.apache.log4j.Logger; import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.transfer.StorageSiteRule; import org.opencadc.permissions.ReadGrant; import org.opencadc.permissions.WriteGrant; import org.opencadc.permissions.client.PermissionsCheck; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Transfer; /** * Abstract class for all that raven action classes have in common, diff --git a/raven/src/main/java/org/opencadc/raven/GetFilesAction.java b/raven/src/main/java/org/opencadc/raven/GetFilesAction.java index 7ecaf48e2..677b36edd 100644 --- a/raven/src/main/java/org/opencadc/raven/GetFilesAction.java +++ b/raven/src/main/java/org/opencadc/raven/GetFilesAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2021. (c) 2021. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -69,10 +69,6 @@ import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.reg.Standards; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Protocol; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.VOS; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URI; @@ -80,6 +76,11 @@ import java.util.Iterator; import java.util.List; import org.apache.log4j.Logger; +import org.opencadc.inventory.transfer.ProtocolsGenerator; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; /** * Class to execute a "files" GET action. @@ -148,7 +149,7 @@ URI getFirstURL() throws ResourceNotFoundException, IOException { } URI ret = URI.create(protos.get(0).getEndpoint()); - if (pg.storageResolverAdded && protos.size() == 1) { + if (pg.getStorageResolverAdded() && protos.size() == 1) { logInfo.setMessage("external redirect: " + ret.toASCIIString()); } // for now return the first URL in the list diff --git a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java index d0a671855..f60764a95 100644 --- a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java +++ b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2021. (c) 2021. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -70,18 +70,19 @@ import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.rest.SyncOutput; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Protocol; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.VOS; import java.net.HttpURLConnection; import java.net.URL; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.db.StorageSiteDAO; +import org.opencadc.inventory.transfer.ProtocolsGenerator; import org.opencadc.permissions.ReadGrant; import org.opencadc.permissions.TokenTool; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; /** * Interface with inventory to get the metadata of an artifact. diff --git a/raven/src/main/java/org/opencadc/raven/PostAction.java b/raven/src/main/java/org/opencadc/raven/PostAction.java index 66cec4129..db68d3f86 100644 --- a/raven/src/main/java/org/opencadc/raven/PostAction.java +++ b/raven/src/main/java/org/opencadc/raven/PostAction.java @@ -70,11 +70,6 @@ import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.rest.InlineContentException; import ca.nrc.cadc.rest.InlineContentHandler; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.TransferReader; -import ca.nrc.cadc.vos.TransferWriter; -import ca.nrc.cadc.vos.VOS; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -82,6 +77,12 @@ import java.util.List; import org.apache.log4j.Logger; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.inventory.transfer.ProtocolsGenerator; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Transfer; +import org.opencadc.vospace.transfer.TransferReader; +import org.opencadc.vospace.transfer.TransferWriter; /** * Given a transfer request object return a transfer response object with all diff --git a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java index b4e09db31..76d9df269 100644 --- a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java +++ b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2021. (c) 2021. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -73,7 +73,6 @@ import ca.nrc.cadc.util.StringUtil; import ca.nrc.cadc.vosi.Availability; import ca.nrc.cadc.vosi.AvailabilityClient; - import java.io.File; import java.net.URI; import java.net.URISyntaxException; @@ -83,17 +82,16 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; - import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingException; - import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.StorageSite; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; +import org.opencadc.inventory.transfer.StorageSiteRule; /** * diff --git a/vault/build.gradle b/vault/build.gradle index 7fd069259..80d31e21b 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -35,7 +35,7 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' - compile 'org.opencadc:cadc-vos-server-alt:[2.0,)' + compile 'org.opencadc:cadc-vos-server:[2.0,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' compile 'org.opencadc:cadc-uws-server:[1.2.19,)' From 234c073dd657533dd1444cc690295ce847114942 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Dec 2023 14:07:15 -0800 Subject: [PATCH 068/186] moved storage site availability check to library --- .../StorageSiteAvailabilityCheck.java | 198 ++++++++++++++++++ .../org/opencadc/raven/RavenInitAction.java | 118 +---------- 2 files changed, 200 insertions(+), 116 deletions(-) create mode 100644 cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java new file mode 100644 index 000000000..fd63250db --- /dev/null +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java @@ -0,0 +1,198 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ + */ + +package org.opencadc.inventory.transfer; + +import ca.nrc.cadc.vosi.Availability; +import ca.nrc.cadc.vosi.AvailabilityClient; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import org.apache.log4j.Logger; +import org.opencadc.inventory.StorageSite; +import org.opencadc.inventory.db.StorageSiteDAO; + +/** + * Background check of storage site availability. This class stores and + * maintains a Map of site availability check results. ProtocolsGenerator + * consults the map when generating URLs to files so it can skip sites that + * are off-line. + * + * @author pdowler + */ +public class StorageSiteAvailabilityCheck implements Runnable { + private static final Logger log = Logger.getLogger(StorageSiteAvailabilityCheck.class); + + static final int AVAILABILITY_CHECK_TIMEOUT = 30; //secs + static final int AVAILABILITY_FULL_CHECK_TIMEOUT = 300; //secs + + private final StorageSiteDAO storageSiteDAO; + private final Map siteStates; + private final Map siteAvailabilities; + + public StorageSiteAvailabilityCheck(StorageSiteDAO storageSiteDAO, String siteAvailabilitiesKey) { + this.storageSiteDAO = storageSiteDAO; + this.siteStates = new HashMap<>(); + this.siteAvailabilities = new HashMap(); + + try { + Context initialContext = new InitialContext(); + // check if key already bound, if so unbind + try { + initialContext.unbind(siteAvailabilitiesKey); + } catch (NamingException ignore) { + // ignore + } + initialContext.bind(siteAvailabilitiesKey, this.siteAvailabilities); + } catch (NamingException e) { + throw new IllegalStateException(String.format("unable to bind %s to initial context: %s", + siteAvailabilitiesKey, e.getMessage()), e); + } + } + + @Override + public void run() { + int lastSiteQuerySecs = 0; + while (true) { + Set sites = storageSiteDAO.list(); + if (lastSiteQuerySecs >= AVAILABILITY_FULL_CHECK_TIMEOUT) { + sites = storageSiteDAO.list(); + lastSiteQuerySecs = 0; + } else { + lastSiteQuerySecs += AVAILABILITY_CHECK_TIMEOUT; + } + + for (StorageSite site : sites) { + URI resourceID = site.getResourceID(); + log.debug("checking site: " + resourceID); + SiteState siteState = this.siteStates.get(resourceID); + if (siteState == null) { + siteState = new SiteState(false, 0); + } + boolean minDetail = siteState.isMinDetail(); + Availability availability; + try { + availability = getAvailability(resourceID, minDetail); + } catch (Exception e) { + availability = new Availability(false, e.getMessage()); + log.debug(String.format("failed %s - %s", resourceID, e.getMessage())); + } + final boolean prev = siteState.available; + siteState.available = availability.isAvailable(); + this.siteStates.put(resourceID, siteState); + this.siteAvailabilities.put(resourceID, availability); + String message = String.format("%s %s - %s", minDetail ? "MIN" : "FULL", + resourceID, siteState.available ? "UP" : "DOWN"); + if (!siteState.available) { + log.warn(message); + } else if (prev != siteState.available) { + log.info(message); + } else { + log.debug(message); + } + } + + try { + log.debug(String.format("sleep availability checks for %d secs", AVAILABILITY_CHECK_TIMEOUT)); + Thread.sleep(AVAILABILITY_CHECK_TIMEOUT * 1000); + } catch (InterruptedException e) { + throw new IllegalStateException("AvailabilityCheck thread interrupted during sleep"); + } + } + } + + private Availability getAvailability(URI resourceID, boolean minDetail) { + AvailabilityClient client = new AvailabilityClient(resourceID, minDetail); + return client.getAvailability(); + } + + private class SiteState { + + public boolean available; + public int lastFullCheckSecs; + + public SiteState(boolean available, int lastFullCheckSecs) { + this.available = available; + this.lastFullCheckSecs = lastFullCheckSecs; + } + + public boolean isMinDetail() { + log.debug(String.format("isMinDetail() available=%b, lastFullCheckSecs=%d", + available, lastFullCheckSecs)); + if (this.available && this.lastFullCheckSecs < AVAILABILITY_FULL_CHECK_TIMEOUT) { + this.lastFullCheckSecs += AVAILABILITY_CHECK_TIMEOUT; + return true; + } + this.lastFullCheckSecs = 0; + return false; + } + } +} diff --git a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java index 76d9df269..236aabb61 100644 --- a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java +++ b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java @@ -71,8 +71,6 @@ import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; import ca.nrc.cadc.util.StringUtil; -import ca.nrc.cadc.vosi.Availability; -import ca.nrc.cadc.vosi.AvailabilityClient; import java.io.File; import java.net.URI; import java.net.URISyntaxException; @@ -80,17 +78,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; -import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingException; import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; -import org.opencadc.inventory.StorageSite; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; +import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; /** @@ -119,9 +115,6 @@ public class RavenInitAction extends InitAction { static final String DEV_AUTH_ONLY_KEY = RAVEN_KEY + ".authenticateOnly"; - static final int AVAILABILITY_CHECK_TIMEOUT = 30; //secs - static final int AVAILABILITY_FULL_CHECK_TIMEOUT = 300; //secs - // set init initConfig, used by subsequent init methods MultiValuedProperties props; private String siteAvailabilitiesKey; @@ -216,7 +209,7 @@ void initAvailabilityCheck() { this.siteAvailabilitiesKey = this.appName + RavenInitAction.JNDI_AVAILABILITY_NAME; terminate(); - this.availabilityCheck = new Thread(new AvailabilityCheck(storageSiteDAO, this.siteAvailabilitiesKey)); + this.availabilityCheck = new Thread(new StorageSiteAvailabilityCheck(storageSiteDAO, this.siteAvailabilitiesKey)); this.availabilityCheck.setDaemon(true); this.availabilityCheck.start(); } @@ -370,111 +363,4 @@ static Map getStorageSiteRules(MultiValuedProperties props } return prefs; } - - private static class AvailabilityCheck implements Runnable { - private final StorageSiteDAO storageSiteDAO; - private final Map siteStates; - private final Map siteAvailabilities; - - public AvailabilityCheck(StorageSiteDAO storageSiteDAO, String siteAvailabilitiesKey) { - this.storageSiteDAO = storageSiteDAO; - this.siteStates = new HashMap(); - this.siteAvailabilities = new HashMap(); - - try { - Context initialContext = new InitialContext(); - // check if key already bound, if so unbind - try { - initialContext.unbind(siteAvailabilitiesKey); - } catch (NamingException ignore) { - // ignore - } - initialContext.bind(siteAvailabilitiesKey, this.siteAvailabilities); - } catch (NamingException e) { - throw new IllegalStateException(String.format("unable to bind %s to initial context: %s", - siteAvailabilitiesKey, e.getMessage()), e); - } - } - - @Override - public void run() { - int lastSiteQuerySecs = 0; - while (true) { - Set sites = storageSiteDAO.list(); - if (lastSiteQuerySecs >= AVAILABILITY_FULL_CHECK_TIMEOUT) { - sites = storageSiteDAO.list(); - lastSiteQuerySecs = 0; - } else { - lastSiteQuerySecs += AVAILABILITY_CHECK_TIMEOUT; - } - - for (StorageSite site: sites) { - URI resourceID = site.getResourceID(); - log.debug("checking site: " + resourceID); - SiteState siteState = this.siteStates.get(resourceID); - if (siteState == null) { - siteState = new SiteState(false, 0); - } - boolean minDetail = siteState.isMinDetail(); - Availability availability; - try { - availability = getAvailability(resourceID, minDetail); - } catch (Exception e) { - availability = new Availability(false, e.getMessage()); - log.debug(String.format("availability check failed %s - %s", resourceID, e.getMessage())); - } - final boolean prev = siteState.available; - siteState.available = availability.isAvailable(); - this.siteStates.put(resourceID, siteState); - this.siteAvailabilities.put(resourceID, availability); - String message = String.format("availability check %s %s - %s", minDetail ? "MIN" : "FULL", - resourceID, siteState.available ? "UP" : "DOWN"); - if (!siteState.available) { - log.warn(message); - } else if (prev != siteState.available) { - log.info(message); - } else { - log.debug(message); - } - } - - try { - log.debug(String.format("sleep availability checks for %d secs", AVAILABILITY_CHECK_TIMEOUT)); - Thread.sleep(AVAILABILITY_CHECK_TIMEOUT * 1000); - } catch (InterruptedException e) { - throw new IllegalStateException("AvailabilityCheck thread interrupted during sleep"); - } - } - } - - private Availability getAvailability(URI resourceID, boolean minDetail) { - AvailabilityClient client = new AvailabilityClient(resourceID, minDetail); - return client.getAvailability(); - } - - private class SiteState { - - public boolean available; - public int lastFullCheckSecs; - - public SiteState(boolean available, int lastFullCheckSecs) { - this.available = available; - this.lastFullCheckSecs = lastFullCheckSecs; - } - - public boolean isMinDetail() { - log.debug(String.format("isMinDetail() available=%b, lastFullCheckSecs=%d", - available, lastFullCheckSecs)); - if (this.available && this.lastFullCheckSecs < AVAILABILITY_FULL_CHECK_TIMEOUT) { - this.lastFullCheckSecs += AVAILABILITY_CHECK_TIMEOUT; - return true; - } - this.lastFullCheckSecs = 0; - return false; - } - - } - - } - } From ff50827ea9f204362da33b2247decf3da59b3129 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Dec 2023 15:59:37 -0800 Subject: [PATCH 069/186] cadc-inventory-db: add KeyPairDAO to support persistence of pre-auth key pairs in raven and vault --- .../opencadc/inventory/db/KeyPairDAOTest.java | 203 ++++++++++++++++++ .../java/org/opencadc/inventory/KeyPair.java | 139 ++++++++++++ .../org/opencadc/inventory/db/KeyPairDAO.java | 144 +++++++++++++ .../opencadc/inventory/db/SQLGenerator.java | 173 ++++++++++++++- .../inventory/db/version/InitDatabase.java | 7 +- .../opencadc/vospace/db/InitDatabaseVOS.java | 6 +- .../src/main/resources/inventory.KeyPair.sql | 12 ++ 7 files changed, 677 insertions(+), 7 deletions(-) create mode 100644 cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/KeyPairDAOTest.java create mode 100644 cadc-inventory-db/src/main/java/org/opencadc/inventory/KeyPair.java create mode 100644 cadc-inventory-db/src/main/java/org/opencadc/inventory/db/KeyPairDAO.java create mode 100644 cadc-inventory-db/src/main/resources/inventory.KeyPair.sql diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/KeyPairDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/KeyPairDAOTest.java new file mode 100644 index 000000000..5eb32c7b4 --- /dev/null +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/KeyPairDAOTest.java @@ -0,0 +1,203 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.inventory.db; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import java.security.MessageDigest; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.opencadc.inventory.KeyPair; +import org.opencadc.inventory.db.version.InitDatabase; + +/** + * + * @author pdowler + */ +public class KeyPairDAOTest { + private static final Logger log = Logger.getLogger(KeyPairDAOTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.db.version", Level.DEBUG); + } + + KeyPairDAO dao = new KeyPairDAO(); + + public KeyPairDAOTest()throws Exception { + DBConfig dbrc = new DBConfig(); + ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); + DBUtil.createJNDIDataSource("jdbc/KeyPairDAOTest", cc); + + Map config = new TreeMap(); + config.put(SQLGenerator.class.getName(), SQLGenerator.class); + config.put("jndiDataSourceName", "jdbc/KeyPairDAOTest"); + config.put("database", TestUtil.DATABASE); + config.put("schema", TestUtil.SCHEMA); + dao.setConfig(config); + } + + @Before + public void setup() + throws Exception + { + log.info("init database..."); + InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + init.doInit(); + log.info("init database... OK"); + + log.info("clearing old content..."); + SQLGenerator gen = dao.getSQLGenerator(); + DataSource ds = dao.getDataSource(); + String sql = "delete from " + gen.getTable(KeyPair.class); + log.info("pre-test cleanup: " + sql); + ds.getConnection().createStatement().execute(sql); + log.info("clearing old content... OK"); + } + + @Test + public void testPutGetUpdateDelete() { + String name = "testPutGetUpdateDelete"; + Random rnd = new Random(); + byte[] publicKey = new byte[128]; + rnd.nextBytes(publicKey); + byte[] privateKey = new byte[512]; + rnd.nextBytes(privateKey); + + + try { + KeyPair expected = new KeyPair(name, publicKey, privateKey); + + KeyPair notFound = dao.get(expected.getID()); + Assert.assertNull(notFound); + + dao.put(expected); + + // persistence assigns entity state before put + Assert.assertNotNull(expected.getLastModified()); + Assert.assertNotNull(expected.getMetaChecksum()); + URI mcs = expected.getMetaChecksum(); + + URI mcs0 = expected.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("put metachecksum", mcs, mcs0); + + // get by ID + KeyPair fid = dao.get(expected.getID()); + Assert.assertNotNull(fid); + Assert.assertEquals(expected.getName(), fid.getName()); + Assert.assertEquals(expected.getPublicKey().length, fid.getPublicKey().length); + Assert.assertEquals(expected.getPrivateKey().length, fid.getPrivateKey().length); + URI mcs1 = fid.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("round trip metachecksum", mcs, mcs1); + + // get by name + fid = dao.get(name); + Assert.assertNotNull(fid); + Assert.assertEquals(expected.getName(), fid.getName()); + Assert.assertEquals(expected.getPublicKey().length, fid.getPublicKey().length); + Assert.assertEquals(expected.getPrivateKey().length, fid.getPrivateKey().length); + URI mcs2 = fid.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("round trip metachecksum", mcs, mcs2); + + // TODO: udpate + + // list + Set keys = dao.list(); + Assert.assertNotNull(keys); + Assert.assertEquals(1, keys.size()); + Iterator iter = keys.iterator(); + Assert.assertTrue(iter.hasNext()); + KeyPair actual = iter.next(); + Assert.assertEquals(expected.getPublicKey().length, fid.getPublicKey().length); + Assert.assertEquals(expected.getPrivateKey().length, fid.getPrivateKey().length); + URI mcs3 = fid.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("round trip metachecksum", mcs, mcs3); + + // delete + dao.delete(expected.getID()); + KeyPair deleted = dao.get(expected.getID()); + Assert.assertNull(deleted); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/KeyPair.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/KeyPair.java new file mode 100644 index 000000000..2559e4f6e --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/KeyPair.java @@ -0,0 +1,139 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.inventory; + +import java.util.Objects; +import java.util.UUID; +import org.apache.log4j.Logger; + +/** + * Entity class to support storing a key pair in the database. + * + * @author pdowler + */ +public class KeyPair extends Entity implements Comparable { + private static final Logger log = Logger.getLogger(KeyPair.class); + + private final String name; + private final byte[] publicKey; + private final byte[] privateKey; + + public KeyPair(String name, byte[] publicKey, byte[] privateKey) { + super(); + InventoryUtil.assertNotNull(KeyPair.class, "name", name); + InventoryUtil.assertNotNull(KeyPair.class, "publicKey", publicKey); + InventoryUtil.assertNotNull(KeyPair.class, "privateKey", privateKey); + this.name = name; + this.publicKey = publicKey; + this.privateKey = privateKey; + } + + // ctor for DAO class + public KeyPair(UUID id, String name, byte[] publicKey, byte[] privateKey) { + super(id); + InventoryUtil.assertNotNull(KeyPair.class, "name", name); + InventoryUtil.assertNotNull(KeyPair.class, "publicKey", publicKey); + InventoryUtil.assertNotNull(KeyPair.class, "privateKey", privateKey); + this.name = name; + this.publicKey = publicKey; + this.privateKey = privateKey; + } + + public String getName() { + return name; + } + + public byte[] getPublicKey() { + return publicKey; + } + + public byte[] getPrivateKey() { + return privateKey; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 43 * hash + Objects.hashCode(this.name); + return hash; + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + KeyPair f = (KeyPair) o; + return this.compareTo(f) == 0; + } + + @Override + public int compareTo(KeyPair t) { + return name.compareTo(t.name); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/KeyPairDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/KeyPairDAO.java new file mode 100644 index 000000000..6c67df8df --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/KeyPairDAO.java @@ -0,0 +1,144 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2022. (c) 2022. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.inventory.db; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import org.apache.log4j.Logger; +import org.opencadc.inventory.KeyPair; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.jdbc.core.JdbcTemplate; + +/** + * Simple DAO class to store and retrieve public-private key pairs. + * + * @author pdowler + */ +public class KeyPairDAO extends AbstractDAO { + private static final Logger log = Logger.getLogger(KeyPairDAO.class); + + public KeyPairDAO() { + super(true); + } + + public KeyPairDAO(AbstractDAO src) { + super(src); + } + + public KeyPair get(UUID id) { + return super.get(KeyPair.class, id); + } + + public KeyPair get(String name) { + if (name == null) { + throw new IllegalArgumentException("name cannot be null"); + } + checkInit(); + log.debug("GET: " + name); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + + SQLGenerator.KeyPairGet get = ( SQLGenerator.KeyPairGet) gen.getEntityGet(KeyPair.class); + get.setName(name); + KeyPair ret = get.execute(jdbc); + return ret; + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("GET: " + name + " " + dt + "ms"); + } + throw new RuntimeException("BUG: should be unreachable"); + } + + public void delete(UUID id) { + super.delete(KeyPair.class, id); + } + + public Set list() { + checkInit(); + log.debug("LIST"); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + EntityList get = gen.getEntityList(KeyPair.class); + Set result = get.query(jdbc); + return result; + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("LIST: " + dt + "ms"); + } + throw new RuntimeException("BUG: should be unreachable"); + } +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 6ac3dcbb0..629571faa 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -94,6 +94,7 @@ import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.inventory.KeyPair; import org.opencadc.inventory.ObsoleteStorageLocation; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; @@ -108,7 +109,6 @@ import org.opencadc.vospace.NodeProperty; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.PreparedStatementCreator; import org.springframework.jdbc.core.ResultSetExtractor; import org.springframework.jdbc.core.RowMapper; @@ -162,6 +162,7 @@ protected void init() { // internal this.tableMap.put(ObsoleteStorageLocation.class, pref + ObsoleteStorageLocation.class.getSimpleName()); this.tableMap.put(HarvestState.class, pref + HarvestState.class.getSimpleName()); + this.tableMap.put(KeyPair.class, pref + KeyPair.class.getSimpleName()); String[] cols = new String[] { "uri", // first column is logical key @@ -218,6 +219,16 @@ protected void init() { }; this.columnMap.put(HarvestState.class, cols); + cols = new String[] { + "name", + "publicKey", + "privateKey", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(KeyPair.class, cols); + // optional vospace log.debug("vosSchema: " + vosSchema); if (vosSchema != null) { @@ -304,7 +315,9 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { if (StorageSite.class.equals(c)) { return new StorageSiteGet(forUpdate); } - + if (KeyPair.class.equals(c)) { + return new KeyPairGet(forUpdate); + } if (Node.class.equals(c)) { return new NodeGet(forUpdate); } @@ -335,6 +348,8 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { return new HarvestStateGet(); } + + throw new UnsupportedOperationException("entity-get: " + c.getName()); } @@ -374,6 +389,9 @@ public EntityList getEntityList(Class c) { if (StorageSite.class.equals(c)) { return new StorageSiteList(); } + if (KeyPair.class.equals(c)) { + return new KeyPairList(); + } throw new UnsupportedOperationException("entity-list: " + c.getName()); } @@ -404,6 +422,9 @@ public EntityPut getEntityPut(Class c, boolean update) { if (HarvestState.class.equals(c)) { return new HarvestStatePut(update); } + if (KeyPair.class.equals(c)) { + return new KeyPairPut(update); + } if (Node.class.isAssignableFrom(c)) { return new NodePut(update); } @@ -831,6 +852,76 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } + class KeyPairGet implements EntityGet { + private UUID id; + private String name; + private final boolean forUpdate; + + public KeyPairGet(boolean forUpdate) { + this.forUpdate = forUpdate; + } + + @Override + public void setID(UUID id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public KeyPair execute(JdbcTemplate jdbc) { + return (KeyPair) jdbc.query(this, new KeyPairExtractor()); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + StringBuilder sb = getSelectFromSQL(KeyPair.class, false); + sb.append(" WHERE "); + if (id != null) { + String col = getKeyColumn(KeyPair.class, true); + sb.append(col).append(" = ?"); + } else if (name != null) { + sb.append("name = ?"); + } else { + throw new IllegalStateException("primary key and name are both null"); + } + if (forUpdate) { + sb.append(" FOR UPDATE"); + } + String sql = sb.toString(); + log.debug("KeyPairGet: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + if (id != null) { + prep.setObject(1, id); + } else { + prep.setString(1, name); + } + return prep; + } + } + + private class KeyPairList implements EntityList { + + @Override + public Set query(JdbcTemplate jdbc) { + List keys = (List) jdbc.query(this, new KeyPairRowMapper()); + Set ret = new TreeSet<>(); + ret.addAll(keys); + return ret; + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + StringBuilder sb = getSelectFromSQL(KeyPair.class, false); + String sql = sb.toString(); + log.debug("KeyPairList: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + return prep; + } + } + public class NodeGet implements EntityGet { private UUID id; private ContainerNode parent; @@ -1266,6 +1357,50 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } + private class KeyPairPut implements EntityPut { + private final Calendar utc = Calendar.getInstance(DateUtil.UTC); + private final boolean update; + private KeyPair value; + + KeyPairPut(boolean update) { + this.update = update; + } + + @Override + public void setValue(KeyPair value) { + this.value = value; + } + + @Override + public void execute(JdbcTemplate jdbc) { + jdbc.update(this); + } + + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + String sql = null; + if (update) { + sql = getUpdateSQL(KeyPair.class); + + } else { + sql = getInsertSQL(KeyPair.class); + } + log.debug("KeyPairPut: " + sql); + PreparedStatement prep = conn.prepareStatement(sql); + int col = 1; + + prep.setString(col++, value.getName()); + prep.setBytes(col++, value.getPublicKey()); + prep.setBytes(col++, value.getPrivateKey()); + + prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); + prep.setString(col++, value.getMetaChecksum().toASCIIString()); + prep.setObject(col++, value.getID()); + + return prep; + } + } + private class NodePut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; @@ -1878,6 +2013,40 @@ public StorageSite extractData(ResultSet rs) throws SQLException, DataAccessExce } } + private class KeyPairRowMapper implements RowMapper { + Calendar utc = Calendar.getInstance(DateUtil.UTC); + + @Override + public KeyPair mapRow(ResultSet rs, int i) throws SQLException { + int col = 1; + final String name = rs.getString(col++); + final byte[] pub = rs.getBytes(col++); + final byte[] priv = rs.getBytes(col++); + + final Date lastModified = Util.getDate(rs, col++, utc); + final URI metaChecksum = Util.getURI(rs, col++); + final UUID id = Util.getUUID(rs, col++); + + KeyPair s = new KeyPair(id, name, pub, priv); + InventoryUtil.assignLastModified(s, lastModified); + InventoryUtil.assignMetaChecksum(s, metaChecksum); + return s; + } + } + + private class KeyPairExtractor implements ResultSetExtractor { + final Calendar utc = Calendar.getInstance(DateUtil.UTC); + + @Override + public KeyPair extractData(ResultSet rs) throws SQLException, DataAccessException { + if (!rs.next()) { + return null; + } + KeyPairRowMapper m = new KeyPairRowMapper(); + return m.mapRow(rs, 1); + } + } + private class DeletedArtifactEventExtractor implements ResultSetExtractor { final Calendar utc = Calendar.getInstance(DateUtil.UTC); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java index 6b299106a..0c632b4fa 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java @@ -79,8 +79,8 @@ public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase { private static final Logger log = Logger.getLogger(InitDatabase.class); public static final String MODEL_NAME = "storage-inventory"; - public static final String MODEL_VERSION = "0.14"; - public static final String PREV_MODEL_VERSION = "0.10"; + public static final String MODEL_VERSION = "0.15"; + public static final String PREV_MODEL_VERSION = "0.14"; //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; static String[] CREATE_SQL = new String[] { @@ -92,11 +92,12 @@ public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase { "inventory.DeletedStorageLocationEvent.sql", "inventory.StorageLocationEvent.sql", "inventory.HarvestState.sql", + "inventory.KeyPair.sql", "inventory.permissions.sql" }; static String[] UPGRADE_SQL = new String[] { - "inventory.StorageLocationEvent.sql", + "inventory.KeyPair.sql", "inventory.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java index 093618755..16ff9ad86 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -80,18 +80,20 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); public static final String MODEL_NAME = "vospace-inventory"; - public static final String MODEL_VERSION = "0.1"; - public static final String PREV_MODEL_VERSION = "n/a"; + public static final String MODEL_VERSION = "0.2"; + public static final String PREV_MODEL_VERSION = "0.1"; //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; static String[] CREATE_SQL = new String[] { "vos.ModelVersion.sql", "vos.Node.sql", "vos.DeletedNodeEvent.sql", + "inventory.KeyPair.sql", "inventory.permissions.sql" }; static String[] UPGRADE_SQL = new String[] { + "inventory.KeyPair.sql", "inventory.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql b/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql new file mode 100644 index 000000000..1902222a9 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql @@ -0,0 +1,12 @@ + +create table .KeyPair ( + name varchar(32) not null, + publicKey bytea not null, + privateKey bytea not null, + + id uuid not null primary key, + lastModified timestamp not null, + metaChecksum varchar(136) not null +); + +create unique index kp_name_index on .KeyPair(name); From a66a21dd1379793236ba8cf950bb7b6fa7ab9671 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Dec 2023 16:16:47 -0800 Subject: [PATCH 070/186] vault: uws subpackage --- .../vault/{ => uws}/RecursiveDeleteNodeJobManager.java | 2 +- .../vault/{ => uws}/RecursiveNodePropsJobManager.java | 2 +- .../java/org/opencadc/vault/{ => uws}/VaultJobManager.java | 2 +- vault/src/main/webapp/WEB-INF/web.xml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename vault/src/main/java/org/opencadc/vault/{ => uws}/RecursiveDeleteNodeJobManager.java (99%) rename vault/src/main/java/org/opencadc/vault/{ => uws}/RecursiveNodePropsJobManager.java (99%) rename vault/src/main/java/org/opencadc/vault/{ => uws}/VaultJobManager.java (99%) diff --git a/vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java b/vault/src/main/java/org/opencadc/vault/uws/RecursiveDeleteNodeJobManager.java similarity index 99% rename from vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java rename to vault/src/main/java/org/opencadc/vault/uws/RecursiveDeleteNodeJobManager.java index 87619dd2a..88cc56629 100644 --- a/vault/src/main/java/org/opencadc/vault/RecursiveDeleteNodeJobManager.java +++ b/vault/src/main/java/org/opencadc/vault/uws/RecursiveDeleteNodeJobManager.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.vault; +package org.opencadc.vault.uws; import ca.nrc.cadc.uws.server.JobExecutor; import ca.nrc.cadc.uws.server.JobPersistence; diff --git a/vault/src/main/java/org/opencadc/vault/RecursiveNodePropsJobManager.java b/vault/src/main/java/org/opencadc/vault/uws/RecursiveNodePropsJobManager.java similarity index 99% rename from vault/src/main/java/org/opencadc/vault/RecursiveNodePropsJobManager.java rename to vault/src/main/java/org/opencadc/vault/uws/RecursiveNodePropsJobManager.java index 6117aa67c..411f8b67e 100644 --- a/vault/src/main/java/org/opencadc/vault/RecursiveNodePropsJobManager.java +++ b/vault/src/main/java/org/opencadc/vault/uws/RecursiveNodePropsJobManager.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.vault; +package org.opencadc.vault.uws; import ca.nrc.cadc.uws.server.JobExecutor; import ca.nrc.cadc.uws.server.JobPersistence; diff --git a/vault/src/main/java/org/opencadc/vault/VaultJobManager.java b/vault/src/main/java/org/opencadc/vault/uws/VaultJobManager.java similarity index 99% rename from vault/src/main/java/org/opencadc/vault/VaultJobManager.java rename to vault/src/main/java/org/opencadc/vault/uws/VaultJobManager.java index 7968e7f52..139390ac8 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultJobManager.java +++ b/vault/src/main/java/org/opencadc/vault/uws/VaultJobManager.java @@ -67,7 +67,7 @@ ************************************************************************ */ -package org.opencadc.vault; +package org.opencadc.vault.uws; import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.uws.server.JobPersistence; diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index c034a2f70..5d1831357 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -81,7 +81,7 @@ ca.nrc.cadc.uws.server.JobManager - org.opencadc.vault.RecursiveDeleteNodeJobManager + org.opencadc.vault.uws.RecursiveDeleteNodeJobManager 3 @@ -104,7 +104,7 @@ ca.nrc.cadc.uws.server.JobManager - org.opencadc.vault.RecursiveNodePropsJobManager + org.opencadc.vault.uws.RecursiveNodePropsJobManager ca.nrc.cadc.rest.InlineContentHandler From 1db9f37ccfa0d8caa2183937c2c12287c8cbc9ef Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Dec 2023 17:18:48 -0800 Subject: [PATCH 071/186] ProtocolsGenerator ctor change changes key files to an already instantiated TokenTool --- .../inventory/transfer/ProtocolsGenerator.java | 15 ++++++--------- .../java/org/opencadc/raven/ArtifactAction.java | 15 +++++++-------- .../java/org/opencadc/raven/GetFilesAction.java | 2 +- .../java/org/opencadc/raven/HeadFilesAction.java | 6 +++--- .../main/java/org/opencadc/raven/PostAction.java | 2 +- 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index cfbce5070..d6bc891ae 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -124,8 +124,7 @@ public class ProtocolsGenerator { private final ArtifactDAO artifactDAO; private final DeletedArtifactEventDAO deletedArtifactEventDAO; private final String user; - private final File publicKeyFile; - private final File privateKeyFile; + private final TokenTool tokenGen; private final Map siteAvailabilities; private final Map siteRules; private final StorageResolver storageResolver; @@ -135,14 +134,13 @@ public class ProtocolsGenerator { boolean storageResolverAdded = false; - public ProtocolsGenerator(ArtifactDAO artifactDAO, File publicKeyFile, File privateKeyFile, String user, + public ProtocolsGenerator(ArtifactDAO artifactDAO, TokenTool tokenGen, String user, Map siteAvailabilities, Map siteRules, boolean preventNotFound, StorageResolver storageResolver) { this.artifactDAO = artifactDAO; this.deletedArtifactEventDAO = new DeletedArtifactEventDAO(this.artifactDAO); this.user = user; - this.publicKeyFile = publicKeyFile; - this.privateKeyFile = privateKeyFile; + this.tokenGen = tokenGen; this.siteAvailabilities = siteAvailabilities; this.siteRules = siteRules; this.preventNotFound = preventNotFound; @@ -156,13 +154,12 @@ public boolean getStorageResolverAdded() { public List getProtocols(Transfer transfer) throws ResourceNotFoundException, IOException { String authToken = null; URI artifactURI = transfer.getTargets().get(0); // see PostAction line ~127 - if (publicKeyFile != null && privateKeyFile != null) { + if (tokenGen != null) { // create an auth token - TokenTool tk = new TokenTool(publicKeyFile, privateKeyFile); if (transfer.getDirection().equals(Direction.pullFromVoSpace)) { - authToken = tk.generateToken(artifactURI, ReadGrant.class, user); + authToken = tokenGen.generateToken(artifactURI, ReadGrant.class, user); } else { - authToken = tk.generateToken(artifactURI, WriteGrant.class, user); + authToken = tokenGen.generateToken(artifactURI, WriteGrant.class, user); } } diff --git a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java index 4776db0a9..14205786c 100644 --- a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java +++ b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java @@ -88,6 +88,7 @@ import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.transfer.StorageSiteRule; import org.opencadc.permissions.ReadGrant; +import org.opencadc.permissions.TokenTool; import org.opencadc.permissions.WriteGrant; import org.opencadc.permissions.client.PermissionsCheck; import org.opencadc.vospace.transfer.Direction; @@ -110,8 +111,7 @@ public abstract class ArtifactAction extends RestAction { // immutable state set in constructor protected final ArtifactDAO artifactDAO; - protected final File publicKeyFile; - protected final File privateKeyFile; + protected final TokenTool tokenGen; protected final List readGrantServices = new ArrayList<>(); protected final List writeGrantServices = new ArrayList<>(); protected StorageResolver storageResolver; @@ -126,8 +126,7 @@ public abstract class ArtifactAction extends RestAction { ArtifactAction(boolean init) { super(); this.authenticateOnly = false; - this.publicKeyFile = null; - this.privateKeyFile = null; + this.tokenGen = null; this.artifactDAO = null; this.preventNotFound = false; this.storageResolver = null; @@ -184,14 +183,14 @@ protected ArtifactAction() { String privkeyFileName = props.getFirstPropertyValue(RavenInitAction.PRIVKEYFILE_KEY); if (pubkeyFileName == null && privkeyFileName == null) { log.debug("public/private key preauth not enabled by config"); - this.publicKeyFile = null; - this.privateKeyFile = null; + this.tokenGen = null; } else { - this.publicKeyFile = new File(System.getProperty("user.home") + "/config/" + pubkeyFileName); - this.privateKeyFile = new File(System.getProperty("user.home") + "/config/" + privkeyFileName); + File publicKeyFile = new File(System.getProperty("user.home") + "/config/" + pubkeyFileName); + File privateKeyFile = new File(System.getProperty("user.home") + "/config/" + privkeyFileName); if (!publicKeyFile.exists() || !privateKeyFile.exists()) { throw new IllegalStateException("invalid config: missing public/private key pair files -- " + publicKeyFile + " | " + privateKeyFile); } + this.tokenGen = new TokenTool(publicKeyFile, privateKeyFile); } Map config = RavenInitAction.getDaoConfig(props); diff --git a/raven/src/main/java/org/opencadc/raven/GetFilesAction.java b/raven/src/main/java/org/opencadc/raven/GetFilesAction.java index 677b36edd..4fe15505f 100644 --- a/raven/src/main/java/org/opencadc/raven/GetFilesAction.java +++ b/raven/src/main/java/org/opencadc/raven/GetFilesAction.java @@ -140,7 +140,7 @@ URI getFirstURL() throws ResourceNotFoundException, IOException { proto.setSecurityMethod(Standards.SECURITY_METHOD_ANON); transfer.getProtocols().add(proto); - ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.publicKeyFile, this.privateKeyFile, + ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.tokenGen, this.user, this.siteAvailabilities, this.siteRules, this.preventNotFound, this.storageResolver); List protos = pg.getProtocols(transfer); diff --git a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java index f60764a95..11b2679ed 100644 --- a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java +++ b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java @@ -112,15 +112,15 @@ public void doAction() throws Exception { if (artifact == null) { if (this.preventNotFound) { // check known storage sites - ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.publicKeyFile, this.privateKeyFile, + ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.tokenGen, this.user, this.siteAvailabilities, this.siteRules, this.preventNotFound, this.storageResolver); StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO); Transfer transfer = new Transfer(artifactURI, Direction.pullFromVoSpace); Protocol proto = new Protocol(VOS.PROTOCOL_HTTPS_GET); proto.setSecurityMethod(Standards.SECURITY_METHOD_ANON); transfer.getProtocols().add(proto); - TokenTool tk = new TokenTool(publicKeyFile, privateKeyFile); - String authToken = tk.generateToken(artifactURI, ReadGrant.class, user); + // TODO: tokenGen is optional so this can fail + String authToken = tokenGen.generateToken(artifactURI, ReadGrant.class, user); artifact = pg.getUnsyncedArtifact(artifactURI, transfer, storageSiteDAO.list(), authToken); } } diff --git a/raven/src/main/java/org/opencadc/raven/PostAction.java b/raven/src/main/java/org/opencadc/raven/PostAction.java index db68d3f86..7de03bed1 100644 --- a/raven/src/main/java/org/opencadc/raven/PostAction.java +++ b/raven/src/main/java/org/opencadc/raven/PostAction.java @@ -160,7 +160,7 @@ public InlineContentHandler.Content accept(String name, String contentType, Inpu public void doAction() throws Exception { initAndAuthorize(); - ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.publicKeyFile, this.privateKeyFile, + ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.tokenGen, this.user, this.siteAvailabilities, this.siteRules, this.preventNotFound, this.storageResolver); Transfer ret = new Transfer(artifactURI, transfer.getDirection()); From fed6c9b33fc63de2596a67b0ced1c385f45286b4 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 12 Dec 2023 11:53:45 -0800 Subject: [PATCH 072/186] change ProtocolsGenerator API incomplete vault synctrans implementation --- .../vault/VaultTransferGenerator.java | 227 ++++++++++++++++++ .../vault/uws/SyncTransferManager.java | 97 ++++++++ 2 files changed, 324 insertions(+) create mode 100644 vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java create mode 100644 vault/src/main/java/org/opencadc/vault/uws/SyncTransferManager.java diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java new file mode 100644 index 000000000..abba154c8 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -0,0 +1,227 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2022. (c) 2022. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.net.ResourceNotFoundException; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.Parameter; +import ca.nrc.cadc.vosi.Availability; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import javax.security.auth.Subject; +import org.apache.log4j.Logger; +import org.opencadc.inventory.KeyPair; +import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.KeyPairDAO; +import org.opencadc.inventory.transfer.ProtocolsGenerator; +import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; +import org.opencadc.inventory.transfer.StorageSiteRule; +import org.opencadc.permissions.Grant; +import org.opencadc.permissions.ReadGrant; +import org.opencadc.permissions.TokenTool; +import org.opencadc.permissions.WriteGrant; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.LinkingException; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeNotFoundException; +import org.opencadc.vospace.VOSURI; +import org.opencadc.vospace.server.PathResolver; +import org.opencadc.vospace.server.auth.VOSpaceAuthorizer; +import org.opencadc.vospace.server.transfers.TransferGenerator; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; + +/** + * + * @author pdowler + */ +public class VaultTransferGenerator implements TransferGenerator { + private static final Logger log = Logger.getLogger(VaultTransferGenerator.class); + + static String KEY_PAIR_NAME = "vault-preauth-keys"; + + private final NodePersistenceImpl nodePersistence; + private final VOSpaceAuthorizer authorizer; + private final ArtifactDAO artifactDAO; + private final KeyPairDAO keyDAO; + + private Map siteRules = new HashMap<>(); + private Map siteAvailabilities; + + public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, KeyPairDAO keyDAO) { + this.nodePersistence = nodePersistence; + this.authorizer = new VOSpaceAuthorizer(nodePersistence); + this.artifactDAO = artifactDAO; + this.keyDAO = keyDAO; + + // TODO: get appname from ??? + String siteAvailabilitiesKey = "vault" + "-" + StorageSiteAvailabilityCheck.class.getName(); + log.debug("siteAvailabilitiesKey: " + siteAvailabilitiesKey); + try { + Context initContext = new InitialContext(); + this.siteAvailabilities = (Map) initContext.lookup(siteAvailabilitiesKey); + log.debug("found siteAvailabilities in JNDI: " + siteAvailabilitiesKey + " = " + siteAvailabilities); + for (Map.Entry me: siteAvailabilities.entrySet()) { + log.debug("found: " + me.getKey() + " = " + me.getValue()); + } + } catch (NamingException e) { + throw new IllegalStateException("JNDI lookup error", e); + } + } + + @Override + public List getEndpoints(VOSURI target, Transfer transfer, Job job, List additionalParams) throws Exception { + log.debug("getEndpoints: " + target); + if (target == null) { + throw new IllegalArgumentException("target is required"); + } + if (transfer == null) { + throw new IllegalArgumentException("transfer is required"); + } + List ret = null; + try { + Direction dir = transfer.getDirection(); + PathResolver ps = new PathResolver(nodePersistence, authorizer, true); + Node n = ps.getNode(target.getParentURI().getPath()); + // assume not null and Container already checked by caller (TransferRunner) + ContainerNode parent = (ContainerNode) n; + Node node = nodePersistence.get(parent, target.getName()); + + Subject currentSubject = AuthenticationUtil.getCurrentSubject(); + if (Direction.pushToVoSpace.equals(dir) && node == null) { + // create new data node?? this currently does not happen because the library + // create DataNode the way that CreateNodeAction would + //ret = handleDataNode(dn, transfer, currentSubject); + throw new RuntimeException("BUG: expected DataNode to be created already: " + target.getPath()); + } else if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + ret = handleDataNode(dn, transfer, currentSubject); + } else { + throw new UnsupportedOperationException(node.getClass().getSimpleName() + " transfer " + + target.getPath()); + } + } catch (NodeNotFoundException ex) { + throw new FileNotFoundException(target.getPath()); + } catch (LinkingException ex) { + throw new RuntimeException("OOPS: failed to resolve link?", ex); + } + return ret; + } + + private List handleDataNode(DataNode node, Transfer trans, Subject s) + throws IOException, ResourceNotFoundException { + log.debug("handleDataNode: " + node); + + Direction dir = trans.getDirection(); + final Map params = new TreeMap<>(); // empty for now + + // vault only supports preauth URLs using persistent key pair + //KeyPair keys = keyDAO.get(KEY_PAIR_NAME); + //TokenTool tokenGen = new TokenTool(keys.getPublicKey(), keys.getPrivateKey()); + TokenTool tokenGen = null; + + IdentityManager im = AuthenticationUtil.getIdentityManager(); + Subject caller = AuthenticationUtil.getCurrentSubject(); + Object userObject = im.toOwner(caller); + String callingUser = (userObject == null ? null : userObject.toString()); + + ProtocolsGenerator pg = new ProtocolsGenerator(artifactDAO, tokenGen, callingUser, siteAvailabilities, siteRules); + //pg.preventNotFound = true; + + Transfer artifactTrans = new Transfer(node.storageID, dir); + for (Protocol p : trans.getProtocols()) { + log.debug("requested protocol: " + p); + URI secM = p.getSecurityMethod(); + // TODO: request preauth URL specifically, not anon + if (secM == null || secM.equals(Standards.SECURITY_METHOD_ANON)) { + p.setSecurityMethod(ProtocolsGenerator.SECURITY_EMBEDDED_TOKEN); + artifactTrans.getProtocols().add(p); + } + } + + List ret = pg.getProtocols(artifactTrans); + log.warn("generated urls: " + ret.size()); + for (Protocol p : ret) { + log.warn(p.getEndpoint() + " using " + p.getSecurityMethod()); + } + return ret; + } +} diff --git a/vault/src/main/java/org/opencadc/vault/uws/SyncTransferManager.java b/vault/src/main/java/org/opencadc/vault/uws/SyncTransferManager.java new file mode 100644 index 000000000..8d5a6774a --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/uws/SyncTransferManager.java @@ -0,0 +1,97 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault.uws; + +import ca.nrc.cadc.uws.server.JobExecutor; +import ca.nrc.cadc.uws.server.JobPersistence; +import ca.nrc.cadc.uws.server.JobUpdater; +import ca.nrc.cadc.uws.server.SyncJobExecutor; +import org.apache.log4j.Logger; +import org.opencadc.vospace.server.transfers.TransferRunner; + +/** + * + * @author pdowler + */ +public class SyncTransferManager extends VaultJobManager { + private static final Logger log = Logger.getLogger(SyncTransferManager.class); + + public SyncTransferManager() { + super(); + JobPersistence jp = createJobPersistence(); + JobUpdater ju = (JobUpdater) jp; + JobExecutor jobExec = new SyncJobExecutor(ju, TransferRunner.class); + super.setJobPersistence(jp); + super.setJobExecutor(jobExec); + + // TODO: would be nice to enable a feature like destroy-on-complete instead of timed destruction + super.setMaxDestruction(60000L); + super.setMaxExecDuration(6000L); + super.setMaxQuote(60000L); + } +} From 0f194e8485cc5206bcd8bf0913b4789a26a3c6be Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Dec 2023 10:15:33 -0800 Subject: [PATCH 073/186] cadc-inventory-db: rename KeyPair to PreauthKeyPair --- cadc-inventory-db/build.gradle | 2 +- ...AOTest.java => PreauthKeyPairDAOTest.java} | 45 ++++++++-------- .../{KeyPair.java => PreauthKeyPair.java} | 24 ++++----- ...KeyPairDAO.java => PreauthKeyPairDAO.java} | 28 +++++----- .../opencadc/inventory/db/SQLGenerator.java | 52 +++++++++---------- .../src/main/resources/inventory.KeyPair.sql | 4 +- 6 files changed, 77 insertions(+), 78 deletions(-) rename cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/{KeyPairDAOTest.java => PreauthKeyPairDAOTest.java} (86%) rename cadc-inventory-db/src/main/java/org/opencadc/inventory/{KeyPair.java => PreauthKeyPair.java} (85%) rename cadc-inventory-db/src/main/java/org/opencadc/inventory/db/{KeyPairDAO.java => PreauthKeyPairDAO.java} (88%) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index d96867abb..f6f235f80 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -25,7 +25,7 @@ def git_url = 'https://github.com/opencadc/storage-inventory' mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { - compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-util:[1.10.3,2.0)' compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' compile 'org.opencadc:cadc-vos:[2.0,3.0)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/KeyPairDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java similarity index 86% rename from cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/KeyPairDAOTest.java rename to cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java index 5eb32c7b4..b07aedcc3 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/KeyPairDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java @@ -71,11 +71,12 @@ import ca.nrc.cadc.db.DBConfig; import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.util.RsaSignatureGenerator; import java.net.URI; +import java.security.KeyPair; import java.security.MessageDigest; import java.util.Iterator; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.TreeMap; import javax.sql.DataSource; @@ -84,31 +85,31 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.opencadc.inventory.KeyPair; +import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.db.version.InitDatabase; /** * * @author pdowler */ -public class KeyPairDAOTest { - private static final Logger log = Logger.getLogger(KeyPairDAOTest.class); +public class PreauthKeyPairDAOTest { + private static final Logger log = Logger.getLogger(PreauthKeyPairDAOTest.class); static { Log4jInit.setLevel("org.opencadc.inventory", Level.DEBUG); Log4jInit.setLevel("ca.nrc.cadc.db.version", Level.DEBUG); } - KeyPairDAO dao = new KeyPairDAO(); + PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); - public KeyPairDAOTest()throws Exception { + public PreauthKeyPairDAOTest()throws Exception { DBConfig dbrc = new DBConfig(); ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); - DBUtil.createJNDIDataSource("jdbc/KeyPairDAOTest", cc); + DBUtil.createJNDIDataSource("jdbc/PreauthKeyPairDAOTest", cc); Map config = new TreeMap(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); - config.put("jndiDataSourceName", "jdbc/KeyPairDAOTest"); + config.put("jndiDataSourceName", "jdbc/PreauthKeyPairDAOTest"); config.put("database", TestUtil.DATABASE); config.put("schema", TestUtil.SCHEMA); dao.setConfig(config); @@ -126,7 +127,7 @@ public void setup() log.info("clearing old content..."); SQLGenerator gen = dao.getSQLGenerator(); DataSource ds = dao.getDataSource(); - String sql = "delete from " + gen.getTable(KeyPair.class); + String sql = "delete from " + gen.getTable(PreauthKeyPair.class); log.info("pre-test cleanup: " + sql); ds.getConnection().createStatement().execute(sql); log.info("clearing old content... OK"); @@ -135,17 +136,15 @@ public void setup() @Test public void testPutGetUpdateDelete() { String name = "testPutGetUpdateDelete"; - Random rnd = new Random(); - byte[] publicKey = new byte[128]; - rnd.nextBytes(publicKey); - byte[] privateKey = new byte[512]; - rnd.nextBytes(privateKey); - - + KeyPair kp = RsaSignatureGenerator.getKeyPair(4096); + byte[] publicKey = kp.getPublic().getEncoded(); + byte[] privateKey = kp.getPrivate().getEncoded(); + log.info("generated keys (4096): " + publicKey.length + "," + privateKey.length); try { - KeyPair expected = new KeyPair(name, publicKey, privateKey); + + PreauthKeyPair expected = new PreauthKeyPair(name, publicKey, privateKey); - KeyPair notFound = dao.get(expected.getID()); + PreauthKeyPair notFound = dao.get(expected.getID()); Assert.assertNull(notFound); dao.put(expected); @@ -159,7 +158,7 @@ public void testPutGetUpdateDelete() { Assert.assertEquals("put metachecksum", mcs, mcs0); // get by ID - KeyPair fid = dao.get(expected.getID()); + PreauthKeyPair fid = dao.get(expected.getID()); Assert.assertNotNull(fid); Assert.assertEquals(expected.getName(), fid.getName()); Assert.assertEquals(expected.getPublicKey().length, fid.getPublicKey().length); @@ -179,12 +178,12 @@ public void testPutGetUpdateDelete() { // TODO: udpate // list - Set keys = dao.list(); + Set keys = dao.list(); Assert.assertNotNull(keys); Assert.assertEquals(1, keys.size()); - Iterator iter = keys.iterator(); + Iterator iter = keys.iterator(); Assert.assertTrue(iter.hasNext()); - KeyPair actual = iter.next(); + PreauthKeyPair actual = iter.next(); Assert.assertEquals(expected.getPublicKey().length, fid.getPublicKey().length); Assert.assertEquals(expected.getPrivateKey().length, fid.getPrivateKey().length); URI mcs3 = fid.computeMetaChecksum(MessageDigest.getInstance("MD5")); @@ -192,7 +191,7 @@ public void testPutGetUpdateDelete() { // delete dao.delete(expected.getID()); - KeyPair deleted = dao.get(expected.getID()); + PreauthKeyPair deleted = dao.get(expected.getID()); Assert.assertNull(deleted); } catch (Exception unexpected) { diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/KeyPair.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/PreauthKeyPair.java similarity index 85% rename from cadc-inventory-db/src/main/java/org/opencadc/inventory/KeyPair.java rename to cadc-inventory-db/src/main/java/org/opencadc/inventory/PreauthKeyPair.java index 2559e4f6e..3266918be 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/KeyPair.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/PreauthKeyPair.java @@ -76,29 +76,29 @@ * * @author pdowler */ -public class KeyPair extends Entity implements Comparable { - private static final Logger log = Logger.getLogger(KeyPair.class); +public class PreauthKeyPair extends Entity implements Comparable { + private static final Logger log = Logger.getLogger(PreauthKeyPair.class); private final String name; private final byte[] publicKey; private final byte[] privateKey; - public KeyPair(String name, byte[] publicKey, byte[] privateKey) { + public PreauthKeyPair(String name, byte[] publicKey, byte[] privateKey) { super(); - InventoryUtil.assertNotNull(KeyPair.class, "name", name); - InventoryUtil.assertNotNull(KeyPair.class, "publicKey", publicKey); - InventoryUtil.assertNotNull(KeyPair.class, "privateKey", privateKey); + InventoryUtil.assertNotNull(PreauthKeyPair.class, "name", name); + InventoryUtil.assertNotNull(PreauthKeyPair.class, "publicKey", publicKey); + InventoryUtil.assertNotNull(PreauthKeyPair.class, "privateKey", privateKey); this.name = name; this.publicKey = publicKey; this.privateKey = privateKey; } // ctor for DAO class - public KeyPair(UUID id, String name, byte[] publicKey, byte[] privateKey) { + public PreauthKeyPair(UUID id, String name, byte[] publicKey, byte[] privateKey) { super(id); - InventoryUtil.assertNotNull(KeyPair.class, "name", name); - InventoryUtil.assertNotNull(KeyPair.class, "publicKey", publicKey); - InventoryUtil.assertNotNull(KeyPair.class, "privateKey", privateKey); + InventoryUtil.assertNotNull(PreauthKeyPair.class, "name", name); + InventoryUtil.assertNotNull(PreauthKeyPair.class, "publicKey", publicKey); + InventoryUtil.assertNotNull(PreauthKeyPair.class, "privateKey", privateKey); this.name = name; this.publicKey = publicKey; this.privateKey = privateKey; @@ -128,12 +128,12 @@ public boolean equals(Object o) { if (o == null) { return false; } - KeyPair f = (KeyPair) o; + PreauthKeyPair f = (PreauthKeyPair) o; return this.compareTo(f) == 0; } @Override - public int compareTo(KeyPair t) { + public int compareTo(PreauthKeyPair t) { return name.compareTo(t.name); } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/KeyPairDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/PreauthKeyPairDAO.java similarity index 88% rename from cadc-inventory-db/src/main/java/org/opencadc/inventory/db/KeyPairDAO.java rename to cadc-inventory-db/src/main/java/org/opencadc/inventory/db/PreauthKeyPairDAO.java index 6c67df8df..aa8085594 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/KeyPairDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/PreauthKeyPairDAO.java @@ -71,7 +71,7 @@ import java.util.Set; import java.util.UUID; import org.apache.log4j.Logger; -import org.opencadc.inventory.KeyPair; +import org.opencadc.inventory.PreauthKeyPair; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; @@ -80,22 +80,22 @@ * * @author pdowler */ -public class KeyPairDAO extends AbstractDAO { - private static final Logger log = Logger.getLogger(KeyPairDAO.class); +public class PreauthKeyPairDAO extends AbstractDAO { + private static final Logger log = Logger.getLogger(PreauthKeyPairDAO.class); - public KeyPairDAO() { + public PreauthKeyPairDAO() { super(true); } - public KeyPairDAO(AbstractDAO src) { + public PreauthKeyPairDAO(AbstractDAO src) { super(src); } - public KeyPair get(UUID id) { - return super.get(KeyPair.class, id); + public PreauthKeyPair get(UUID id) { + return super.get(PreauthKeyPair.class, id); } - public KeyPair get(String name) { + public PreauthKeyPair get(String name) { if (name == null) { throw new IllegalArgumentException("name cannot be null"); } @@ -106,9 +106,9 @@ public KeyPair get(String name) { try { JdbcTemplate jdbc = new JdbcTemplate(dataSource); - SQLGenerator.KeyPairGet get = ( SQLGenerator.KeyPairGet) gen.getEntityGet(KeyPair.class); + SQLGenerator.KeyPairGet get = ( SQLGenerator.KeyPairGet) gen.getEntityGet(PreauthKeyPair.class); get.setName(name); - KeyPair ret = get.execute(jdbc); + PreauthKeyPair ret = get.execute(jdbc); return ret; } catch (BadSqlGrammarException ex) { handleInternalFail(ex); @@ -120,18 +120,18 @@ public KeyPair get(String name) { } public void delete(UUID id) { - super.delete(KeyPair.class, id); + super.delete(PreauthKeyPair.class, id); } - public Set list() { + public Set list() { checkInit(); log.debug("LIST"); long t = System.currentTimeMillis(); try { JdbcTemplate jdbc = new JdbcTemplate(dataSource); - EntityList get = gen.getEntityList(KeyPair.class); - Set result = get.query(jdbc); + EntityList get = gen.getEntityList(PreauthKeyPair.class); + Set result = get.query(jdbc); return result; } catch (BadSqlGrammarException ex) { handleInternalFail(ex); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 629571faa..4a3471075 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -94,7 +94,7 @@ import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; import org.opencadc.inventory.InventoryUtil; -import org.opencadc.inventory.KeyPair; +import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.ObsoleteStorageLocation; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; @@ -162,7 +162,7 @@ protected void init() { // internal this.tableMap.put(ObsoleteStorageLocation.class, pref + ObsoleteStorageLocation.class.getSimpleName()); this.tableMap.put(HarvestState.class, pref + HarvestState.class.getSimpleName()); - this.tableMap.put(KeyPair.class, pref + KeyPair.class.getSimpleName()); + this.tableMap.put(PreauthKeyPair.class, pref + PreauthKeyPair.class.getSimpleName()); String[] cols = new String[] { "uri", // first column is logical key @@ -227,7 +227,7 @@ protected void init() { "metaChecksum", "id" // last column is always PK }; - this.columnMap.put(KeyPair.class, cols); + this.columnMap.put(PreauthKeyPair.class, cols); // optional vospace log.debug("vosSchema: " + vosSchema); @@ -315,7 +315,7 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) { if (StorageSite.class.equals(c)) { return new StorageSiteGet(forUpdate); } - if (KeyPair.class.equals(c)) { + if (PreauthKeyPair.class.equals(c)) { return new KeyPairGet(forUpdate); } if (Node.class.equals(c)) { @@ -389,7 +389,7 @@ public EntityList getEntityList(Class c) { if (StorageSite.class.equals(c)) { return new StorageSiteList(); } - if (KeyPair.class.equals(c)) { + if (PreauthKeyPair.class.equals(c)) { return new KeyPairList(); } throw new UnsupportedOperationException("entity-list: " + c.getName()); @@ -422,7 +422,7 @@ public EntityPut getEntityPut(Class c, boolean update) { if (HarvestState.class.equals(c)) { return new HarvestStatePut(update); } - if (KeyPair.class.equals(c)) { + if (PreauthKeyPair.class.equals(c)) { return new KeyPairPut(update); } if (Node.class.isAssignableFrom(c)) { @@ -852,7 +852,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } - class KeyPairGet implements EntityGet { + class KeyPairGet implements EntityGet { private UUID id; private String name; private final boolean forUpdate; @@ -871,16 +871,16 @@ public void setName(String name) { } @Override - public KeyPair execute(JdbcTemplate jdbc) { - return (KeyPair) jdbc.query(this, new KeyPairExtractor()); + public PreauthKeyPair execute(JdbcTemplate jdbc) { + return (PreauthKeyPair) jdbc.query(this, new KeyPairExtractor()); } @Override public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { - StringBuilder sb = getSelectFromSQL(KeyPair.class, false); + StringBuilder sb = getSelectFromSQL(PreauthKeyPair.class, false); sb.append(" WHERE "); if (id != null) { - String col = getKeyColumn(KeyPair.class, true); + String col = getKeyColumn(PreauthKeyPair.class, true); sb.append(col).append(" = ?"); } else if (name != null) { sb.append("name = ?"); @@ -902,19 +902,19 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } } - private class KeyPairList implements EntityList { + private class KeyPairList implements EntityList { @Override - public Set query(JdbcTemplate jdbc) { - List keys = (List) jdbc.query(this, new KeyPairRowMapper()); - Set ret = new TreeSet<>(); + public Set query(JdbcTemplate jdbc) { + List keys = (List) jdbc.query(this, new KeyPairRowMapper()); + Set ret = new TreeSet<>(); ret.addAll(keys); return ret; } @Override public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { - StringBuilder sb = getSelectFromSQL(KeyPair.class, false); + StringBuilder sb = getSelectFromSQL(PreauthKeyPair.class, false); String sql = sb.toString(); log.debug("KeyPairList: " + sql); PreparedStatement prep = conn.prepareStatement(sql); @@ -1357,17 +1357,17 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } - private class KeyPairPut implements EntityPut { + private class KeyPairPut implements EntityPut { private final Calendar utc = Calendar.getInstance(DateUtil.UTC); private final boolean update; - private KeyPair value; + private PreauthKeyPair value; KeyPairPut(boolean update) { this.update = update; } @Override - public void setValue(KeyPair value) { + public void setValue(PreauthKeyPair value) { this.value = value; } @@ -1380,10 +1380,10 @@ public void execute(JdbcTemplate jdbc) { public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { String sql = null; if (update) { - sql = getUpdateSQL(KeyPair.class); + sql = getUpdateSQL(PreauthKeyPair.class); } else { - sql = getInsertSQL(KeyPair.class); + sql = getInsertSQL(PreauthKeyPair.class); } log.debug("KeyPairPut: " + sql); PreparedStatement prep = conn.prepareStatement(sql); @@ -2013,11 +2013,11 @@ public StorageSite extractData(ResultSet rs) throws SQLException, DataAccessExce } } - private class KeyPairRowMapper implements RowMapper { + private class KeyPairRowMapper implements RowMapper { Calendar utc = Calendar.getInstance(DateUtil.UTC); @Override - public KeyPair mapRow(ResultSet rs, int i) throws SQLException { + public PreauthKeyPair mapRow(ResultSet rs, int i) throws SQLException { int col = 1; final String name = rs.getString(col++); final byte[] pub = rs.getBytes(col++); @@ -2027,18 +2027,18 @@ public KeyPair mapRow(ResultSet rs, int i) throws SQLException { final URI metaChecksum = Util.getURI(rs, col++); final UUID id = Util.getUUID(rs, col++); - KeyPair s = new KeyPair(id, name, pub, priv); + PreauthKeyPair s = new PreauthKeyPair(id, name, pub, priv); InventoryUtil.assignLastModified(s, lastModified); InventoryUtil.assignMetaChecksum(s, metaChecksum); return s; } } - private class KeyPairExtractor implements ResultSetExtractor { + private class KeyPairExtractor implements ResultSetExtractor { final Calendar utc = Calendar.getInstance(DateUtil.UTC); @Override - public KeyPair extractData(ResultSet rs) throws SQLException, DataAccessException { + public PreauthKeyPair extractData(ResultSet rs) throws SQLException, DataAccessException { if (!rs.next()) { return null; } diff --git a/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql b/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql index 1902222a9..50fc0dcc2 100644 --- a/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql +++ b/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql @@ -1,5 +1,5 @@ -create table .KeyPair ( +create table .PreauthKeyPair ( name varchar(32) not null, publicKey bytea not null, privateKey bytea not null, @@ -9,4 +9,4 @@ create table .KeyPair ( metaChecksum varchar(136) not null ); -create unique index kp_name_index on .KeyPair(name); +create unique index kp_name_index on .PreauthKeyPair(name); From ff3281f1f579ca004ba1585f066cf09c70579346 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Dec 2023 10:19:04 -0800 Subject: [PATCH 074/186] rename some sql files --- .../opencadc/inventory/db/version/InitDatabase.java | 10 +++++----- .../java/org/opencadc/vospace/db/InitDatabaseVOS.java | 8 ++++---- ...nventory.KeyPair.sql => generic.PreauthKeyPair.sql} | 0 ...ventory.permissions.sql => generic.permissions.sql} | 0 4 files changed, 9 insertions(+), 9 deletions(-) rename cadc-inventory-db/src/main/resources/{inventory.KeyPair.sql => generic.PreauthKeyPair.sql} (100%) rename cadc-inventory-db/src/main/resources/{inventory.permissions.sql => generic.permissions.sql} (100%) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java index 0c632b4fa..46ba83171 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2020. (c) 2020. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -92,13 +92,13 @@ public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase { "inventory.DeletedStorageLocationEvent.sql", "inventory.StorageLocationEvent.sql", "inventory.HarvestState.sql", - "inventory.KeyPair.sql", - "inventory.permissions.sql" + "generic.PreauthKeyPair.sql", + "generic.permissions.sql" }; static String[] UPGRADE_SQL = new String[] { - "inventory.KeyPair.sql", - "inventory.permissions.sql" + "generic.PreauthKeyPair.sql", + "generic.permissions.sql" }; public InitDatabase(DataSource ds, String database, String schema) { diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java index 16ff9ad86..ab32ba1cb 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -88,13 +88,13 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { "vos.ModelVersion.sql", "vos.Node.sql", "vos.DeletedNodeEvent.sql", - "inventory.KeyPair.sql", - "inventory.permissions.sql" + "generic.PreauthKeyPair.sql", + "generic.permissions.sql" }; static String[] UPGRADE_SQL = new String[] { - "inventory.KeyPair.sql", - "inventory.permissions.sql" + "generic.PreauthKeyPair.sql", + "generic.permissions.sql" }; public InitDatabaseVOS(DataSource ds, String database, String schema) { diff --git a/cadc-inventory-db/src/main/resources/inventory.KeyPair.sql b/cadc-inventory-db/src/main/resources/generic.PreauthKeyPair.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/inventory.KeyPair.sql rename to cadc-inventory-db/src/main/resources/generic.PreauthKeyPair.sql diff --git a/cadc-inventory-db/src/main/resources/inventory.permissions.sql b/cadc-inventory-db/src/main/resources/generic.permissions.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/inventory.permissions.sql rename to cadc-inventory-db/src/main/resources/generic.permissions.sql From 595fda88d55ed17f84587d69d636ad486048ddd5 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Dec 2023 10:45:19 -0800 Subject: [PATCH 075/186] checkstyle fix --- .../src/main/java/org/opencadc/inventory/db/SQLGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 4a3471075..58f913c13 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -94,8 +94,8 @@ import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; import org.opencadc.inventory.InventoryUtil; -import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.ObsoleteStorageLocation; +import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StorageLocationEvent; From c9a37ccd83d2ebe0ce41511176dfda7482d55a1b Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Dec 2023 13:48:06 -0800 Subject: [PATCH 076/186] ProtocolsGenerator ctor change and public members to denote optional features --- .../transfer/ProtocolsGenerator.java | 52 ++++++++++++++----- .../StorageSiteAvailabilityCheck.java | 2 +- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index d6bc891ae..88afa0913 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -119,24 +119,48 @@ public class ProtocolsGenerator { private static final Logger log = Logger.getLogger(ProtocolsGenerator.class); + public static final URI SECURITY_EMBEDDED_TOKEN = URI.create("https://www.opencadc.org/std/storage#embedded-token"); + public static final String ARTIFACT_ID_HDR = "x-artifact-id"; // matches minoc.HeadAction.ARTIFACT_ID_HDR private final ArtifactDAO artifactDAO; private final DeletedArtifactEventDAO deletedArtifactEventDAO; - private final String user; - private final TokenTool tokenGen; + private final Map siteAvailabilities; private final Map siteRules; - private final StorageResolver storageResolver; - private final boolean preventNotFound; - // for use by FilesAction subclasses + /** + * Optional StorageResolver to resolve Artifact.uri to an external data provider. + */ + public StorageResolver storageResolver; + + /** + * Optional flag to enable prevention of 404 NotFound failure due to eventual + * consistency. Setting this to true will cause the code to make HTTP HEAD + * requests to all known storage sites looking for an artifact that is not + * in the local database. + */ + public boolean preventNotFound = false; + + /** + * Optional user value to put into generated preauth token. + */ + public String user; + + /** + * Optional TokenTool to generate and inject preauth tokens into otherwise anon URL. + */ + public TokenTool tokenGen; + + /** + * Optional restriction so that all anon URLs must have a preauth token. + */ + public boolean requirePreauthAnon = false; + + // for use by FilesAction subclasses to enhance logging boolean storageResolverAdded = false; - - public ProtocolsGenerator(ArtifactDAO artifactDAO, TokenTool tokenGen, String user, - Map siteAvailabilities, Map siteRules, - boolean preventNotFound, StorageResolver storageResolver) { + public ProtocolsGenerator(ArtifactDAO artifactDAO, Map siteAvailabilities, Map siteRules) { this.artifactDAO = artifactDAO; this.deletedArtifactEventDAO = new DeletedArtifactEventDAO(this.artifactDAO); this.user = user; @@ -146,7 +170,7 @@ public ProtocolsGenerator(ArtifactDAO artifactDAO, TokenTool tokenGen, String us this.preventNotFound = preventNotFound; this.storageResolver = storageResolver; } - + public boolean getStorageResolverAdded() { return storageResolverAdded; } @@ -438,7 +462,7 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT log.warn("storage site is offline: " + storageSite.getResourceID()); continue; } - + //log.warn("PUT: " + storageSite); Capability filesCap = null; try { @@ -456,7 +480,7 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT if (storageSite.getAllowWrite()) { // less generic request for service that implements // HACK: this is filesCap specific in here - if (proto.getUri().equals(filesCap.getStandardID().toASCIIString())) { + if (proto.getUri().equals(filesCap.getStandardID())) { Protocol p = new Protocol(proto.getUri()); p.setEndpoint(storageSite.getResourceID().toASCIIString()); protos.add(p); @@ -465,15 +489,17 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT if (sec == null) { sec = Standards.SECURITY_METHOD_ANON; } + boolean incToken = Standards.SECURITY_METHOD_ANON.equals(sec); Interface iface = filesCap.findInterface(sec); log.debug("PUT: " + storageSite + " proto: " + proto + " iface: " + iface); if (iface != null) { URL baseURL = iface.getAccessURL().getURL(); //log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); if (protocolCompat(proto, baseURL)) { + StringBuilder sb = new StringBuilder(); sb.append(baseURL.toExternalForm()).append("/"); - if (proto.getSecurityMethod() == null || Standards.SECURITY_METHOD_ANON.equals(proto.getSecurityMethod())) { + if (authToken != null && incToken) { sb.append(authToken).append("/"); } sb.append(artifactURI.toASCIIString()); diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java index fd63250db..656d2ec00 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/StorageSiteAvailabilityCheck.java @@ -101,7 +101,7 @@ public class StorageSiteAvailabilityCheck implements Runnable { public StorageSiteAvailabilityCheck(StorageSiteDAO storageSiteDAO, String siteAvailabilitiesKey) { this.storageSiteDAO = storageSiteDAO; this.siteStates = new HashMap<>(); - this.siteAvailabilities = new HashMap(); + this.siteAvailabilities = new HashMap<>(); try { Context initialContext = new InitialContext(); From 5bb8441bb1a29f6a9bfb8718d217bd2f91906e36 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Dec 2023 14:33:13 -0800 Subject: [PATCH 077/186] vault: lib changes, keygen, transfer progress --- vault/build.gradle | 2 + .../opencadc/vault/NodePersistenceImpl.java | 42 +++++-- .../org/opencadc/vault/VaultInitAction.java | 111 ++++++++++++++++-- .../vault/VaultTransferGenerator.java | 30 ++--- vault/src/main/webapp/WEB-INF/web.xml | 110 ++++++++++++++++- vault/src/main/webapp/capabilities.xml | 56 +-------- 6 files changed, 258 insertions(+), 93 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index 6a637f96c..ce2424e23 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -44,6 +44,8 @@ dependencies { compile 'org.opencadc:cadc-registry:[1.7.4,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory-server:[0.3,1.0)' + compile 'org.opencadc:cadc-permissions:[0.3.5,1.0)' testCompile 'junit:junit:[4.0,)' diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 1fb0905d7..dcfe9805b 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -70,8 +70,6 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.HttpPrincipal; import ca.nrc.cadc.auth.IdentityManager; -import ca.nrc.cadc.auth.PrincipalExtractor; -import ca.nrc.cadc.auth.X509CertificateChain; import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; @@ -82,11 +80,9 @@ import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; import java.net.URI; -import java.security.Principal; import java.text.DateFormat; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; @@ -95,22 +91,26 @@ import java.util.TreeSet; import java.util.UUID; import javax.security.auth.Subject; -import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; import org.opencadc.gms.GroupURI; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedArtifactEventDAO; +import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.permissions.TokenTool; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotSupportedException; import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; +import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.db.NodeDAO; +import org.opencadc.vospace.server.LocalServiceURI; import org.opencadc.vospace.server.NodePersistence; import org.opencadc.vospace.server.Views; import org.opencadc.vospace.server.transfers.TransferGenerator; @@ -225,7 +225,11 @@ public Views getViews() { @Override public TransferGenerator getTransferGenerator() { - throw new UnsupportedOperationException(); + PreauthKeyPairDAO keyDAO = new PreauthKeyPairDAO(); + keyDAO.setConfig(nodeDaoConfig); + PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); + TokenTool tt = new TokenTool(kp.getPublicKey(), kp.getPrivateKey()); + return new VaultTransferGenerator(this, getArtifactDAO(), tt); } private NodeDAO getDAO() { @@ -526,7 +530,31 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException @Override public void move(Node node, ContainerNode dest, String newName) { - throw new UnsupportedOperationException(); + if (node == null || dest == null) { + throw new IllegalArgumentException("args cannot be null"); + } + if (node.parent == null || dest.parent == null) { + throw new IllegalArgumentException("args must both be peristent nodes before move"); + } + + Subject caller = AuthenticationUtil.getCurrentSubject(); + node.owner = caller; + node.ownerID = null; + node.ownerDisplay = null; + node.parent = dest; + node.parentID = null; + if (newName != null) { + node.setName(newName); + } + try { + Node result = put(node); + log.debug("moved: " + result); + } catch (NodeNotSupportedException ex) { + LocalServiceURI loc = new LocalServiceURI(getResourceID()); + VOSURI srcURI = loc.getURI(node); + throw new RuntimeException("BUG: failed to move node because of detected type change: " + + srcURI.getPath() + " type=" + node.getClass().getSimpleName(), ex); + } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index b33bb9d87..95b803237 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -71,9 +71,11 @@ import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.util.RsaSignatureGenerator; import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; import java.net.URI; import java.net.URISyntaxException; +import java.security.KeyPair; import java.util.Map; import java.util.TreeMap; import javax.naming.Context; @@ -82,8 +84,14 @@ import javax.sql.DataSource; import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.PreauthKeyPair; +import org.opencadc.inventory.db.PreauthKeyPairDAO; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.db.StorageSiteDAO; +import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; +import org.springframework.dao.DataIntegrityViolationException; /** * @@ -93,6 +101,8 @@ public class VaultInitAction extends InitAction { private static final Logger log = Logger.getLogger(VaultInitAction.class); + static String KEY_PAIR_NAME = "vault-preauth-keys"; + static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml @@ -112,6 +122,9 @@ public class VaultInitAction extends InitAction { private Map daoConfig; private String jndiNodePersistence; + private String jndiPreauthKeys; + private String jndiSiteAvailabilities; + private Thread availabilityCheck; public VaultInitAction() { super(); @@ -123,8 +136,29 @@ public void doInit() { initDatabase(); initUWSDatabase(); initNodePersistence(); + initKeyPair(); + initAvailabilityCheck(); } + @Override + public void doShutdown() { + try { + Context ctx = new InitialContext(); + ctx.unbind(jndiNodePersistence); + } catch (Exception oops) { + log.error("unbind failed during destroy", oops); + } + + try { + Context ctx = new InitialContext(); + ctx.unbind(jndiPreauthKeys); + } catch (Exception oops) { + log.error("unbind failed during destroy", oops); + } + + terminateAvailabilityCheck(); + } + /** * Read config file and verify that all required entries are present. * @@ -183,7 +217,9 @@ static MultiValuedProperties getConfig() { } static Map getDaoConfig(MultiValuedProperties props) { + Map ret = new TreeMap<>(); + ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); @@ -232,7 +268,7 @@ private void initUWSDatabase() { } } - protected void initNodePersistence() { + private void initNodePersistence() { jndiNodePersistence = appName + "-" + NodePersistence.class.getName(); try { Context ctx = new InitialContext(); @@ -249,15 +285,76 @@ protected void initNodePersistence() { log.error("Failed to create JNDI Key " + jndiNodePersistence, ex); } } - - @Override - public void doShutdown() { + + private void initKeyPair() { + log.info("initKeyPair: START"); + jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); try { + PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); + dao.setConfig(daoConfig); + PreauthKeyPair keys = dao.get(KEY_PAIR_NAME); + if (keys == null) { + KeyPair kp = RsaSignatureGenerator.getKeyPair(4096); + keys = new PreauthKeyPair(KEY_PAIR_NAME, kp.getPublic().getEncoded(), kp.getPrivate().getEncoded()); + try { + dao.put(keys); + log.info("initKeyPair: new keys created - OK"); + + } catch (DataIntegrityViolationException oops) { + log.warn("persist new " + PreauthKeyPair.class.getSimpleName() + " failed (" + oops + ") -- probably race condition"); + keys = dao.get(KEY_PAIR_NAME); + if (keys != null) { + log.info("race condition confirmed: another instance created keys - OK"); + } else { + throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", oops); + } + } + } else { + log.info("initKeyPair: re-use existing keys - OK"); + } Context ctx = new InitialContext(); - ctx.unbind(jndiNodePersistence); - } catch (Exception oops) { - log.error("unbind failed during destroy", oops); + try { + ctx.unbind(jndiPreauthKeys); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); + } + ctx.bind(jndiPreauthKeys, keys); + log.info("initKeyPair: created JNDI key: " + jndiPreauthKeys); + } catch (Exception ex) { + throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); } } + + void initAvailabilityCheck() { + StorageSiteDAO storageSiteDAO = new StorageSiteDAO(); + storageSiteDAO.setConfig(getDaoConfig(props)); + this.jndiSiteAvailabilities = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); + terminateAvailabilityCheck(); + this.availabilityCheck = new Thread(new StorageSiteAvailabilityCheck(storageSiteDAO, this.jndiSiteAvailabilities)); + this.availabilityCheck.setDaemon(true); + this.availabilityCheck.start(); + } + + private final void terminateAvailabilityCheck() { + if (this.availabilityCheck != null) { + try { + log.info("terminating AvailabilityCheck Thread..."); + this.availabilityCheck.interrupt(); + this.availabilityCheck.join(); + log.info("terminating AvailabilityCheck Thread... [OK]"); + } catch (Throwable t) { + log.info("failed to terminate AvailabilityCheck thread", t); + } finally { + this.availabilityCheck = null; + } + } + try { + InitialContext initialContext = new InitialContext(); + initialContext.unbind(this.jndiSiteAvailabilities); + } catch (NamingException e) { + log.debug(String.format("unable to unbind %s - %s", this.jndiSiteAvailabilities, e.getMessage())); + } + } + } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index abba154c8..f0c8184a1 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -86,16 +86,11 @@ import javax.naming.NamingException; import javax.security.auth.Subject; import org.apache.log4j.Logger; -import org.opencadc.inventory.KeyPair; import org.opencadc.inventory.db.ArtifactDAO; -import org.opencadc.inventory.db.KeyPairDAO; import org.opencadc.inventory.transfer.ProtocolsGenerator; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; -import org.opencadc.permissions.Grant; -import org.opencadc.permissions.ReadGrant; import org.opencadc.permissions.TokenTool; -import org.opencadc.permissions.WriteGrant; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.LinkingException; @@ -116,21 +111,19 @@ public class VaultTransferGenerator implements TransferGenerator { private static final Logger log = Logger.getLogger(VaultTransferGenerator.class); - static String KEY_PAIR_NAME = "vault-preauth-keys"; - private final NodePersistenceImpl nodePersistence; private final VOSpaceAuthorizer authorizer; private final ArtifactDAO artifactDAO; - private final KeyPairDAO keyDAO; + private final TokenTool tokenTool; private Map siteRules = new HashMap<>(); private Map siteAvailabilities; - public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, KeyPairDAO keyDAO) { + public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool) { this.nodePersistence = nodePersistence; this.authorizer = new VOSpaceAuthorizer(nodePersistence); this.artifactDAO = artifactDAO; - this.keyDAO = keyDAO; + this.tokenTool = tokenTool; // TODO: get appname from ??? String siteAvailabilitiesKey = "vault" + "-" + StorageSiteAvailabilityCheck.class.getName(); @@ -190,29 +183,22 @@ private List handleDataNode(DataNode node, Transfer trans, Subject s) throws IOException, ResourceNotFoundException { log.debug("handleDataNode: " + node); - Direction dir = trans.getDirection(); - final Map params = new TreeMap<>(); // empty for now - - // vault only supports preauth URLs using persistent key pair - //KeyPair keys = keyDAO.get(KEY_PAIR_NAME); - //TokenTool tokenGen = new TokenTool(keys.getPublicKey(), keys.getPrivateKey()); - TokenTool tokenGen = null; - IdentityManager im = AuthenticationUtil.getIdentityManager(); Subject caller = AuthenticationUtil.getCurrentSubject(); Object userObject = im.toOwner(caller); String callingUser = (userObject == null ? null : userObject.toString()); - ProtocolsGenerator pg = new ProtocolsGenerator(artifactDAO, tokenGen, callingUser, siteAvailabilities, siteRules); + ProtocolsGenerator pg = new ProtocolsGenerator(artifactDAO, siteAvailabilities, siteRules); + pg.tokenGen = tokenTool; + pg.user = callingUser; + pg.requirePreauthAnon = true; //pg.preventNotFound = true; - Transfer artifactTrans = new Transfer(node.storageID, dir); + Transfer artifactTrans = new Transfer(node.storageID, trans.getDirection()); for (Protocol p : trans.getProtocols()) { log.debug("requested protocol: " + p); URI secM = p.getSecurityMethod(); - // TODO: request preauth URL specifically, not anon if (secM == null || secM.equals(Standards.SECURITY_METHOD_ANON)) { - p.setSecurityMethod(ProtocolsGenerator.SECURITY_EMBEDDED_TOKEN); artifactTrans.getProtocols().add(p); } } diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 5d1831357..2d4793089 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -18,8 +18,9 @@ logLevelPackages org.opencadc.vault - ca.nrc.cadc.db + org.opencadc.inventory org.opencadc.vospace + ca.nrc.cadc.db ca.nrc.cadc.rest ca.nrc.cadc.util ca.nrc.cadc.vosi @@ -33,6 +34,19 @@ 1 + + PubKeyServlet + ca.nrc.cadc.rest.RestServlet + + augmentSubject + false + + + get + org.opencadc.vault.GetKeyAction + + + NodesServlet ca.nrc.cadc.rest.RestServlet @@ -63,6 +77,64 @@ 2 + + + SyncTransferServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.SyncGetAction + + + post + ca.nrc.cadc.uws.web.SyncPostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.web.SyncPostAction.execOnPOST + true + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.uws.SyncTransferManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.vospace.server.transfers.InlineTransferHandler + + 3 + + + + + AsyncTransferServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.uws.AsyncTransferManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.vospace.server.transfers.InlineTransferHandler + + 3 + + RecursiveDeleteNodeServlet @@ -149,11 +221,28 @@ availabilityProperties vault-availability.properties - 4 + 3 + + + + TransferDetailsServlet + org.opencadc.vospace.server.transfers.TransferDetailsServlet + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.vault.uws.SyncTransferManager + + 3 + + + + PubKeyServlet + /pubkey + + NodesServlet /nodes/* @@ -168,7 +257,22 @@ RecursiveNodePropsServlet /async-setprops/* - + + + AsyncTransferServlet + /transfers/* + + + + SyncTransferServlet + /synctrans/* + + + + TransferDetailsServlet + /xfer/* + + AvailabilityServlet diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index 28cfb09bc..bd833eb73 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -61,7 +61,6 @@ - - - - - + - - - - - - diff --git a/raven/src/test/java/org/opencadc/raven/RavenInitActionTest.java b/raven/src/test/java/org/opencadc/raven/RavenInitActionTest.java index 5a12e69d1..7305eb8be 100644 --- a/raven/src/test/java/org/opencadc/raven/RavenInitActionTest.java +++ b/raven/src/test/java/org/opencadc/raven/RavenInitActionTest.java @@ -124,8 +124,6 @@ public void doInit() { String message = e.getMessage(); log.debug(message); Assert.assertTrue(message.contains(String.format("%s: MISSING", RavenInitAction.SCHEMA_KEY))); - Assert.assertTrue(message.contains(String.format("%s: MISSING", RavenInitAction.PUBKEYFILE_KEY))); - Assert.assertTrue(message.contains(String.format("%s: MISSING", RavenInitAction.PRIVKEYFILE_KEY))); } finally { System.setProperty("user.home", USER_HOME); } From 71391430efd8c2f34d21c7d5673083e34b9c6785 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 14 Dec 2023 11:12:21 -0800 Subject: [PATCH 080/186] Small update --- raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java index ff4584936..2e8d279ef 100644 --- a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java @@ -844,8 +844,7 @@ public void testPreauthURL() throws Exception { Protocol files = new Protocol(Standards.SI_FILES); requested.add(files); - URI resourceID = URI.create("ivo://opencadc.org/minoc"); - StorageSite site = new StorageSite(resourceID, "site1", true, true); + StorageSite site = new StorageSite(CONSIST_RESOURCE_ID, "site1", true, true); try { // get raven pub key URL pubKeyURL = anonURL.toURI().resolve("./pubkey").toURL(); From 2775f23311afa089f6148b0bb2ab3cc427107275 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 14 Dec 2023 13:01:05 -0800 Subject: [PATCH 081/186] tweak ProtocolsGenerator to work in a storage site db (storageLocation not null) --- .../transfer/ProtocolsGenerator.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index 88afa0913..c34267f54 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -330,8 +330,7 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) List protos = new ArrayList<>(); Artifact artifact = artifactDAO.get(artifactURI); - // produce URLs to each of the copies for each of the protocols - List storageSites = new ArrayList<>(); + if (artifact == null) { if (this.preventNotFound) { log.debug("Artifact " + artifactURI.toASCIIString() + " not found in global. Check sites."); @@ -339,10 +338,23 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) } } + List storageSites = new ArrayList<>(); if (artifact != null) { - for (SiteLocation site : artifact.siteLocations) { - StorageSite storageSite = getSite(sites, site.getSiteID()); - storageSites.add(storageSite); + if (artifact.storageLocation != null) { + // this is a single storage site + Iterator iter = sites.iterator(); + if (iter.hasNext()) { + storageSites.add(iter.next()); + } + if (iter.hasNext()) { + log.error("BUG: found second StorageSite in database with assigned Artifact.storageLocation"); + } + } else { + // this is a global inventory + for (SiteLocation site : artifact.siteLocations) { + StorageSite storageSite = getSite(sites, site.getSiteID()); + storageSites.add(storageSite); + } } } @@ -426,10 +438,8 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) } } - if (protos.isEmpty() && ((artifact == null) || artifact.siteLocations.size() == 0)) { - // artifact not find internally and has no external resolvers either - // TODO: second condition can currently happen but maybe should not: - // --- when the last siteLocation is removed, the artifact should be deleted (fenwick, ratik) + if (protos.isEmpty()) { + // unable to generate any URLs throw new ResourceNotFoundException("not found: " + artifactURI.toString()); } From 8328264693c3fe69f42533f24e8cd73a2fe6b678 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 14 Dec 2023 13:51:34 -0800 Subject: [PATCH 082/186] wteak vault transfer code for unstored data note --- vault/README.md | 37 ++++++++----------- .../vault/VaultTransferGenerator.java | 16 +++++--- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/vault/README.md b/vault/README.md index c4615fc99..0ffeb6b3a 100644 --- a/vault/README.md +++ b/vault/README.md @@ -67,19 +67,20 @@ org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. -The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due to configuration -limitations in luskan. +The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The +account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due +to configuration limitations in luskan. -The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must be in the same database and some operations may join tables -in the two schemas (probably just vospace.node join inventory.artifact). +The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must currently be in the same database. +TODO: augment config to support separate inventory and vospace pools. -The root node owner has full read and write permission in the root container, so it can create and delete container -nodes at the root and assign container node properties that are normally read-only to normal users: owner, quota, -etc. This is probably an X509 distingushed name of the user (to start). **not fully implemented** TBD. +The _root.owner_ owns the root node and has full read and write permission in the root container, so it can +create and delete container nodes at the root and assign container node properties that are normally read-only +to normal users: owner, quota, etc. This must be set to the username of the admin. -The _namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. This only -applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't -want to change this... prevent change? TBD. +The _storage.namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. +This only applies to new data nodes that are created and will not effect previously created nodes and artifacts. +Probably don't want to change this... prevent change? TBD. ### vault-availability.properties (optional) @@ -91,9 +92,10 @@ Example: ``` users = {user identity} ``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities -(X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is not found or does not -list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. +`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user +identities (X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is +not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the +state will not be changeable. ## building it ``` @@ -111,12 +113,3 @@ docker run --rm -it vault:latest /bin/bash docker run --rm --user tomcat:tomcat --volume=/path/to/external/config:/config:ro --name vault vault:latest ``` -## apply semantic version tags -```bash -. VERSION && echo "tags: $TAGS" -for t in $TAGS; do - docker image tag vault:latest vault:$t -done -unset TAGS -docker image list vault -``` diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index f0c8184a1..d812343a2 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -77,10 +77,10 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingException; @@ -203,11 +203,15 @@ private List handleDataNode(DataNode node, Transfer trans, Subject s) } } - List ret = pg.getProtocols(artifactTrans); - log.warn("generated urls: " + ret.size()); - for (Protocol p : ret) { - log.warn(p.getEndpoint() + " using " + p.getSecurityMethod()); + try { + List ret = pg.getProtocols(artifactTrans); + log.warn("generated urls: " + ret.size()); + for (Protocol p : ret) { + log.warn(p.getEndpoint() + " using " + p.getSecurityMethod()); + } + return ret; + } catch (ResourceNotFoundException ex) { + return new ArrayList(); } - return ret; } } From 53bb95503b1f06d94a367965f1f9cd39ec630748 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 14 Dec 2023 13:51:56 -0800 Subject: [PATCH 083/186] minoc: re-org config and add initial trusted service key handling, still rough --- minoc/README.md | 17 +- .../org/opencadc/minoc/ArtifactAction.java | 209 +++++----------- .../java/org/opencadc/minoc/DeleteAction.java | 1 + .../java/org/opencadc/minoc/GetAction.java | 1 + .../java/org/opencadc/minoc/HeadAction.java | 1 + .../org/opencadc/minoc/MinocInitAction.java | 223 ++++-------------- .../java/org/opencadc/minoc/PostAction.java | 1 + .../java/org/opencadc/minoc/PutAction.java | 1 + .../opencadc/minoc/ServiceAvailability.java | 13 +- 9 files changed, 127 insertions(+), 340 deletions(-) diff --git a/minoc/README.md b/minoc/README.md index d4c09685d..c6db67305 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -74,16 +74,27 @@ currently must be "inventory" due to configuration limitations in raven. +The optional _trust.preauth_ key(s) configure `minoc` to trust an external service to have performed +authorization checks. Such services may include a signed token in the URL and `minoc` will validate +the request using a public key retrieved from the service instead of performing authorization checks +itself. Example: +``` +# trust a SI global inventory +org.opencadc.minoc.trust.preauth = ivo://example.net/raven + +# trust a SI VOSpace service +org.opencadc.minoc.trust.preauth = ivo://example.net/vault +``` The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for operations. Multiple values of the granting service resourceID(s) may be provided by including multiple property diff --git a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java index 591f0bf1c..e7e764e73 100644 --- a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java @@ -77,26 +77,21 @@ import ca.nrc.cadc.rest.SyncInput; import ca.nrc.cadc.rest.Version; import ca.nrc.cadc.util.MultiValuedProperties; -import ca.nrc.cadc.util.PropertiesReader; -import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.security.AccessControlException; import java.security.cert.CertificateException; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.TreeMap; import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; -import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.storage.StorageAdapter; import org.opencadc.permissions.Grant; import org.opencadc.permissions.ReadGrant; @@ -133,76 +128,28 @@ public abstract class ArtifactAction extends RestAction { // servlet path minus the auth token String loggablePath; - // immutable state set in constructor - protected final MultiValuedProperties config; - protected final File publicKey; - protected final List readGrantServices = new ArrayList<>(); - protected final List writeGrantServices = new ArrayList<>(); + protected MinocConfig config; // lazy init protected ArtifactDAO artifactDAO; protected StorageAdapter storageAdapter; - private final boolean authenticateOnly; - // constructor for unit tests with no config/init ArtifactAction(boolean init) { super(); this.config = null; this.artifactDAO = null; this.storageAdapter = null; - this.authenticateOnly = false; - this.publicKey = null; } protected ArtifactAction() { super(); - this.config = MinocInitAction.getConfig(); - - List readGrants = config.getProperty(MinocInitAction.READ_GRANTS_KEY); - if (readGrants != null) { - for (String s : readGrants) { - try { - URI u = new URI(s); - readGrantServices.add(u); - } catch (URISyntaxException ex) { - throw new IllegalStateException("invalid config: " + MinocInitAction.READ_GRANTS_KEY + "=" + s + " INVALID", ex); - } - } - } - - List writeGrants = config.getProperty(MinocInitAction.WRITE_GRANTS_KEY); - if (writeGrants != null) { - for (String s : writeGrants) { - try { - URI u = new URI(s); - writeGrantServices.add(u); - } catch (URISyntaxException ex) { - throw new IllegalStateException("invalid config: " + MinocInitAction.WRITE_GRANTS_KEY + "=" + s + " INVALID", ex); - } - } - } - - String ao = config.getFirstPropertyValue(MinocInitAction.DEV_AUTH_ONLY_KEY); - if (ao != null) { - try { - this.authenticateOnly = Boolean.valueOf(ao); - if (authenticateOnly) { - log.warn("(configuration) authenticateOnly = " + authenticateOnly); - } - } catch (Exception ex) { - throw new IllegalStateException("invalid config: " + MinocInitAction.DEV_AUTH_ONLY_KEY + "=" + ao + " must be true|false or not set"); - } - } else { - authenticateOnly = false; - } + } - String pubkeyFileName = config.getFirstPropertyValue(MinocInitAction.PUBKEYFILE_KEY); - if (pubkeyFileName != null) { - this.publicKey = new File(System.getProperty("user.home") + "/config/" + pubkeyFileName); - } else { - this.publicKey = null; // no pre-auth - } + @Override + public void initAction() throws Exception { + super.initAction(); + this.config = MinocInitAction.getConfig(appName); } @Override @@ -258,32 +205,47 @@ protected void initAndAuthorize(Class grantClass, boolean allow // do authorization (with token or subject) Subject subject = AuthenticationUtil.getCurrentSubject(); if (authToken != null) { - if (publicKey == null) { - throw new IllegalArgumentException("unexpected pre-auth token in URL"); - } - TokenTool tk = new TokenTool(publicKey); - String tokenUser; - if (allowReadWithWriteGrant && ReadGrant.class.isAssignableFrom(grantClass)) { - // treat a WriteGrant as also granting read permission - tokenUser = tk.validateToken(authToken, artifactURI, grantClass, WriteGrant.class); - } else { - tokenUser = tk.validateToken(authToken, artifactURI, grantClass); - } - subject.getPrincipals().clear(); - if (tokenUser != null) { - subject.getPrincipals().add(new HttpPrincipal(tokenUser)); - } - logInfo.setSubject(subject); - logInfo.setResource(artifactURI); - logInfo.setPath(syncInput.getContextPath() + syncInput.getComponentPath()); - if (ReadGrant.class.isAssignableFrom(grantClass)) { - logInfo.setGrant("read: preauth-token"); - } else if (WriteGrant.class.isAssignableFrom(grantClass)) { - logInfo.setGrant("write: preauth-token"); - } else { - throw new IllegalStateException("Unsupported grant class: " + grantClass); + //if (publicKey == null) { + // throw new IllegalArgumentException("unexpected pre-auth token in URL"); + //} + //TokenTool tk = new TokenTool(publicKey); + for (Map.Entry me : config.getTrustedServices().entrySet()) { + URI ts = me.getKey(); + if (me.getValue() != null) { + TokenTool tk = new TokenTool(me.getValue()); + log.warn("validate preauth with key from " + me.getKey()); + try { + String tokenUser; + if (allowReadWithWriteGrant && ReadGrant.class.isAssignableFrom(grantClass)) { + // treat a WriteGrant as also granting read permission + tokenUser = tk.validateToken(authToken, artifactURI, grantClass, WriteGrant.class); + } else { + tokenUser = tk.validateToken(authToken, artifactURI, grantClass); + } + subject.getPrincipals().clear(); + if (tokenUser != null) { + subject.getPrincipals().add(new HttpPrincipal(tokenUser)); + } + logInfo.setSubject(subject); + logInfo.setResource(artifactURI); + logInfo.setPath(syncInput.getContextPath() + syncInput.getComponentPath()); + if (ReadGrant.class.isAssignableFrom(grantClass)) { + logInfo.setGrant("read: preauth-token from " + ts); + } else if (WriteGrant.class.isAssignableFrom(grantClass)) { + logInfo.setGrant("write: preauth-token from " + ts); + } else { + throw new IllegalStateException("Unsupported grant class: " + grantClass); + } + return; + } catch (AccessControlException ex) { + log.warn("token invalid vs keys from " + ts, ex); + } + } else { + log.warn("no keys from " + ts + " -- SKIP"); + } + // no return from inside check + throw new AccessControlException("invalid auth token"); } - return; } // augment subject (minoc is configured so augment is not done in rest library) @@ -291,13 +253,13 @@ protected void initAndAuthorize(Class grantClass, boolean allow logInfo.setSubject(subject); logInfo.setResource(artifactURI); logInfo.setPath(syncInput.getContextPath() + syncInput.getComponentPath()); - PermissionsCheck permissionsCheck = new PermissionsCheck(artifactURI, authenticateOnly, logInfo); + PermissionsCheck permissionsCheck = new PermissionsCheck(artifactURI, config.isAuthenticateOnly(), logInfo); // TODO: allowReadWithWriteGrant could be implemented here, but grant services are probably configured // that way already so it's complexity that probably won't allow/enable any actions if (ReadGrant.class.isAssignableFrom(grantClass)) { - permissionsCheck.checkReadPermission(readGrantServices); + permissionsCheck.checkReadPermission(config.getReadGrantServices()); } else if (WriteGrant.class.isAssignableFrom(grantClass)) { - permissionsCheck.checkWritePermission(writeGrantServices); + permissionsCheck.checkWritePermission(config.getWriteGrantServices()); } else { throw new IllegalStateException("Unsupported grant class: " + grantClass); } @@ -311,7 +273,7 @@ void init() { protected void initDAO() { if (artifactDAO == null) { - Map configMap = MinocInitAction.getDaoConfig(config); + Map configMap = config.getDaoConfig(); this.artifactDAO = new ArtifactDAO(); artifactDAO.setConfig(configMap); // connectivity tested } @@ -319,12 +281,10 @@ protected void initDAO() { protected void initStorageAdapter() { if (storageAdapter == null) { - this.storageAdapter = InventoryUtil.loadPlugin(config.getFirstPropertyValue(MinocInitAction.SA_KEY)); - List rec = MinocInitAction.getRecoverableNamespaces(config); - storageAdapter.setRecoverableNamespaces(rec); + this.storageAdapter = config.getStorageAdapter(); } } - + /** * Parse the request path. */ @@ -372,67 +332,4 @@ private URI createArtifactURI(String uri) { } return ret; } - - protected List getReadGrantServices(MultiValuedProperties props) { - String key = ReadGrant.class.getName() + ".resourceID"; - List values = props.getProperty(key); - if (values == null) { - return Collections.emptyList(); - } - return values; - } - - protected List getWriteGrantServices(MultiValuedProperties props) { - String key = WriteGrant.class.getName() + ".resourceID"; - List values = props.getProperty(key); - if (values == null) { - return Collections.emptyList(); - } - return values; - } - - static MultiValuedProperties readConfig() { - PropertiesReader pr = new PropertiesReader("minoc.properties"); - MultiValuedProperties props = pr.getAllProperties(); - - if (log.isDebugEnabled()) { - log.debug("minoc.properties:"); - Set keys = props.keySet(); - for (String key : keys) { - log.debug(" " + key + " = " + props.getProperty(key)); - } - } - return props; - } - - static Map getDaoConfig(MultiValuedProperties props) { - Map config = new HashMap(); - Class cls = null; - List sqlGenList = props.getProperty(MinocInitAction.SQLGEN_KEY); - if (sqlGenList != null && sqlGenList.size() > 0) { - try { - String sqlGenClass = sqlGenList.get(0); - cls = Class.forName(sqlGenClass); - } catch (ClassNotFoundException e) { - throw new IllegalStateException( - "could not load SQLGenerator class: " + e.getMessage(), e); - } - } else { - // use the default SQL generator - cls = SQLGenerator.class; - } - - config.put(MinocInitAction.SQLGEN_KEY, cls); - config.put("jndiDataSourceName", MinocInitAction.JNDI_DATASOURCE); - List schemaList = props.getProperty(MinocInitAction.SCHEMA_KEY); - if (schemaList == null || schemaList.size() < 1) { - throw new IllegalStateException("a value for " + MinocInitAction.SCHEMA_KEY - + " is needed in minoc.properties"); - } - config.put("schema", schemaList.get(0)); - config.put("database", null); - - return config; - } - } diff --git a/minoc/src/main/java/org/opencadc/minoc/DeleteAction.java b/minoc/src/main/java/org/opencadc/minoc/DeleteAction.java index 2ce41e46e..28f18d810 100644 --- a/minoc/src/main/java/org/opencadc/minoc/DeleteAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/DeleteAction.java @@ -98,6 +98,7 @@ public DeleteAction() { */ @Override public void initAction() throws Exception { + super.initAction(); checkWritable(); initAndAuthorize(WriteGrant.class); initDAO(); diff --git a/minoc/src/main/java/org/opencadc/minoc/GetAction.java b/minoc/src/main/java/org/opencadc/minoc/GetAction.java index 9521591d9..a74e227de 100644 --- a/minoc/src/main/java/org/opencadc/minoc/GetAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/GetAction.java @@ -138,6 +138,7 @@ public GetAction() { */ @Override public void initAction() throws Exception { + super.initAction(); checkReadable(); initAndAuthorize(ReadGrant.class); initDAO(); diff --git a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java index f80f27564..5e16d17a4 100644 --- a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java @@ -100,6 +100,7 @@ public HeadAction() { */ @Override public void initAction() throws Exception { + super.initAction(); checkReadable(); initAndAuthorize(ReadGrant.class, true); // allowReadWithWriteGrant for head after put initDAO(); diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java index 37b70f3fd..03274739f 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -79,6 +79,9 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; import javax.sql.DataSource; import org.apache.log4j.Logger; import org.opencadc.inventory.InventoryUtil; @@ -96,31 +99,10 @@ public class MinocInitAction extends InitAction { private static final Logger log = Logger.getLogger(MinocInitAction.class); - static final String JNDI_DATASOURCE = "jdbc/inventory"; // context.xml - - // config keys - private static final String MINOC_KEY = "org.opencadc.minoc"; - static final String RESOURCE_ID_KEY = MINOC_KEY + ".resourceID"; - - static final String SQLGEN_KEY = SQLGenerator.class.getName(); - - static final String SCHEMA_KEY = MINOC_KEY + ".inventory.schema"; - - static final String SA_KEY = StorageAdapter.class.getName(); - - static final String PUBKEYFILE_KEY = MINOC_KEY + ".publicKeyFile"; - static final String READ_GRANTS_KEY = MINOC_KEY + ".readGrantProvider"; - static final String WRITE_GRANTS_KEY = MINOC_KEY + ".writeGrantProvider"; - - static final String RECOVERABLE_NS_KEY = MINOC_KEY + ".recoverableNamespace"; - - static final String DEV_AUTH_ONLY_KEY = MINOC_KEY + ".authenticateOnly"; - // set init initConfig, used by subsequent init methods - - MultiValuedProperties props; + private MinocConfig config; + private String jndiConfigKey; private URI resourceID; - private Map daoConfig; public MinocInitAction() { super(); @@ -133,175 +115,68 @@ public void doInit() { initStorageSite(); initStorageAdapter(); } - - /** - * Read config file and verify that all required entries are present. - * - * @return MultiValuedProperties containing the application config - * @throws IllegalStateException if required config items are missing - */ - static MultiValuedProperties getConfig() { - PropertiesReader r = new PropertiesReader("minoc.properties"); - MultiValuedProperties mvp = r.getAllProperties(); - - StringBuilder sb = new StringBuilder(); - sb.append("incomplete config: "); - boolean ok = true; - - String rid = mvp.getFirstPropertyValue(RESOURCE_ID_KEY); - sb.append("\n\t" + RESOURCE_ID_KEY + ": "); - if (rid == null) { - sb.append("MISSING"); - ok = false; - } else { - sb.append("OK"); - } - - String sac = mvp.getFirstPropertyValue(SA_KEY); - sb.append("\n\t").append(SA_KEY).append(": "); - if (sac == null) { - sb.append("MISSING"); - ok = false; - } else { - sb.append("OK"); - } - String sqlgen = mvp.getFirstPropertyValue(SQLGEN_KEY); - sb.append("\n\t").append(SQLGEN_KEY).append(": "); - if (sqlgen == null) { - sb.append("MISSING"); - ok = false; - } else { - try { - Class c = Class.forName(sqlgen); - sb.append("OK"); - } catch (ClassNotFoundException ex) { - sb.append("class not found: " + sqlgen); - ok = false; - } - } - - String schema = mvp.getFirstPropertyValue(SCHEMA_KEY); - sb.append("\n\t").append(SCHEMA_KEY).append(": "); - if (schema == null) { - sb.append("MISSING"); - ok = false; - } else { - sb.append("OK"); - } - - // optional - String pubkeyFileName = mvp.getFirstPropertyValue(PUBKEYFILE_KEY); - sb.append("\n\t").append(PUBKEYFILE_KEY).append(": "); - if (pubkeyFileName != null) { - File pk = new File(System.getProperty("user.home") + "/config/" + pubkeyFileName); - if (!pk.exists()) { - sb.append(" NOT FOUND ").append(pk.getAbsolutePath()); - ok = false; - } else { - sb.append("OK"); - } - } - - // optional - List readGrants = mvp.getProperty(READ_GRANTS_KEY); - if (readGrants != null) { - for (String s : readGrants) { - sb.append("\n\t").append(READ_GRANTS_KEY + "=").append(s); - try { - URI u = new URI(s); - sb.append(" OK"); - } catch (URISyntaxException ex) { - sb.append(" INVALID"); - ok = false; - } - } - } - - // optional - List writeGrants = mvp.getProperty(WRITE_GRANTS_KEY); - if (writeGrants != null) { - for (String s : writeGrants) { - sb.append("\n\t").append(WRITE_GRANTS_KEY + "=").append(s); - try { - URI u = new URI(s); - sb.append(" OK"); - } catch (URISyntaxException ex) { - sb.append(" INVALID"); - ok = false; - } - } - } - - // optional - List rawRecNS = mvp.getProperty(RECOVERABLE_NS_KEY); - if (rawRecNS != null) { - for (String s : rawRecNS) { - sb.append("\n\t").append(RECOVERABLE_NS_KEY + "=").append(s); - try { - Namespace ns = new Namespace(s); - } catch (Exception ex) { - sb.append(" INVALID"); - } - } - } - - if (!ok) { - throw new IllegalStateException(sb.toString()); + @Override + public void doShutdown() { + super.doShutdown(); + try { + Context ctx = new InitialContext(); + ctx.unbind(jndiConfigKey); + } catch (NamingException ex) { + log.debug("failed to remove config from JNDI", ex); } - - return mvp; } - static Map getDaoConfig(MultiValuedProperties props) { - String cname = props.getFirstPropertyValue(SQLGenerator.class.getName()); + // get config from JNDI + static MinocConfig getConfig(String appName) { + String key = appName + "-" + MinocConfig.class.getName(); try { - Map ret = new TreeMap<>(); - Class clz = Class.forName(cname); - ret.put(SQLGenerator.class.getName(), clz); - ret.put("jndiDataSourceName", MinocInitAction.JNDI_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(MinocInitAction.SCHEMA_KEY)); - //config.put("database", null); + Context ctx = new InitialContext(); + MinocConfig ret = (MinocConfig) ctx.lookup(key); return ret; - } catch (ClassNotFoundException ex) { - throw new IllegalStateException("invalid config: failed to load SQLGenerator: " + cname); - } - } - - static List getRecoverableNamespaces(MultiValuedProperties props) { - List ret = new ArrayList<>(); - List rawRecNS = props.getProperty(RECOVERABLE_NS_KEY); - if (rawRecNS != null) { - for (String s : rawRecNS) { - ret.add(new Namespace(s)); - } + } catch (NamingException ex) { + throw new RuntimeException("BUG: failed to get config from JNDI", ex); } - return ret; } private void initConfig() { log.info("initConfig: START"); - this.props = getConfig(); - String rid = props.getFirstPropertyValue(RESOURCE_ID_KEY); + this.config = new MinocConfig(); + MultiValuedProperties mvp = config.getProperties(); + String rid = mvp.getFirstPropertyValue(MinocConfig.RESOURCE_ID_KEY); + jndiConfigKey = appName + "-" + MinocConfig.class.getName(); + try { + Context ctx = new InitialContext(); + try { + ctx.unbind(jndiConfigKey); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiConfigKey + ") failed... ignoring"); + } + ctx.bind(jndiConfigKey, config); + + log.info("created JNDI key: " + jndiConfigKey + " object: " + config.getClass().getName()); + } catch (Exception ex) { + log.error("Failed to create JNDI Key " + jndiConfigKey, ex); + } try { this.resourceID = new URI(rid); - this.daoConfig = getDaoConfig(props); log.info("initConfig: OK"); } catch (URISyntaxException ex) { - throw new IllegalStateException("invalid config: " + RESOURCE_ID_KEY + " must be a valid URI"); + throw new IllegalStateException("invalid config: " + MinocConfig.RESOURCE_ID_KEY + " must be a valid URI"); } } private void initDatabase() { log.info("initDatabase: START"); try { - DataSource ds = DBUtil.findJNDIDataSource(JNDI_DATASOURCE); + Map daoConfig = config.getDaoConfig(); + DataSource ds = DBUtil.findJNDIDataSource(MinocConfig.JNDI_DATASOURCE); String database = (String) daoConfig.get("database"); String schema = (String) daoConfig.get("schema"); InitDatabase init = new InitDatabase(ds, database, schema); init.doInit(); - log.info("initDatabase: " + JNDI_DATASOURCE + " " + schema + " OK"); + log.info("initDatabase: " + MinocConfig.JNDI_DATASOURCE + " " + schema + " OK"); } catch (Exception ex) { throw new IllegalStateException("check/init database failed", ex); } @@ -309,6 +184,7 @@ private void initDatabase() { private void initStorageSite() { log.info("initStorageSite: START"); + Map daoConfig = config.getDaoConfig(); StorageSiteDAO ssdao = new StorageSiteDAO(); ssdao.setConfig(daoConfig); @@ -321,8 +197,8 @@ private void initStorageSite() { name = name.substring(1); } - boolean allowRead = !props.getProperty(READ_GRANTS_KEY).isEmpty(); - boolean allowWrite = !props.getProperty(WRITE_GRANTS_KEY).isEmpty(); + boolean allowRead = !config.getReadGrantServices().isEmpty(); + boolean allowWrite = !config.getWriteGrantServices().isEmpty(); StorageSite self = null; if (curlist.isEmpty()) { @@ -345,12 +221,7 @@ private void initStorageSite() { private void initStorageAdapter() { log.info("initStorageAdapter: START"); - StorageAdapter storageAdapter = InventoryUtil.loadPlugin(props.getFirstPropertyValue(MinocInitAction.SA_KEY)); - List rec = MinocInitAction.getRecoverableNamespaces(props); - for (Namespace ns : rec) { - log.info("initStorageAdapter: recoverableNamespace = " + ns.getNamespace()); - } - storageAdapter.setRecoverableNamespaces(rec); - log.info("initStorageAdapter: " + storageAdapter.getClass().getName() + " OK"); + StorageAdapter sa = config.getStorageAdapter(); + log.info("initStorageAdapter: " + sa.getClass().getName() + " OK"); } } diff --git a/minoc/src/main/java/org/opencadc/minoc/PostAction.java b/minoc/src/main/java/org/opencadc/minoc/PostAction.java index b383feadf..38a07ce63 100644 --- a/minoc/src/main/java/org/opencadc/minoc/PostAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/PostAction.java @@ -97,6 +97,7 @@ public PostAction() { */ @Override public void initAction() throws Exception { + super.initAction(); checkWritable(); initAndAuthorize(WriteGrant.class); initDAO(); diff --git a/minoc/src/main/java/org/opencadc/minoc/PutAction.java b/minoc/src/main/java/org/opencadc/minoc/PutAction.java index 4e4aec402..44c099dc7 100644 --- a/minoc/src/main/java/org/opencadc/minoc/PutAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/PutAction.java @@ -123,6 +123,7 @@ public Content accept(String name, String contentType, InputStream inputStream) @Override public void initAction() throws Exception { + super.initAction(); checkWritable(); initAndAuthorize(WriteGrant.class); initDAO(); diff --git a/minoc/src/main/java/org/opencadc/minoc/ServiceAvailability.java b/minoc/src/main/java/org/opencadc/minoc/ServiceAvailability.java index 2dad0c9e8..d5139b84a 100644 --- a/minoc/src/main/java/org/opencadc/minoc/ServiceAvailability.java +++ b/minoc/src/main/java/org/opencadc/minoc/ServiceAvailability.java @@ -142,15 +142,15 @@ public Availability getStatus() { String note = "service is accepting requests"; try { - MultiValuedProperties props = MinocInitAction.getConfig(); - Map config = MinocInitAction.getDaoConfig(props); - ArtifactDAO dao = new ArtifactDAO(); - dao.setConfig(config); // connectivity tested - String state = getState(); if (RestAction.STATE_OFFLINE.equals(state)) { return new Availability(false, RestAction.STATE_OFFLINE_MSG); } + + MinocConfig config = MinocInitAction.getConfig(appName); + ArtifactDAO dao = new ArtifactDAO(); + dao.setConfig(config.getDaoConfig()); // connectivity tested + if (RestAction.STATE_READ_ONLY.equals(state)) { return new Availability(false, RestAction.STATE_READ_ONLY_MSG); } @@ -213,6 +213,9 @@ public Availability getStatus() { } } + // TODO: check grant providers + // TODO: check for null pubkeys for trusted services + } catch (CheckException ce) { // tests determined that the resource is not working isGood = false; From 19f09f8a9ae2c2508c226609f5c7d783acb4f5c3 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 14 Dec 2023 14:51:57 -0800 Subject: [PATCH 084/186] Re-work after code review --- .../src/intTest/java/org/opencadc/raven/NegotiationTest.java | 4 ---- raven/src/main/java/org/opencadc/raven/RavenInitAction.java | 3 --- raven/src/main/webapp/WEB-INF/web.xml | 2 +- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java index 2e8d279ef..a9b3742f5 100644 --- a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java @@ -840,10 +840,7 @@ public Object run() throws Exception { @Test public void testPreauthURL() throws Exception { - List requested = new ArrayList<>(); - Protocol files = new Protocol(Standards.SI_FILES); - requested.add(files); StorageSite site = new StorageSite(CONSIST_RESOURCE_ID, "site1", true, true); try { // get raven pub key @@ -872,7 +869,6 @@ public Object run() throws Exception { p.setSecurityMethod(Standards.SECURITY_METHOD_ANON); Transfer transfer = new Transfer(artifactURI, Direction.pushToVoSpace); transfer.getProtocols().add(p); - transfer.getProtocols().add(files); transfer.version = VOS.VOSPACE_21; Transfer negotiated = negotiate(transfer); List endPoints = negotiated.getAllEndpoints(VOS.PROTOCOL_HTTPS_PUT.toASCIIString()); diff --git a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java index f89eb820a..9594df9cf 100644 --- a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java +++ b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java @@ -83,7 +83,6 @@ import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; -import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; @@ -148,8 +147,6 @@ void initDAO() { Map dc = getDaoConfig(props); ArtifactDAO artifactDAO = new ArtifactDAO(); artifactDAO.setConfig(dc); // connectivity tested - PreauthKeyPairDAO kpDAO = new PreauthKeyPairDAO(); - kpDAO.setConfig(dc); log.info("initDAO: OK"); } diff --git a/raven/src/main/webapp/WEB-INF/web.xml b/raven/src/main/webapp/WEB-INF/web.xml index 5736dceb3..113720a06 100644 --- a/raven/src/main/webapp/WEB-INF/web.xml +++ b/raven/src/main/webapp/WEB-INF/web.xml @@ -119,7 +119,7 @@ PubKeyServlet - /pubkey/* + /pubkey From 0d811ea27288ea4286351c45b82292937cf79415 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 14 Dec 2023 15:57:01 -0800 Subject: [PATCH 085/186] Fixed typo --- raven/src/intTest/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/src/intTest/README.md b/raven/src/intTest/README.md index a37bbdca0..fabf61573 100644 --- a/raven/src/intTest/README.md +++ b/raven/src/intTest/README.md @@ -46,5 +46,5 @@ org.opencadc.raven.putPreference=@SITE3 org.opencadc.raven.consistency.preventNotFound=true # external resolvers -ca.nrc.cadc.net.StorageResolver=mast ca.nrc.cadc.caom2.artifact.resolvers.MastResolver +ca.nrc.cadc.net.StorageResolver=ca.nrc.cadc.caom2.artifact.resolvers.MastResolver ``` From b261935da69213f93982933c60df17119abe8d1f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 15 Dec 2023 15:16:51 -0800 Subject: [PATCH 086/186] move getKeyAction from vault to cadc-inventory-server --- .../inventory/transfer}/GetKeyAction.java | 6 +- .../java/org/opencadc/minoc/MinocConfig.java | 386 ++++++++++++++++++ vault/src/main/webapp/WEB-INF/web.xml | 27 +- 3 files changed, 404 insertions(+), 15 deletions(-) rename {vault/src/main/java/org/opencadc/vault => cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer}/GetKeyAction.java (97%) create mode 100644 minoc/src/main/java/org/opencadc/minoc/MinocConfig.java diff --git a/vault/src/main/java/org/opencadc/vault/GetKeyAction.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java similarity index 97% rename from vault/src/main/java/org/opencadc/vault/GetKeyAction.java rename to cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java index d0b1fa46e..fa2075e84 100644 --- a/vault/src/main/java/org/opencadc/vault/GetKeyAction.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java @@ -65,7 +65,7 @@ ************************************************************************ */ -package org.opencadc.vault; +package org.opencadc.inventory.transfer; import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.rest.RestAction; @@ -77,7 +77,9 @@ import org.opencadc.inventory.PreauthKeyPair; /** - * + * Simple GET action that finds a PreauthKeyPair via JNDI and writes + * the binary public key to the output. + * * @author pdowler */ public class GetKeyAction extends RestAction { diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java new file mode 100644 index 000000000..52c1f8130 --- /dev/null +++ b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java @@ -0,0 +1,386 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2023. (c) 2023. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.minoc; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.net.HttpTransfer; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.util.MultiValuedProperties; +import ca.nrc.cadc.util.PropertiesReader; +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.log4j.Logger; +import org.opencadc.inventory.InventoryUtil; +import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.storage.StorageAdapter; + +/** + * Configuration object that can be stored in JNDI and used by actions. + * + * @author pdowler + */ +public class MinocConfig { + private static final Logger log = Logger.getLogger(MinocConfig.class); + + static final String JNDI_DATASOURCE = "jdbc/inventory"; // context.xml + + // config keys + private static final String MINOC_KEY = "org.opencadc.minoc"; + static final String RESOURCE_ID_KEY = MINOC_KEY + ".resourceID"; + static final String SQLGEN_KEY = SQLGenerator.class.getName(); + static final String SCHEMA_KEY = MINOC_KEY + ".inventory.schema"; + static final String SA_KEY = StorageAdapter.class.getName(); + static final String TRUST_KEY = MINOC_KEY + ".trust.preauth"; + static final String READ_GRANTS_KEY = MINOC_KEY + ".readGrantProvider"; + static final String WRITE_GRANTS_KEY = MINOC_KEY + ".writeGrantProvider"; + static final String RECOVERABLE_NS_KEY = MINOC_KEY + ".recoverableNamespace"; + static final String DEV_AUTH_ONLY_KEY = MINOC_KEY + ".authenticateOnly"; + + private final MultiValuedProperties configProperties; + + private final Map trustedServices = new TreeMap<>(); + private boolean trustedServiceKeySync = false; + private final List readGrantServices = new ArrayList<>(); + private final List writeGrantServices = new ArrayList<>(); + private final List recoverableNamespaces = new ArrayList<>(); + + final boolean authenticateOnly; + + public MinocConfig() { + PropertiesReader r = new PropertiesReader("minoc.properties"); + this.configProperties = r.getAllProperties(); + + validateConfigProps(configProperties); + + // from here on, fail on invalid config + List readGrants = configProperties.getProperty(READ_GRANTS_KEY); + if (readGrants != null) { + for (String s : readGrants) { + try { + URI u = new URI(s); + readGrantServices.add(u); + } catch (URISyntaxException ex) { + throw new IllegalStateException("invalid config: " + READ_GRANTS_KEY + "=" + s + " INVALID", ex); + } + } + } + + List writeGrants = configProperties.getProperty(WRITE_GRANTS_KEY); + if (writeGrants != null) { + for (String s : writeGrants) { + try { + URI u = new URI(s); + writeGrantServices.add(u); + } catch (URISyntaxException ex) { + throw new IllegalStateException("invalid config: " + WRITE_GRANTS_KEY + "=" + s + " INVALID", ex); + } + } + } + + String ao = configProperties.getFirstPropertyValue(DEV_AUTH_ONLY_KEY); + if (ao != null) { + try { + this.authenticateOnly = Boolean.valueOf(ao); + if (authenticateOnly) { + log.warn("(configuration) authenticateOnly = " + authenticateOnly); + } + } catch (Exception ex) { + throw new IllegalStateException("invalid config: " + DEV_AUTH_ONLY_KEY + "=" + ao + " must be true|false or not set"); + } + } else { + authenticateOnly = false; + } + + List trusted = configProperties.getProperty(TRUST_KEY); + if (trusted != null) { + for (String s : trusted) { + try { + URI u = new URI(s); + trustedServices.put(u, null); + } catch (URISyntaxException ex) { + throw new IllegalStateException("invalid config: " + TRUST_KEY + "=" + s + " INVALID", ex); + } + } + trustedServiceKeySync = false; + syncKeys(); + } + + List recov = configProperties.getProperty(RECOVERABLE_NS_KEY); + if (recov != null) { + for (String s : recov) { + } + } + } + + private void validateConfigProps(MultiValuedProperties mvp) { + // plain validate once at startup + StringBuilder sb = new StringBuilder(); + sb.append("incomplete config: "); + boolean ok = true; + + String rid = mvp.getFirstPropertyValue(MinocConfig.RESOURCE_ID_KEY); + sb.append("\n\t" + RESOURCE_ID_KEY + ": "); + if (rid == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + + String sac = mvp.getFirstPropertyValue(SA_KEY); + sb.append("\n\t").append(SA_KEY).append(": "); + if (sac == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + + String sqlgen = mvp.getFirstPropertyValue(SQLGEN_KEY); + sb.append("\n\t").append(SQLGEN_KEY).append(": "); + if (sqlgen == null) { + sb.append("MISSING"); + ok = false; + } else { + try { + Class c = Class.forName(sqlgen); + sb.append("OK"); + } catch (ClassNotFoundException ex) { + sb.append("class not found: " + sqlgen); + ok = false; + } + } + + String schema = mvp.getFirstPropertyValue(SCHEMA_KEY); + sb.append("\n\t").append(SCHEMA_KEY).append(": "); + if (schema == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } + + // optional + List trusted = mvp.getProperty(TRUST_KEY); + if (trusted != null) { + for (String s : trusted) { + sb.append("\n\t").append(TRUST_KEY + "=").append(s); + try { + URI uri = new URI(s); + sb.append(" OK"); + } catch (URISyntaxException ex) { + sb.append(" INVALID"); + ok = false; + } + } + } + + // optional + List readGrants = mvp.getProperty(READ_GRANTS_KEY); + if (readGrants != null) { + for (String s : readGrants) { + sb.append("\n\t").append(READ_GRANTS_KEY + "=").append(s); + try { + URI u = new URI(s); + sb.append(" OK"); + } catch (URISyntaxException ex) { + sb.append(" INVALID"); + ok = false; + } + } + } + + // optional + List writeGrants = mvp.getProperty(WRITE_GRANTS_KEY); + if (writeGrants != null) { + for (String s : writeGrants) { + sb.append("\n\t").append(WRITE_GRANTS_KEY + "=").append(s); + try { + URI u = new URI(s); + sb.append(" OK"); + } catch (URISyntaxException ex) { + sb.append(" INVALID"); + ok = false; + } + } + } + + // optional + List rawRecNS = mvp.getProperty(RECOVERABLE_NS_KEY); + if (rawRecNS != null) { + for (String s : rawRecNS) { + sb.append("\n\t").append(RECOVERABLE_NS_KEY + "=").append(s); + try { + Namespace ns = new Namespace(s); + } catch (Exception ex) { + sb.append(" INVALID"); + } + } + } + + if (!ok) { + throw new IllegalStateException(sb.toString()); + } + } + + public MultiValuedProperties getProperties() { + return configProperties; + } + + public Map getTrustedServices() { + syncKeys(); + return trustedServices; + } + + private void syncKeys() { + if (!trustedServiceKeySync) { + int numFails = 0; + RegistryClient reg = new RegistryClient(); + // check map for null keys and try to retrieve them + for (Map.Entry me : trustedServices.entrySet()) { + if (me.getValue() == null) { + try { + log.info("get trusted pubkey: " + me.getKey()); + URL capURL = reg.getAccessURL(RegistryClient.Query.CAPABILITIES, me.getKey()); + String s = capURL.toExternalForm().replace("/capabilities", "/pubkey"); + URL keyURL = new URL(s); + log.info("get trusted pubkey: " + me.getKey() + " -> " + keyURL); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(keyURL, bos); + get.setConnectionTimeout(6000); + get.setReadTimeout(6000); + get.setRetry(0, 0, HttpTransfer.RetryReason.NONE); + get.run(); + if (get.getThrowable() != null) { + throw (Exception) get.getThrowable(); + } + byte[] key = bos.toByteArray(); + trustedServices.put(me.getKey(), key); + log.info("get trusted pubkey: " + me.getKey() + " OK"); + } catch (Exception ex) { + log.warn("failed to get public key from " + me.getKey() + ": " + ex); + numFails++; + } + } + } + if (numFails == 0) { + trustedServiceKeySync = true; + } + } + } + + public List getReadGrantServices() { + return readGrantServices; + } + + public List getWriteGrantServices() { + return writeGrantServices; + } + + public boolean isAuthenticateOnly() { + return authenticateOnly; + } + + public List getRecoverableNamespaces() { + return recoverableNamespaces; + } + + public StorageAdapter getStorageAdapter() { + String cname = configProperties.getFirstPropertyValue(MinocConfig.SA_KEY); + StorageAdapter storageAdapter = InventoryUtil.loadPlugin(cname); + for (Namespace ns : recoverableNamespaces) { + log.info("initStorageAdapter: recoverableNamespace = " + ns.getNamespace()); + } + storageAdapter.setRecoverableNamespaces(recoverableNamespaces); + return storageAdapter; + } + + public Map getDaoConfig() { + String cname = configProperties.getFirstPropertyValue(SQLGenerator.class.getName()); + try { + Map ret = new TreeMap<>(); + Class clz = Class.forName(cname); + ret.put(SQLGenerator.class.getName(), clz); + ret.put("jndiDataSourceName", JNDI_DATASOURCE); + ret.put("schema", configProperties.getFirstPropertyValue(SCHEMA_KEY)); + //config.put("database", null); + return ret; + } catch (ClassNotFoundException ex) { + throw new IllegalStateException("invalid config: failed to load SQLGenerator: " + cname); + } + } +} diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 2d4793089..cdfc34518 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -34,19 +34,6 @@ 1 - - PubKeyServlet - ca.nrc.cadc.rest.RestServlet - - augmentSubject - false - - - get - org.opencadc.vault.GetKeyAction - - - NodesServlet ca.nrc.cadc.rest.RestServlet @@ -76,6 +63,20 @@ 2 + + + PubKeyServlet + ca.nrc.cadc.rest.RestServlet + + augmentSubject + false + + + get + org.opencadc.inventory.transfer.GetKeyAction + + 3 + From 7e87b280dae07453b0e263efa6d2c21283497f1e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 15 Dec 2023 15:17:57 -0800 Subject: [PATCH 087/186] add reg logging to minoc --- minoc/src/main/webapp/WEB-INF/web.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/minoc/src/main/webapp/WEB-INF/web.xml b/minoc/src/main/webapp/WEB-INF/web.xml index 6616cfdca..d58ecc74c 100644 --- a/minoc/src/main/webapp/WEB-INF/web.xml +++ b/minoc/src/main/webapp/WEB-INF/web.xml @@ -25,6 +25,7 @@ ca.nrc.cadc.util ca.nrc.cadc.vosi ca.nrc.cadc.db + ca.nrc.cadc.reg From 23a5a97ced461e6671ecebd21b50c30b2e853a76 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 15 Dec 2023 16:38:42 -0800 Subject: [PATCH 088/186] warn to debug in lib --- .../java/org/opencadc/inventory/transfer/GetKeyAction.java | 4 ++-- .../org/opencadc/inventory/transfer/ProtocolsGenerator.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java index fa2075e84..e3396d87b 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java @@ -99,9 +99,9 @@ public void doAction() throws Exception { String jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); Context ctx = new InitialContext(); try { - log.warn("lookup: " + jndiPreauthKeys); + log.debug("lookup: " + jndiPreauthKeys); PreauthKeyPair keys = (PreauthKeyPair) ctx.lookup(jndiPreauthKeys); - log.warn("found: " + keys); + log.debug("found: " + keys); byte[] pub = keys.getPublicKey(); syncOutput.setHeader("content-length", pub.length); syncOutput.setHeader("content-type", "application/octet-stream"); diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index c34267f54..5cf4a5329 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -395,7 +395,7 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) log.debug("added: " + p); // add a plain anon URL - if (authToken != null && Standards.SECURITY_METHOD_ANON.equals(sec)) { + if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) { sb = new StringBuilder(); sb.append(baseURL.toExternalForm()).append("/"); sb.append(artifactURI.toASCIIString()); From 54abbbbb656bbb01154f7af929488dff62eb8294 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 15 Dec 2023 16:39:20 -0800 Subject: [PATCH 089/186] integrate and test magic key pair code --- .../org/opencadc/minoc/ArtifactAction.java | 31 +++----- .../org/opencadc/raven/ArtifactAction.java | 51 +++++------- .../org/opencadc/raven/RavenInitAction.java | 78 ++++++++++++++++--- raven/src/main/webapp/WEB-INF/web.xml | 20 +++-- .../opencadc/vault/NodePersistenceImpl.java | 19 +---- .../org/opencadc/vault/VaultInitAction.java | 3 + .../vault/VaultTransferGenerator.java | 4 +- 7 files changed, 119 insertions(+), 87 deletions(-) diff --git a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java index e7e764e73..5cb3aa59e 100644 --- a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -76,21 +76,16 @@ import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.rest.SyncInput; import ca.nrc.cadc.rest.Version; -import ca.nrc.cadc.util.MultiValuedProperties; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.security.AccessControlException; import java.security.cert.CertificateException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import java.util.TreeMap; import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.InventoryUtil; -import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.storage.StorageAdapter; import org.opencadc.permissions.Grant; @@ -205,15 +200,12 @@ protected void initAndAuthorize(Class grantClass, boolean allow // do authorization (with token or subject) Subject subject = AuthenticationUtil.getCurrentSubject(); if (authToken != null) { - //if (publicKey == null) { - // throw new IllegalArgumentException("unexpected pre-auth token in URL"); - //} - //TokenTool tk = new TokenTool(publicKey); - for (Map.Entry me : config.getTrustedServices().entrySet()) { - URI ts = me.getKey(); + Map trusted = config.getTrustedServices(); + log.debug("trusted services: " + trusted.size()); + for (Map.Entry me : trusted.entrySet()) { if (me.getValue() != null) { TokenTool tk = new TokenTool(me.getValue()); - log.warn("validate preauth with key from " + me.getKey()); + log.debug("validate preauth with key from " + me.getKey()); try { String tokenUser; if (allowReadWithWriteGrant && ReadGrant.class.isAssignableFrom(grantClass)) { @@ -230,22 +222,23 @@ protected void initAndAuthorize(Class grantClass, boolean allow logInfo.setResource(artifactURI); logInfo.setPath(syncInput.getContextPath() + syncInput.getComponentPath()); if (ReadGrant.class.isAssignableFrom(grantClass)) { - logInfo.setGrant("read: preauth-token from " + ts); + logInfo.setGrant("read:preauth-token:" + me.getKey()); } else if (WriteGrant.class.isAssignableFrom(grantClass)) { - logInfo.setGrant("write: preauth-token from " + ts); + logInfo.setGrant("write:preauth-token:" + me.getKey()); } else { throw new IllegalStateException("Unsupported grant class: " + grantClass); } + // granted return; } catch (AccessControlException ex) { - log.warn("token invalid vs keys from " + ts, ex); + log.debug("token invalid vs keys from " + me.getKey()); } } else { - log.warn("no keys from " + ts + " -- SKIP"); + log.warn("no keys from " + me.getKey() + " -- SKIP"); } - // no return from inside check - throw new AccessControlException("invalid auth token"); } + // no return from inside check + throw new AccessControlException("invalid auth token"); } // augment subject (minoc is configured so augment is not done in rest library) diff --git a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java index 3ba5e79d8..1b13ce7b3 100644 --- a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java +++ b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java @@ -89,6 +89,7 @@ import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.PreauthKeyPairDAO; +import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; import org.opencadc.permissions.ReadGrant; import org.opencadc.permissions.TokenTool; @@ -114,9 +115,7 @@ public abstract class ArtifactAction extends RestAction { // immutable state set in constructor protected final ArtifactDAO artifactDAO; - protected final PreauthKeyPairDAO preauthKeyPairDAO; - protected final TokenTool tokenGen; - protected final byte[] publicKey; + protected TokenTool tokenGen; protected final List readGrantServices = new ArrayList<>(); protected final List writeGrantServices = new ArrayList<>(); protected StorageResolver storageResolver; @@ -131,12 +130,9 @@ public abstract class ArtifactAction extends RestAction { ArtifactAction(boolean init) { super(); this.authenticateOnly = false; - this.tokenGen = null; this.artifactDAO = null; - this.preauthKeyPairDAO = null; this.preventNotFound = false; this.storageResolver = null; - this.publicKey = null; } protected ArtifactAction() { @@ -179,29 +175,7 @@ protected ArtifactAction() { authenticateOnly = false; } - initResolver(); - Map config = RavenInitAction.getDaoConfig(props); - this.preauthKeyPairDAO = new PreauthKeyPairDAO(); - preauthKeyPairDAO.setConfig(config); // connectivity tested - PreauthKeyPair preauthKP = preauthKeyPairDAO.get(RavenInitAction.PREAUTH_KEYPAIR); - if (preauthKP == null) { - log.info("Generate the pre-auth key pair"); - KeyPair keyPair = RsaSignatureGenerator.getKeyPair(4096); //TODO size? - preauthKP = new PreauthKeyPair(RavenInitAction.PREAUTH_KEYPAIR, keyPair.getPublic().getEncoded(), keyPair.getPrivate().getEncoded()); - try { - preauthKeyPairDAO.put(preauthKP); - } catch (Exception ex) { - throw new IllegalStateException("Could not persist the pre-auth key pair.", ex); - } - this.tokenGen = new TokenTool(keyPair.getPublic().getEncoded(), keyPair.getPrivate().getEncoded()); - this.publicKey = keyPair.getPublic().getEncoded(); - } else { - log.debug("Use existing pre-auth key pair"); - this.tokenGen = new TokenTool(preauthKP.getPublicKey(), preauthKP.getPrivateKey()); - this.publicKey = preauthKP.getPublicKey(); - } - this.artifactDAO = new ArtifactDAO(); artifactDAO.setConfig(config); // connectivity tested @@ -216,7 +190,24 @@ protected ArtifactAction() { } } - protected void initResolver() { + @Override + public void initAction() throws Exception { + super.initAction(); + initResolver(); + + String jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); + Context ctx = new InitialContext(); + try { + log.debug("lookup: " + jndiPreauthKeys); + PreauthKeyPair keys = (PreauthKeyPair) ctx.lookup(jndiPreauthKeys); + log.debug("found: " + keys); + this.tokenGen = new TokenTool(keys.getPublicKey(), keys.getPrivateKey()); + } catch (NamingException ex) { + throw new RuntimeException("BUG: failed to find keys via JNDI", ex); + } + } + + void initResolver() { MultiValuedProperties props = RavenInitAction.getConfig(); String resolverName = props.getFirstPropertyValue(RavenInitAction.RESOLVER_ENTRY); if (resolverName != null) { @@ -273,7 +264,7 @@ void init() throws Exception { throw new IllegalArgumentException("Missing artifact URI from path or request content"); } - String siteAvailabilitiesKey = this.appName + RavenInitAction.JNDI_AVAILABILITY_NAME; + String siteAvailabilitiesKey = this.appName + "-" + StorageSiteAvailabilityCheck.class.getName(); log.debug("siteAvailabilitiesKey: " + siteAvailabilitiesKey); try { Context initContext = new InitialContext(); diff --git a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java index 9594df9cf..e5ddb042b 100644 --- a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java +++ b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java @@ -67,26 +67,34 @@ package org.opencadc.raven; +import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.InitAction; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.util.RsaSignatureGenerator; import ca.nrc.cadc.util.StringUtil; import java.net.URI; import java.net.URISyntaxException; +import java.security.KeyPair; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; +import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingException; +import javax.sql.DataSource; import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; +import org.springframework.dao.DataIntegrityViolationException; /** * @@ -95,26 +103,25 @@ public class RavenInitAction extends InitAction { private static final Logger log = Logger.getLogger(RavenInitAction.class); + static String KEY_PAIR_NAME = "raven-preauth-keys"; + // config keys private static final String RAVEN_KEY = "org.opencadc.raven"; private static final String RAVEN_CONSIST_KEY = "org.opencadc.raven.consistency"; static final String JNDI_DATASOURCE = "jdbc/inventory"; // context.xml - static final String JNDI_AVAILABILITY_NAME = ".availabilities"; static final String SCHEMA_KEY = RAVEN_KEY + ".inventory.schema"; - - static final String PREAUTH_KEYPAIR = RAVEN_KEY + ".keyPair"; static final String READ_GRANTS_KEY = RAVEN_KEY + ".readGrantProvider"; static final String WRITE_GRANTS_KEY = RAVEN_KEY + ".writeGrantProvider"; - static final String RESOLVER_ENTRY = "ca.nrc.cadc.net.StorageResolver"; static final String PREVENT_NOT_FOUND_KEY = RAVEN_CONSIST_KEY + ".preventNotFound"; - static final String DEV_AUTH_ONLY_KEY = RAVEN_KEY + ".authenticateOnly"; // set init initConfig, used by subsequent init methods MultiValuedProperties props; + + private String jndiPreauthKeys; private String siteAvailabilitiesKey; private Thread availabilityCheck; @@ -126,6 +133,7 @@ public RavenInitAction() { public void doInit() { initConfig(); initDAO(); + initKeyPair(); initGrantProviders(); initStorageSiteRules(); initAvailabilityCheck(); @@ -133,7 +141,14 @@ public void doInit() { @Override public void doShutdown() { - terminate(); + try { + Context ctx = new InitialContext(); + ctx.unbind(jndiPreauthKeys); + } catch (Exception oops) { + log.error("unbind failed during destroy", oops); + } + + terminateAvailabilityCheck(); } void initConfig() { @@ -184,18 +199,61 @@ void initStorageSiteRules() { log.info("initStorageSiteRules: OK"); } + private void initKeyPair() { + log.info("initKeyPair: START"); + jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); + try { + Map dc = getDaoConfig(props); + PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); + dao.setConfig(dc); + PreauthKeyPair keys = dao.get(KEY_PAIR_NAME); + if (keys == null) { + KeyPair kp = RsaSignatureGenerator.getKeyPair(4096); + keys = new PreauthKeyPair(KEY_PAIR_NAME, kp.getPublic().getEncoded(), kp.getPrivate().getEncoded()); + try { + dao.put(keys); + log.info("initKeyPair: new keys created - OK"); + + } catch (DataIntegrityViolationException oops) { + log.warn("persist new " + PreauthKeyPair.class.getSimpleName() + " failed (" + oops + ") -- probably race condition"); + keys = dao.get(KEY_PAIR_NAME); + if (keys != null) { + log.info("race condition confirmed: another instance created keys - OK"); + } else { + throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", oops); + } + } + } else { + log.info("initKeyPair: re-use existing keys - OK"); + } + Context ctx = new InitialContext(); + try { + ctx.unbind(jndiPreauthKeys); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); + } + ctx.bind(jndiPreauthKeys, keys); + log.info("initKeyPair: created JNDI key: " + jndiPreauthKeys); + + Object o = ctx.lookup(jndiPreauthKeys); + log.info("checking... found: " + jndiPreauthKeys + " = " + o + " in " + ctx); + } catch (Exception ex) { + throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); + } + } + void initAvailabilityCheck() { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(); storageSiteDAO.setConfig(getDaoConfig(props)); - this.siteAvailabilitiesKey = this.appName + RavenInitAction.JNDI_AVAILABILITY_NAME; - terminate(); - this.availabilityCheck = new Thread(new StorageSiteAvailabilityCheck(storageSiteDAO, this.siteAvailabilitiesKey)); + this.siteAvailabilitiesKey = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); + terminateAvailabilityCheck(); + this.availabilityCheck = new Thread(new StorageSiteAvailabilityCheck(storageSiteDAO, siteAvailabilitiesKey)); this.availabilityCheck.setDaemon(true); this.availabilityCheck.start(); } - private final void terminate() { + private final void terminateAvailabilityCheck() { if (this.availabilityCheck != null) { try { log.info("terminating AvailabilityCheck Thread..."); diff --git a/raven/src/main/webapp/WEB-INF/web.xml b/raven/src/main/webapp/WEB-INF/web.xml index 113720a06..c76bf745d 100644 --- a/raven/src/main/webapp/WEB-INF/web.xml +++ b/raven/src/main/webapp/WEB-INF/web.xml @@ -48,14 +48,18 @@ - PubKeyServlet - ca.nrc.cadc.rest.RestServlet - - get - org.opencadc.raven.GetPubKeyAction - - 4 - + PubKeyServlet + ca.nrc.cadc.rest.RestServlet + + augmentSubject + false + + + get + org.opencadc.inventory.transfer.GetKeyAction + + 3 + FilesServlet diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index dcfe9805b..0499727d3 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -155,7 +155,6 @@ public class NodePersistenceImpl implements NodePersistence { private final Map nodeDaoConfig = new TreeMap<>(); private final ContainerNode root; - private final ContainerNode trash; private final Namespace storageNamespace; private final boolean localGroupsOnly; @@ -181,27 +180,11 @@ public NodePersistenceImpl(URI resourceID) { this.root = new ContainerNode(rootID, ""); root.owner = getRootOwner(config, identityManager); root.ownerDisplay = identityManager.toDisplayString(root.owner); - log.warn("ROOT owner: " + root.owner); + log.info("ROOT owner: " + root.owner); root.ownerID = identityManager.toOwner(root.owner); root.isPublic = true; root.inheritPermissions = false; - // trash node - // TODO: do this setup in a txn with a lock on something - NodeDAO dao = getDAO(); - ContainerNode tn = (ContainerNode) dao.get(root, ".trash"); - if (tn == null) { - tn = new ContainerNode(".trash"); - } - // always reset props to current config - tn.ownerID = root.ownerID; - tn.owner = root.owner; - tn.isPublic = false; - tn.inheritPermissions = false; - tn.parentID = rootID; - dao.put(tn); - this.trash = tn; - String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); this.storageNamespace = new Namespace(ns); diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 95b803237..e291fd651 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -320,6 +320,9 @@ private void initKeyPair() { } ctx.bind(jndiPreauthKeys, keys); log.info("initKeyPair: created JNDI key: " + jndiPreauthKeys); + + Object o = ctx.lookup(jndiPreauthKeys); + log.info("checking... found: " + jndiPreauthKeys + " = " + o + " in " + ctx); } catch (Exception ex) { throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index d812343a2..ee5ea66c9 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -205,9 +205,9 @@ private List handleDataNode(DataNode node, Transfer trans, Subject s) try { List ret = pg.getProtocols(artifactTrans); - log.warn("generated urls: " + ret.size()); + log.debug("generated urls: " + ret.size()); for (Protocol p : ret) { - log.warn(p.getEndpoint() + " using " + p.getSecurityMethod()); + log.debug(p.getEndpoint() + " using " + p.getSecurityMethod()); } return ret; } catch (ResourceNotFoundException ex) { From 8110acc2820b1c1c35753950f1c291376f803a9c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 13:12:56 -0800 Subject: [PATCH 090/186] raven: add an admin pool that can init the inventory db and store preauth keys --- raven/README.md | 27 +++- raven/VERSION | 2 +- .../org/opencadc/raven/ArtifactAction.java | 33 +++-- .../org/opencadc/raven/RavenInitAction.java | 137 +++++++++++------- raven/src/main/webapp/META-INF/context.xml | 16 +- .../opencadc/raven/StorageResolverTest.java | 5 +- .../raven.properties | 6 +- .../testValidConfig/raven.properties | 5 +- 8 files changed, 150 insertions(+), 81 deletions(-) diff --git a/raven/README.md b/raven/README.md index 7801a3f85..c121b4191 100644 --- a/raven/README.md +++ b/raven/README.md @@ -11,14 +11,23 @@ Runtime configuration must be made available via the `/config` directory. When running raven.war in tomcat, parameters of the connection pool in META-INF/context.xml need to be configured in catalina.properties: ``` -# database connection pools +# query pool for user requests org.opencadc.raven.query.maxActive={max connections for query pool} org.opencadc.raven.query.username={database username for query pool} org.opencadc.raven.query.password={database password for query pool} org.opencadc.raven.query.url=jdbc:postgresql://{server}/{database} +# admin pool for setup +org.opencadc.raven.inventory.maxActive={max connections for query pool} +org.opencadc.raven.inventory.username={database username for query pool} +org.opencadc.raven.inventory.password={database password for query pool} +org.opencadc.raven.inventory.url=jdbc:postgresql://{server}/{database} ``` -The _query_ pool is used to query inventory for the requested Artifact. +The _query_ account is used to query inventory for the requested Artifact; this pool can be +configured with a read-only database account. + +The _inventory_ account owns and manages (create, alter, drop) inventory database objects and +(optional) URL signing keys (see _keys.preauth_ below). ### cadc-registry.properties @@ -33,10 +42,17 @@ org.opencadc.raven.inventory.schema={schema} # consistency settings org.opencadc.raven.consistency.preventNotFound=true|false + +# url signing key usage +org.opencadc.raven.keys.preauth={true|false} ``` -`raven` can be configured prevent artifact-not-found errors that might result due to the eventual consistency nature of -the system by directly checking for the artifact at the sites (`preventNotFound=true`). This however introduces an -overhead for the genuine not-found cases. +The _preventNotFound_ key can be used to configure `raven` to prevent artifact-not-found errors that might +result due to the eventual consistency nature of the system by directly checking for the artifact at +_all known_ sites. This feature introduces an overhead for the genuine not-found cases. + +The _keys.preauth_ key configures `raven` to use URL-signing. When enabled, `raven` can generate a signed token +and embeds it into the URL; `minoc` services can validate the token and grant access without further permission +checks. With transfer negotiation, the signed URL gets added as an additional "anonymous" URL. The following optional keys configure raven to use external service(s) to obtain grant information in order to perform authorization checks: @@ -44,7 +60,6 @@ to perform authorization checks: org.opencadc.raven.readGrantProvider={resourceID of a permission granting service} org.opencadc.raven.writeGrantProvider={resourceID of a permission granting service} ``` - The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for operations. Multiple values of the permission granting service resourceID(s) may be provided by including multiple property settings. All services will be consulted but a single positive result is sufficient to grant permission for an diff --git a/raven/VERSION b/raven/VERSION index b73f3577f..ee967b01d 100644 --- a/raven/VERSION +++ b/raven/VERSION @@ -2,6 +2,6 @@ # semantic version tag: major.minor[.patch] # build version tag: timestamp # tag: {semantic}-{build} -VER=0.7.10 +VER=0.8.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java index 1b13ce7b3..a4c4cf98c 100644 --- a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java +++ b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java @@ -125,6 +125,7 @@ public abstract class ArtifactAction extends RestAction { protected Map siteRules; protected final boolean preventNotFound; + protected final boolean preauthKeys; // constructor for unit tests with no config/init ArtifactAction(boolean init) { @@ -132,6 +133,7 @@ public abstract class ArtifactAction extends RestAction { this.authenticateOnly = false; this.artifactDAO = null; this.preventNotFound = false; + this.preauthKeys = false; this.storageResolver = null; } @@ -175,12 +177,13 @@ protected ArtifactAction() { authenticateOnly = false; } - Map config = RavenInitAction.getDaoConfig(props); + Map config = RavenInitAction.getDaoConfig(props, RavenInitAction.JNDI_QUERY_DATASOURCE); this.artifactDAO = new ArtifactDAO(); artifactDAO.setConfig(config); // connectivity tested // get the storage site rules this.siteRules = RavenInitAction.getStorageSiteRules(props); + String pnf = props.getFirstPropertyValue(RavenInitAction.PREVENT_NOT_FOUND_KEY); if (pnf != null) { this.preventNotFound = Boolean.valueOf(pnf); @@ -188,6 +191,14 @@ protected ArtifactAction() { } else { throw new IllegalStateException("invalid config: missing preventNotFound configuration"); } + + String pak = props.getFirstPropertyValue(RavenInitAction.PREAUTH_KEY); + if (pak != null) { + this.preauthKeys = Boolean.valueOf(pak); + log.debug("Using preauth keys: " + this.preauthKeys); + } else { + throw new IllegalStateException("invalid config: missing keys.preauth configuration"); + } } @Override @@ -195,15 +206,17 @@ public void initAction() throws Exception { super.initAction(); initResolver(); - String jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); - Context ctx = new InitialContext(); - try { - log.debug("lookup: " + jndiPreauthKeys); - PreauthKeyPair keys = (PreauthKeyPair) ctx.lookup(jndiPreauthKeys); - log.debug("found: " + keys); - this.tokenGen = new TokenTool(keys.getPublicKey(), keys.getPrivateKey()); - } catch (NamingException ex) { - throw new RuntimeException("BUG: failed to find keys via JNDI", ex); + if (preauthKeys) { + String jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); + Context ctx = new InitialContext(); + try { + log.debug("lookup: " + jndiPreauthKeys); + PreauthKeyPair keys = (PreauthKeyPair) ctx.lookup(jndiPreauthKeys); + log.debug("found: " + keys); + this.tokenGen = new TokenTool(keys.getPublicKey(), keys.getPrivateKey()); + } catch (NamingException ex) { + throw new RuntimeException("BUG: failed to find keys via JNDI", ex); + } } } diff --git a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java index e5ddb042b..95c6f0c05 100644 --- a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java +++ b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java @@ -92,6 +92,7 @@ import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; +import org.opencadc.inventory.db.version.InitDatabase; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; import org.springframework.dao.DataIntegrityViolationException; @@ -107,15 +108,18 @@ public class RavenInitAction extends InitAction { // config keys private static final String RAVEN_KEY = "org.opencadc.raven"; - private static final String RAVEN_CONSIST_KEY = "org.opencadc.raven.consistency"; - static final String JNDI_DATASOURCE = "jdbc/inventory"; // context.xml + static final String JNDI_QUERY_DATASOURCE = "jdbc/query"; // context.xml + static final String JNDI_ADMIN_DATASOURCE = "jdbc/inventory"; // context.xml static final String SCHEMA_KEY = RAVEN_KEY + ".inventory.schema"; + static final String PREVENT_NOT_FOUND_KEY = RAVEN_KEY + ".consistency.preventNotFound"; + static final String PREAUTH_KEY = RAVEN_KEY + ".keys.preauth"; + static final String READ_GRANTS_KEY = RAVEN_KEY + ".readGrantProvider"; static final String WRITE_GRANTS_KEY = RAVEN_KEY + ".writeGrantProvider"; static final String RESOLVER_ENTRY = "ca.nrc.cadc.net.StorageResolver"; - static final String PREVENT_NOT_FOUND_KEY = RAVEN_CONSIST_KEY + ".preventNotFound"; + static final String DEV_AUTH_ONLY_KEY = RAVEN_KEY + ".authenticateOnly"; // set init initConfig, used by subsequent init methods @@ -132,8 +136,9 @@ public RavenInitAction() { @Override public void doInit() { initConfig(); - initDAO(); + initDatabase(); initKeyPair(); + initQueryDAO(); initGrantProviders(); initStorageSiteRules(); initAvailabilityCheck(); @@ -157,55 +162,35 @@ void initConfig() { log.info("initConfig: OK"); } - void initDAO() { - log.info("initDAO: START"); - Map dc = getDaoConfig(props); - ArtifactDAO artifactDAO = new ArtifactDAO(); - artifactDAO.setConfig(dc); // connectivity tested - log.info("initDAO: OK"); - } - - void initGrantProviders() { - log.info("initGrantProviders: START"); - List readGrants = props.getProperty(RavenInitAction.READ_GRANTS_KEY); - if (readGrants != null) { - for (String s : readGrants) { - try { - URI u = new URI(s); - log.debug(RavenInitAction.READ_GRANTS_KEY + ": " + u); - } catch (URISyntaxException ex) { - throw new IllegalStateException("invalid config: " + RavenInitAction.READ_GRANTS_KEY + "=" + s + " must be a valid URI"); - } - } - } - - List writeGrants = props.getProperty(RavenInitAction.WRITE_GRANTS_KEY); - if (writeGrants != null) { - for (String s : writeGrants) { - try { - URI u = new URI(s); - log.debug(RavenInitAction.WRITE_GRANTS_KEY + ": " + u); - } catch (URISyntaxException ex) { - throw new IllegalStateException("invalid config: " + RavenInitAction.WRITE_GRANTS_KEY + "=" + s + " must be a valid URI"); - } - } + private void initDatabase() { + log.info("initDatabase: START"); + try { + Map daoConfig = getDaoConfig(props, JNDI_ADMIN_DATASOURCE); + String jndiDataSourceName = (String) daoConfig.get("jndiDataSourceName"); + String database = (String) daoConfig.get("database"); + String schema = (String) daoConfig.get("schema"); + DataSource ds = DBUtil.findJNDIDataSource(jndiDataSourceName); + InitDatabase init = new InitDatabase(ds, database, schema); + init.doInit(); + log.info("initDatabase: " + jndiDataSourceName + " " + schema + " OK"); + } catch (Exception ex) { + throw new IllegalStateException("check/init database failed", ex); } - log.info("initGrantProviders: OK"); } - - void initStorageSiteRules() { - log.info("initStorageSiteRules: START"); - getStorageSiteRules(getConfig()); - log.info("initStorageSiteRules: OK"); - } - + private void initKeyPair() { + String enablePreauthKeys = props.getFirstPropertyValue(PREAUTH_KEY); + if (enablePreauthKeys == null || !"true".equals(enablePreauthKeys)) { + log.info("initKeyPair: " + PREAUTH_KEY + " == " + enablePreauthKeys + " - SKIP"); + return; + } + log.info("initKeyPair: START"); jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); try { - Map dc = getDaoConfig(props); + Map daoConfig = getDaoConfig(props, JNDI_ADMIN_DATASOURCE); PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); - dao.setConfig(dc); + dao.setConfig(daoConfig); PreauthKeyPair keys = dao.get(KEY_PAIR_NAME); if (keys == null) { KeyPair kp = RsaSignatureGenerator.getKeyPair(4096); @@ -234,17 +219,56 @@ private void initKeyPair() { } ctx.bind(jndiPreauthKeys, keys); log.info("initKeyPair: created JNDI key: " + jndiPreauthKeys); - - Object o = ctx.lookup(jndiPreauthKeys); - log.info("checking... found: " + jndiPreauthKeys + " = " + o + " in " + ctx); } catch (Exception ex) { throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); } } + void initQueryDAO() { + log.info("initDAO: START"); + Map dc = getDaoConfig(props, JNDI_QUERY_DATASOURCE); + ArtifactDAO artifactDAO = new ArtifactDAO(); + artifactDAO.setConfig(dc); // connectivity tested + log.info("initDAO: OK"); + } + + void initGrantProviders() { + log.info("initGrantProviders: START"); + List readGrants = props.getProperty(RavenInitAction.READ_GRANTS_KEY); + if (readGrants != null) { + for (String s : readGrants) { + try { + URI u = new URI(s); + log.debug(RavenInitAction.READ_GRANTS_KEY + ": " + u); + } catch (URISyntaxException ex) { + throw new IllegalStateException("invalid config: " + RavenInitAction.READ_GRANTS_KEY + "=" + s + " must be a valid URI"); + } + } + } + + List writeGrants = props.getProperty(RavenInitAction.WRITE_GRANTS_KEY); + if (writeGrants != null) { + for (String s : writeGrants) { + try { + URI u = new URI(s); + log.debug(RavenInitAction.WRITE_GRANTS_KEY + ": " + u); + } catch (URISyntaxException ex) { + throw new IllegalStateException("invalid config: " + RavenInitAction.WRITE_GRANTS_KEY + "=" + s + " must be a valid URI"); + } + } + } + log.info("initGrantProviders: OK"); + } + + void initStorageSiteRules() { + log.info("initStorageSiteRules: START"); + getStorageSiteRules(getConfig()); + log.info("initStorageSiteRules: OK"); + } + void initAvailabilityCheck() { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(); - storageSiteDAO.setConfig(getDaoConfig(props)); + storageSiteDAO.setConfig(getDaoConfig(props, JNDI_QUERY_DATASOURCE)); this.siteAvailabilitiesKey = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); terminateAvailabilityCheck(); @@ -305,6 +329,15 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } + + String preauthKeys = mvp.getFirstPropertyValue(RavenInitAction.PREAUTH_KEY); + sb.append("\n\t").append(RavenInitAction.PREAUTH_KEY).append(": "); + if (preauthKeys == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } if (!ok) { throw new IllegalStateException(sb.toString()); @@ -313,13 +346,13 @@ static MultiValuedProperties getConfig() { return mvp; } - static Map getDaoConfig(MultiValuedProperties props) { + static Map getDaoConfig(MultiValuedProperties props, String pool) { String cname = props.getFirstPropertyValue(SQLGenerator.class.getName()); try { Map ret = new TreeMap<>(); Class clz = Class.forName(cname); ret.put(SQLGenerator.class.getName(), clz); - ret.put("jndiDataSourceName", RavenInitAction.JNDI_DATASOURCE); + ret.put("jndiDataSourceName", pool); ret.put("schema", props.getFirstPropertyValue(RavenInitAction.SCHEMA_KEY)); //config.put("database", null); return ret; diff --git a/raven/src/main/webapp/META-INF/context.xml b/raven/src/main/webapp/META-INF/context.xml index 6e60e23c9..ad1a38896 100644 --- a/raven/src/main/webapp/META-INF/context.xml +++ b/raven/src/main/webapp/META-INF/context.xml @@ -3,7 +3,8 @@ WEB-INF/web.xml - + + + + diff --git a/raven/src/test/java/org/opencadc/raven/StorageResolverTest.java b/raven/src/test/java/org/opencadc/raven/StorageResolverTest.java index 44745f092..a89b880f2 100644 --- a/raven/src/test/java/org/opencadc/raven/StorageResolverTest.java +++ b/raven/src/test/java/org/opencadc/raven/StorageResolverTest.java @@ -86,9 +86,8 @@ public class StorageResolverTest { private static final String DEFAULT_RAVEN_CONFIG = "org.opencadc.raven.inventory.schema=inventory\n" - + "org.opencadc.raven.publicKeyFile=raven-pub.key\n" - + "org.opencadc.raven.privateKeyFile=raven-priv.key\n" - + "org.opencadc.raven.consistency.preventNotFound=true"; + + "org.opencadc.raven.consistency.preventNotFound=true\n" + + "org.opencadc.raven.keys.preauth=false\n"; static { Log4jInit.setLevel("org.opencadc.raven", Level.DEBUG); diff --git a/raven/src/test/resources/testInvalidStorageSiteRules/raven.properties b/raven/src/test/resources/testInvalidStorageSiteRules/raven.properties index 15e735dcb..925216004 100644 --- a/raven/src/test/resources/testInvalidStorageSiteRules/raven.properties +++ b/raven/src/test/resources/testInvalidStorageSiteRules/raven.properties @@ -1,10 +1,8 @@ # valid config org.opencadc.raven.inventory.schema=inventory - -org.opencadc.raven.publicKeyFile=raven-pub.key -org.opencadc.raven.privateKeyFile=raven-priv.key -org.opencadc.raven.consistency.preventNotFound=true +org.opencadc.raven.consistency.preventNotFound=false +org.opencadc.raven.keys.preauth=true # invalid storage site rules diff --git a/raven/src/test/resources/testValidConfig/raven.properties b/raven/src/test/resources/testValidConfig/raven.properties index d349689d8..c01ddaa79 100644 --- a/raven/src/test/resources/testValidConfig/raven.properties +++ b/raven/src/test/resources/testValidConfig/raven.properties @@ -1,11 +1,8 @@ # valid config org.opencadc.inventory.db.SQLGenerator=org.opencadc.inventory.db.SQLGenerator org.opencadc.raven.inventory.schema=inventory - -org.opencadc.raven.publicKeyFile=raven-pub.key -org.opencadc.raven.privateKeyFile=raven-priv.key - org.opencadc.raven.consistency.preventNotFound=true +org.opencadc.raven.keys.preauth=false org.opencadc.raven.readGrantProvider=ivo://cadc.nrc.ca/baldur org.opencadc.raven.writeGrantProvider=ivo://cadc.nrc.ca/baldur From 6f7ba0c4f11dee188a0f25b06d7be080bf71751a Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 13:31:40 -0800 Subject: [PATCH 091/186] vault: made preventNotFound configurable --- vault/README.md | 18 +++++++ .../java/org/opencadc/vault/TransferTest.java | 53 ++++++++++++++++++- .../opencadc/vault/NodePersistenceImpl.java | 11 +++- .../org/opencadc/vault/VaultInitAction.java | 10 ++++ .../vault/VaultTransferGenerator.java | 6 ++- 5 files changed, 93 insertions(+), 5 deletions(-) diff --git a/vault/README.md b/vault/README.md index 0ffeb6b3a..b2888f648 100644 --- a/vault/README.md +++ b/vault/README.md @@ -39,12 +39,21 @@ org.opencadc.vault.nodes.maxActive={max connections for vospace pool} org.opencadc.vault.nodes.username={username for vospace pool} org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} + +org.opencadc.vault.uws.maxActive={max connections for uws pool} +org.opencadc.vault.uws.username={username for uws pool} +org.opencadc.vault.uws.password={password for uws pool} +org.opencadc.vault.uws.url=jdbc:postgresql://{server}/{database} ``` The _nodes_ account owns and manages (create, alter, drop) vospace database objects and manages all the content (insert, update, delete). The database is specified in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. +The _uws_ account owns and manages (create, alter, drop) uws database objects in the `uws` schema and manages all +the content (insert, update, delete). The database is specified in the JDBC URLFailure to connect or initialize the +database will show up in logs and in the VOSI-availability output. + ### cadc-registry.properties See cadc-registry. @@ -55,6 +64,9 @@ A vault.properties file in /config is required to run this service. The followi # service identity org.opencadc.vault.resourceID = ivo://{authority}/{name} +# consistency settings +org.opencadc.vault.consistency.preventNotFound=true|false + # vault database settings org.opencadc.vault.inventory.schema = {inventory schema name} org.opencadc.vault.vospace.schema = {vospace schema name} @@ -67,6 +79,12 @@ org.opencadc.vault.storage.namespace = {a storage inventory namespace to use} ``` The vault _resourceID_ is the resourceID of _this_ vault service. +The _preventNotFound_ key can be used to configure `vault` to prevent artifact-not-found errors that might +result due to the eventual consistency nature of the storage system by directly checking for the artifact at +_all known_ sites. It only makes sense to enable this when `vault` is running in a global inventory (along with +`raven` and/or `fenwick` instances syncing artifact metadata. This feature introduces an overhead for the +genuine not-found cases: transfer negotiation to GET the file that was never PUT. + The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due to configuration limitations in luskan. diff --git a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java index 4c15f7e04..18a7d0308 100644 --- a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java @@ -67,13 +67,23 @@ package org.opencadc.vault; +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.Log4jInit; import java.io.File; import java.net.URI; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.junit.Before; import org.junit.Ignore; +import org.opencadc.inventory.StorageSite; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.db.StorageSiteDAO; /** * @@ -83,15 +93,54 @@ public class TransferTest extends org.opencadc.conformance.vos.TransferTest { private static final Logger log = Logger.getLogger(TransferTest.class); static { - Log4jInit.setLevel("org.opencadc.conformance.vos", Level.DEBUG); - Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vault", Level.INFO); + Log4jInit.setLevel("org.opencadc.conformance.vos", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.net", Level.INFO); } + // these are the same as raven intTest + static String SERVER = "INVENTORY_TEST"; + static String DATABASE = "cadctest"; + static String SCHEMA = "inventory"; + private static File ADMIN_CERT = FileUtil.getFileFromResource("vault-test.pem", NodesTest.class); public TransferTest() { super(URI.create("ivo://opencadc.org/vault"), ADMIN_CERT); } + + @Before + public void checkGlobal() throws Exception { + // make sure inventory.StorageSite has a readable/writable ivo://opencadc.org/minoc + + try { + DBConfig dbrc = new DBConfig(); + ConnectionConfig cc = dbrc.getConnectionConfig(SERVER, DATABASE); + DBUtil.createJNDIDataSource("jdbc/inventory", cc); + + Map config = new TreeMap(); + config.put(SQLGenerator.class.getName(), SQLGenerator.class); + config.put("jndiDataSourceName", "jdbc/inventory"); + config.put("schema", SCHEMA); + + StorageSiteDAO dao = new StorageSiteDAO(false); + dao.setConfig(config); + Set sites = dao.list(); + if (sites.isEmpty()) { + StorageSite ss = new StorageSite(URI.create("ivo://opencadc.org/minoc"), "test-minoc", true, true); + dao.put(ss); + log.info("created record in global: " + ss); + } + sites = dao.list(); + for (StorageSite ss : sites) { + log.info("storage sites in global: " + ss); + } + } catch (Exception ex) { + log.error("setup failed", ex); + throw ex; + } + } @Override public void asyncMoveTest() { diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0499727d3..968d6eb31 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -159,6 +159,7 @@ public class NodePersistenceImpl implements NodePersistence { private final boolean localGroupsOnly; private URI resourceID; + private final boolean preventNotFound; public NodePersistenceImpl(URI resourceID) { if (resourceID == null) { @@ -189,6 +190,14 @@ public NodePersistenceImpl(URI resourceID) { this.storageNamespace = new Namespace(ns); this.localGroupsOnly = false; + + String pnf = config.getFirstPropertyValue(VaultInitAction.PREVENT_NOT_FOUND_KEY); + if (pnf != null) { + this.preventNotFound = Boolean.valueOf(pnf); + log.debug("Using consistency strategy: " + this.preventNotFound); + } else { + throw new IllegalStateException("invalid config: missing/invalid preventNotFound configuration"); + } } private Subject getRootOwner(MultiValuedProperties mvp, IdentityManager im) { @@ -212,7 +221,7 @@ public TransferGenerator getTransferGenerator() { keyDAO.setConfig(nodeDaoConfig); PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); TokenTool tt = new TokenTool(kp.getPublicKey(), kp.getPrivateKey()); - return new VaultTransferGenerator(this, getArtifactDAO(), tt); + return new VaultTransferGenerator(this, getArtifactDAO(), tt, preventNotFound); } private NodeDAO getDAO() { diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index e291fd651..a9e37e9ec 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -109,6 +109,7 @@ public class VaultInitAction extends InitAction { // config keys private static final String VAULT_KEY = "org.opencadc.vault"; static final String RESOURCE_ID_KEY = VAULT_KEY + ".resourceID"; + static final String PREVENT_NOT_FOUND_KEY = VAULT_KEY + ".consistency.preventNotFound"; static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; @@ -181,6 +182,15 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } + + String pnf = mvp.getFirstPropertyValue(PREVENT_NOT_FOUND_KEY); + sb.append("\n\t" + PREVENT_NOT_FOUND_KEY + ": "); + if (pnf == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } String invSchema = mvp.getFirstPropertyValue(INVENTORY_SCHEMA_KEY); sb.append("\n\t").append(INVENTORY_SCHEMA_KEY).append(": "); diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index ee5ea66c9..b41a702e7 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -115,15 +115,17 @@ public class VaultTransferGenerator implements TransferGenerator { private final VOSpaceAuthorizer authorizer; private final ArtifactDAO artifactDAO; private final TokenTool tokenTool; + private final boolean preventNotFound; private Map siteRules = new HashMap<>(); private Map siteAvailabilities; - public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool) { + public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { this.nodePersistence = nodePersistence; this.authorizer = new VOSpaceAuthorizer(nodePersistence); this.artifactDAO = artifactDAO; this.tokenTool = tokenTool; + this.preventNotFound = preventNotFound; // TODO: get appname from ??? String siteAvailabilitiesKey = "vault" + "-" + StorageSiteAvailabilityCheck.class.getName(); @@ -192,7 +194,7 @@ private List handleDataNode(DataNode node, Transfer trans, Subject s) pg.tokenGen = tokenTool; pg.user = callingUser; pg.requirePreauthAnon = true; - //pg.preventNotFound = true; + pg.preventNotFound = preventNotFound; Transfer artifactTrans = new Transfer(node.storageID, trans.getDirection()); for (Protocol p : trans.getProtocols()) { From b8100363b5fb79cd8b16ba7682e8fde60414d16f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 13:43:52 -0800 Subject: [PATCH 092/186] vault-specific dbrc entry for test setup --- .../src/intTest/java/org/opencadc/vault/TransferTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java index 18a7d0308..37c236bb9 100644 --- a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java @@ -100,7 +100,7 @@ public class TransferTest extends org.opencadc.conformance.vos.TransferTest { } // these are the same as raven intTest - static String SERVER = "INVENTORY_TEST"; + static String SERVER = "VAULT_TEST"; static String DATABASE = "cadctest"; static String SCHEMA = "inventory"; @@ -128,13 +128,13 @@ public void checkGlobal() throws Exception { dao.setConfig(config); Set sites = dao.list(); if (sites.isEmpty()) { - StorageSite ss = new StorageSite(URI.create("ivo://opencadc.org/minoc"), "test-minoc", true, true); + StorageSite ss = new StorageSite(URI.create("ivo://opencadc.org/minoc"), "vault-test-minoc", true, true); dao.put(ss); - log.info("created record in global: " + ss); + log.info("created record in local db: " + ss); } sites = dao.list(); for (StorageSite ss : sites) { - log.info("storage sites in global: " + ss); + log.info("storage sites in local db: " + ss); } } catch (Exception ex) { log.error("setup failed", ex); From 2f409690ab2bef9820d3e22f4c134173c1cb4a19 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 14:09:05 -0800 Subject: [PATCH 093/186] rework to correctly setup recoverableNamespace config --- minoc/src/main/java/org/opencadc/minoc/MinocConfig.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java index 52c1f8130..fff166449 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java @@ -182,6 +182,12 @@ public MinocConfig() { List recov = configProperties.getProperty(RECOVERABLE_NS_KEY); if (recov != null) { for (String s : recov) { + try { + Namespace ns = new Namespace(s); + recoverableNamespaces.add(ns); + } catch (Exception ex) { + throw new IllegalStateException("invalid config: " + RECOVERABLE_NS_KEY + "=" + s + " INVALID", ex); + } } } } From 243ba7e000802024857b797fa2e90eba2e34a62b Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Dec 2023 16:01:46 -0800 Subject: [PATCH 094/186] added NodeDAO(boolean origin) ctor --- .../src/main/java/org/opencadc/vospace/db/NodeDAO.java | 5 +++++ vault/build.gradle | 5 +++-- .../java/org/opencadc/vault/NodePersistenceImpl.java | 10 ++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index acf81b961..58033b796 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -87,6 +87,11 @@ public class NodeDAO extends AbstractDAO { public NodeDAO() { super(true); } + + // needed by vault migration tool: untested + public NodeDAO(boolean origin) { + super(origin); + } @Override public void put(Node val) { diff --git a/vault/build.gradle b/vault/build.gradle index ce2424e23..bba3646dc 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -24,8 +24,9 @@ war { include 'VERSION' } } -description = 'OpenCADC VOSpace server' -def git_url = 'https://github.com/opencadc/vos' + +description = 'OpenCADC vault service' +def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 968d6eb31..d8ce9ef61 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -161,6 +161,10 @@ public class NodePersistenceImpl implements NodePersistence { private URI resourceID; private final boolean preventNotFound; + // possibly temporary hack so migration tool can set this to false and + // preserve lastModified timestamps on nodes + public boolean nodeOrigin = true; + public NodePersistenceImpl(URI resourceID) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); @@ -225,7 +229,7 @@ public TransferGenerator getTransferGenerator() { } private NodeDAO getDAO() { - NodeDAO instance = new NodeDAO(); + NodeDAO instance = new NodeDAO(nodeOrigin); instance.setConfig(nodeDaoConfig); return instance; } @@ -302,10 +306,12 @@ public Node get(ContainerNode parent, String name) throws TransientException { Artifact a = artifactDAO.get(dn.storageID); if (a != null) { DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(a.getContentLastModified()))); + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, a.getContentLength().toString())); // assume MD5 ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTMD5, a.getContentChecksum().getSchemeSpecificPart())); - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(a.getContentLastModified()))); + if (a.contentEncoding != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTENCODING, a.contentEncoding)); } From 17587ca10f8538b38d89d58b63b5afc708fa63da Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 12 Jan 2024 12:52:37 -0800 Subject: [PATCH 095/186] vault: fix date related node properties fix bug in remove artifact node props --- .../opencadc/vault/NodePersistenceImpl.java | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index d8ce9ef61..995270e1a 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -83,6 +83,7 @@ import java.text.DateFormat; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; @@ -110,6 +111,7 @@ import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.db.NodeDAO; +import org.opencadc.vospace.io.NodeWriter; import org.opencadc.vospace.server.LocalServiceURI; import org.opencadc.vospace.server.NodePersistence; import org.opencadc.vospace.server.Views; @@ -134,7 +136,7 @@ public class NodePersistenceImpl implements NodePersistence { VOS.PROPERTY_URI_AVAILABLESPACE, VOS.PROPERTY_URI_CONTENTLENGTH, VOS.PROPERTY_URI_CONTENTMD5, - VOS.PROPERTY_URI_CREATION_DATE, + VOS.PROPERTY_URI_CONTENTDATE, VOS.PROPERTY_URI_DATE, VOS.PROPERTY_URI_CREATOR, VOS.PROPERTY_URI_QUOTA @@ -145,7 +147,7 @@ public class NodePersistenceImpl implements NodePersistence { Arrays.asList( VOS.PROPERTY_URI_CONTENTLENGTH, VOS.PROPERTY_URI_CONTENTMD5, - VOS.PROPERTY_URI_CREATION_DATE, + VOS.PROPERTY_URI_CONTENTDATE, VOS.PROPERTY_URI_DATE, // mutable VOS.PROPERTY_URI_CONTENTENCODING, @@ -305,8 +307,23 @@ public Node get(ContainerNode parent, String name) throws TransientException { ArtifactDAO artifactDAO = getArtifactDAO(); Artifact a = artifactDAO.get(dn.storageID); if (a != null) { - DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(a.getContentLastModified()))); + DateFormat df = NodeWriter.getDateFormat(); + + Date d = ret.getLastModified(); + Date cd = null; + if (ret.getLastModified().before(a.getLastModified())) { + d = a.getLastModified(); + } + if (d.before(a.getContentLastModified())) { + // probably not possible + d = a.getContentLastModified(); + } else { + cd = a.getContentLastModified(); + } + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(d))); + if (cd != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTDATE, df.format(cd))); + } ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, a.getContentLength().toString())); // assume MD5 @@ -500,20 +517,17 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException if (contentType != null || contentEncoding != null) { // optimization ArtifactDAO artifactDAO = getArtifactDAO(); Artifact a = artifactDAO.get(dn.storageID); + log.warn("put: " + contentType + " " + contentEncoding + " -> " + a); if (a != null) { - if (contentType != null) { - if (contentType.isMarkedForDeletion()) { - a.contentType = null; - } else { - a.contentType = contentType.getValue(); - } + if (contentType == null) { + a.contentType = null; + } else { + a.contentType = contentType.getValue(); } - if (contentEncoding != null) { - if (contentEncoding.isMarkedForDeletion()) { - a.contentEncoding = null; - } else { - a.contentEncoding = contentEncoding.getValue(); - } + if (contentEncoding == null) { + a.contentEncoding = null; + } else { + a.contentEncoding = contentEncoding.getValue(); } artifactDAO.put(a); } From 6270d4015b542094b17df8a26454fa3bbaa64341 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 15 Jan 2024 10:42:21 -0800 Subject: [PATCH 096/186] bug fix for immutable property handling --- .../opencadc/vault/NodePersistenceImpl.java | 81 ++++++++++--------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 995270e1a..5f3ce17f7 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -70,7 +70,6 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.HttpPrincipal; import ca.nrc.cadc.auth.IdentityManager; -import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import ca.nrc.cadc.net.TransientException; @@ -124,27 +123,28 @@ public class NodePersistenceImpl implements NodePersistence { private static final Logger log = Logger.getLogger(NodePersistenceImpl.class); - static final Set ADMIN_PROPS = new TreeSet<>( + private static final Set ADMIN_PROPS = new TreeSet<>( Arrays.asList( VOS.PROPERTY_URI_CREATOR, VOS.PROPERTY_URI_QUOTA ) ); - static final Set IMMUTABLE_PROPS = new TreeSet<>( + private static final Set IMMUTABLE_PROPS = new TreeSet<>( Arrays.asList( VOS.PROPERTY_URI_AVAILABLESPACE, VOS.PROPERTY_URI_CONTENTLENGTH, VOS.PROPERTY_URI_CONTENTMD5, VOS.PROPERTY_URI_CONTENTDATE, - VOS.PROPERTY_URI_DATE, VOS.PROPERTY_URI_CREATOR, + VOS.PROPERTY_URI_DATE, VOS.PROPERTY_URI_QUOTA ) ); private static final Set ARTIFACT_PROPS = new TreeSet<>( Arrays.asList( + // immutable VOS.PROPERTY_URI_CONTENTLENGTH, VOS.PROPERTY_URI_CONTENTMD5, VOS.PROPERTY_URI_CONTENTDATE, @@ -482,6 +482,7 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException if (node instanceof DataNode) { DataNode dn = (DataNode) node; + boolean knownNoArtifact = false; if (dn.storageID == null) { // new data node? if lastModified is assigned, this looks sketchy if (dn.getLastModified() != null) { @@ -493,44 +494,44 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException // once someone puts the file to minoc, so Node.storageID == Artifact.uri // but the artifact may or may not exist dn.storageID = generateStorageID(); + knownNoArtifact = true; + } + + // need to remove all artifact props from the node.getProperties() + // and use artifactDAO to set the mutable ones + NodeProperty contentType = null; + NodeProperty contentEncoding = null; + Iterator i = dn.getProperties().iterator(); + while (i.hasNext()) { + NodeProperty np = i.next(); + if (VOS.PROPERTY_URI_TYPE.equals(np.getKey())) { + contentType = np; + } else if (VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { + contentEncoding = np; + } - } else { - // update existing data node - // need to remove all artifact props from the node.getProperties() - // and use artifactDAO to set the mutable ones - NodeProperty contentType = null; - NodeProperty contentEncoding = null; - - Iterator i = dn.getProperties().iterator(); - while (i.hasNext()) { - NodeProperty np = i.next(); - if (VOS.PROPERTY_URI_TYPE.equals(np.getKey())) { - contentType = np; - } else if (VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { - contentEncoding = np; - } - - if (ARTIFACT_PROPS.contains(np.getKey())) { - i.remove(); - } + if (ARTIFACT_PROPS.contains(np.getKey())) { + i.remove(); } - if (contentType != null || contentEncoding != null) { // optimization - ArtifactDAO artifactDAO = getArtifactDAO(); - Artifact a = artifactDAO.get(dn.storageID); - log.warn("put: " + contentType + " " + contentEncoding + " -> " + a); - if (a != null) { - if (contentType == null) { - a.contentType = null; - } else { - a.contentType = contentType.getValue(); - } - if (contentEncoding == null) { - a.contentEncoding = null; - } else { - a.contentEncoding = contentEncoding.getValue(); - } - artifactDAO.put(a); + } + // optimization: avoid query when we know artifact doesn't exist + // and: avoid query if we don't need to update it + if (!knownNoArtifact && (contentType != null || contentEncoding != null)) { + ArtifactDAO artifactDAO = getArtifactDAO(); + Artifact a = artifactDAO.get(dn.storageID); + log.warn("put: " + contentType + " " + contentEncoding + " -> " + a); + if (a != null) { + if (contentType == null) { + a.contentType = null; + } else { + a.contentType = contentType.getValue(); + } + if (contentEncoding == null) { + a.contentEncoding = null; + } else { + a.contentEncoding = contentEncoding.getValue(); } + artifactDAO.put(a); } } } From d6e9d12a30ede41fbe36e826ab8e9382bc8f038f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 16 Jan 2024 09:46:52 -0800 Subject: [PATCH 097/186] vault :supress unchecked cast warning --- vault/src/main/java/org/opencadc/vault/ServiceAvailability.java | 2 +- .../main/java/org/opencadc/vault/VaultTransferGenerator.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index 53e415636..0b0005a42 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index b41a702e7..b75d9ffd8 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -120,6 +120,7 @@ public class VaultTransferGenerator implements TransferGenerator { private Map siteRules = new HashMap<>(); private Map siteAvailabilities; + @SuppressWarnings("unchecked") public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { this.nodePersistence = nodePersistence; this.authorizer = new VOSpaceAuthorizer(nodePersistence); From 8ffe5e10851b0752e1ecd11b5f8c762067aa298c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 18 Jan 2024 10:24:57 -0800 Subject: [PATCH 098/186] vault: add support for separate nodes and inventory pools added inventory table init --- vault/README.md | 24 +++ vault/VERSION | 2 +- .../opencadc/vault/NodePersistenceImpl.java | 165 ++++++++++++------ .../org/opencadc/vault/VaultInitAction.java | 67 +++++-- vault/src/main/webapp/META-INF/context.xml | 12 ++ 5 files changed, 196 insertions(+), 74 deletions(-) diff --git a/vault/README.md b/vault/README.md index b2888f648..081fbefa7 100644 --- a/vault/README.md +++ b/vault/README.md @@ -40,6 +40,12 @@ org.opencadc.vault.nodes.username={username for vospace pool} org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} +org.opencadc.vault.inventory.maxActive={max connections for inventory pool} +# optional: config for separate inventory pool +org.opencadc.vault.inventory.username={username for inventory pool} +org.opencadc.vault.inventory.password={password for inventory pool} +org.opencadc.vault.inventory.url=jdbc:postgresql://{server}/{database} + org.opencadc.vault.uws.maxActive={max connections for uws pool} org.opencadc.vault.uws.username={username for uws pool} org.opencadc.vault.uws.password={password for uws pool} @@ -50,6 +56,15 @@ all the content (insert, update, delete). The database is specified in the JDBC in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. +The _inventory_ account owns and manages (create, alter, drop) inventory database objects and manages +all the content (update and delete Artifact, insert DeletedArtifactEvent). The database is specified +in the JDBC URL and the schema name is specified in the minoc.properties (below). Failure to connect or +initialize the database will show up in logs and in the VOSI-availability output. The _inventory_ content +may be in the same database as the _nodes_, in a different database in the same server, or in a different +server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. Note: it is a good +idea to set `maxActive` to a valid integer (e.g. 0) when using a single pool; this avoids an ugly but +meaningless stack trace in the logs at startup. + The _uws_ account owns and manages (create, alter, drop) uws database objects in the `uws` schema and manages all the content (insert, update, delete). The database is specified in the JDBC URLFailure to connect or initialize the database will show up in logs and in the VOSI-availability output. @@ -70,6 +85,7 @@ org.opencadc.vault.consistency.preventNotFound=true|false # vault database settings org.opencadc.vault.inventory.schema = {inventory schema name} org.opencadc.vault.vospace.schema = {vospace schema name} +org.opencadc.vault.singlePool = {true|false} # root container nodes org.opencadc.vault.root.owner = {owner of root node} @@ -92,6 +108,14 @@ to configuration limitations in luskan. The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must currently be in the same database. TODO: augment config to support separate inventory and vospace pools. +The _singlePool_ key configures `vault` to use a single pool (the _nodes_ pool) for both vospace and inventory +operations. The inventory and vospace content must be in the same database for this to work. When configured +to use a single pool, delete node operations can delete a DataNode and the associated Artifact and create the +DeletedArtifactEvent in a single transaction. When configured to use separate pools, the delete Artifact and create +DeletedArtifactEvent are done in a separate transaction and if that fails the Artifact will be left behind and +orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` allows +the content to be stored in two separate databases or servers. + The _root.owner_ owns the root node and has full read and write permission in the root container, so it can create and delete container nodes at the root and assign container node properties that are normally read-only to normal users: owner, quota, etc. This must be set to the username of the admin. diff --git a/vault/VERSION b/vault/VERSION index 71963f952..9d05f397f 100644 --- a/vault/VERSION +++ b/vault/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.1.0 +VER=0.2.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 5f3ce17f7..15f3e67e6 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -155,7 +155,10 @@ public class NodePersistenceImpl implements NodePersistence { ) ); - private final Map nodeDaoConfig = new TreeMap<>(); + private final Map nodeDaoConfig; + private final Map invDaoConfig; + private final boolean singlePool; + private final ContainerNode root; private final Namespace storageNamespace; @@ -173,13 +176,9 @@ public NodePersistenceImpl(URI resourceID) { } this.resourceID = resourceID; MultiValuedProperties config = VaultInitAction.getConfig(); - String dataSourceName = VaultInitAction.JNDI_DATASOURCE; - String inventorySchema = config.getFirstPropertyValue(VaultInitAction.INVENTORY_SCHEMA_KEY); - String vospaceSchema = config.getFirstPropertyValue(VaultInitAction.VOSPACE_SCHEMA_KEY); - nodeDaoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); - nodeDaoConfig.put("jndiDataSourceName", dataSourceName); - nodeDaoConfig.put("schema", inventorySchema); - nodeDaoConfig.put("vosSchema", vospaceSchema); + this.nodeDaoConfig = VaultInitAction.getDaoConfig(config); + this.invDaoConfig = VaultInitAction.getInvConfig(config); + this.singlePool = nodeDaoConfig.get("jndiDataSourceName").equals(invDaoConfig.get("jndiDataSourceName")); // root node IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); @@ -480,9 +479,28 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException } } + NodeProperty contentType = null; + NodeProperty contentEncoding = null; + // need to remove all artifact props from the node.getProperties() + // and use artifactDAO to set the mutable ones + Iterator i = node.getProperties().iterator(); + while (i.hasNext()) { + NodeProperty np = i.next(); + if (VOS.PROPERTY_URI_TYPE.equals(np.getKey())) { + contentType = np; + } else if (VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { + contentEncoding = np; + } + + if (ARTIFACT_PROPS.contains(np.getKey())) { + i.remove(); + } + } + + ArtifactDAO artifactDAO = null; + Artifact a = null; if (node instanceof DataNode) { DataNode dn = (DataNode) node; - boolean knownNoArtifact = false; if (dn.storageID == null) { // new data node? if lastModified is assigned, this looks sketchy if (dn.getLastModified() != null) { @@ -494,50 +512,45 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException // once someone puts the file to minoc, so Node.storageID == Artifact.uri // but the artifact may or may not exist dn.storageID = generateStorageID(); - knownNoArtifact = true; - } - - // need to remove all artifact props from the node.getProperties() - // and use artifactDAO to set the mutable ones - NodeProperty contentType = null; - NodeProperty contentEncoding = null; - Iterator i = dn.getProperties().iterator(); - while (i.hasNext()) { - NodeProperty np = i.next(); - if (VOS.PROPERTY_URI_TYPE.equals(np.getKey())) { - contentType = np; - } else if (VOS.PROPERTY_URI_CONTENTENCODING.equals(np.getKey())) { - contentEncoding = np; - } - - if (ARTIFACT_PROPS.contains(np.getKey())) { - i.remove(); - } - } - // optimization: avoid query when we know artifact doesn't exist - // and: avoid query if we don't need to update it - if (!knownNoArtifact && (contentType != null || contentEncoding != null)) { - ArtifactDAO artifactDAO = getArtifactDAO(); - Artifact a = artifactDAO.get(dn.storageID); - log.warn("put: " + contentType + " " + contentEncoding + " -> " + a); - if (a != null) { - if (contentType == null) { - a.contentType = null; - } else { - a.contentType = contentType.getValue(); - } - if (contentEncoding == null) { - a.contentEncoding = null; - } else { - a.contentEncoding = contentEncoding.getValue(); - } - artifactDAO.put(a); + } else { + if (contentType != null || contentEncoding != null) { + // update possibly required + artifactDAO = getArtifactDAO(); + a = artifactDAO.get(dn.storageID); + } else { + log.debug("no artifact props to update - skipping ArtifactDAO.get"); } } } + + boolean useTxn = singlePool && a != null; // TODO + // update node NodeDAO dao = getDAO(); dao.put(node); + + // update artifact after node + if (a != null) { + if (contentType == null || contentType.isMarkedForDeletion()) { + a.contentType = null; + } else { + a.contentType = contentType.getValue(); + } + if (contentEncoding == null || contentEncoding.isMarkedForDeletion()) { + a.contentEncoding = null; + } else { + a.contentEncoding = contentEncoding.getValue(); + } + artifactDAO.put(a); + + // re-add node props + if (contentType != null && !contentType.isMarkedForDeletion()) { + node.getProperties().add(contentType); + } + if (contentEncoding != null && !contentEncoding.isMarkedForDeletion()) { + node.getProperties().add(contentEncoding); + } + } return node; } @@ -583,12 +596,21 @@ public void delete(Node node) throws TransientException { throw new IllegalArgumentException("arg cannot be null: node"); } + Artifact a = null; final NodeDAO dao = getDAO(); final ArtifactDAO artifactDAO = getArtifactDAO(); TransactionManager txn = dao.getTransactionManager(); - + TransactionManager atxn = null; try { - log.debug("starting transaction"); + if (node instanceof DataNode) { + DataNode dn = (DataNode) node; + a = artifactDAO.get(dn.storageID); + } + if (a != null && !singlePool) { + atxn = artifactDAO.getTransactionManager(); + } + + log.debug("starting node transaction"); txn.startTransaction(); log.debug("start txn: OK"); @@ -614,30 +636,52 @@ public void delete(Node node) throws TransientException { } } // else: LinkNode can always be deleted - if (storageID != null) { - Artifact a = artifactDAO.get(storageID); - if (a != null) { - DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); - DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); - daeDAO.put(dae); - artifactDAO.delete(a.getID()); - } + if (singlePool && a != null) { + // inventory ops inside main txn + DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); + DeletedArtifactEvent dae = new DeletedArtifactEvent(a.getID()); + daeDAO.put(dae); + artifactDAO.delete(a.getID()); } + // TODO: need DeletedNodeDAO to create DeletedNodeEvent dao.delete(node.getID()); } else { log.debug("failed to lock node " + node.getID() + " - assume deleted by another process"); } - + log.debug("commit txn..."); txn.commitTransaction(); log.debug("commit txn: OK"); + + if (!singlePool && a != null) { + log.warn("starting artifact transaction"); + atxn.startTransaction(); + log.debug("start txn: OK"); + + Artifact alock = artifactDAO.lock(a); + if (alock != null) { + DeletedArtifactEventDAO daeDAO = new DeletedArtifactEventDAO(artifactDAO); + DeletedArtifactEvent dae = new DeletedArtifactEvent(alock.getID()); + daeDAO.put(dae); + artifactDAO.delete(alock.getID()); + } + log.debug("commit artifact txn..."); + atxn.commitTransaction(); + atxn = null; + log.warn("commit artifact txn: OK"); + } } catch (Exception ex) { if (txn.isOpen()) { log.error("failed to delete " + node.getID() + " aka " + node.getName(), ex); txn.rollbackTransaction(); log.debug("rollback txn: OK"); } + if (atxn != null && atxn.isOpen()) { + log.error("failed to delete " + a.getID() + " aka " + a.getURI(), ex); + atxn.rollbackTransaction(); + log.debug("rollback artifact txn: OK"); + } throw ex; } finally { if (txn.isOpen()) { @@ -645,6 +689,11 @@ public void delete(Node node) throws TransientException { txn.rollbackTransaction(); log.error("rollback txn: OK"); } + if (atxn != null && atxn.isOpen()) { + log.error("BUG - open artifact transaction in finally"); + atxn.rollbackTransaction(); + log.error("rollback artifact txn: OK"); + } } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index a9e37e9ec..7f8a53bf4 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -88,6 +88,7 @@ import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; +import org.opencadc.inventory.db.version.InitDatabase; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; @@ -103,7 +104,8 @@ public class VaultInitAction extends InitAction { static String KEY_PAIR_NAME = "vault-preauth-keys"; - static final String JNDI_DATASOURCE = "jdbc/nodes"; // context.xml + static final String JNDI_VOS_DATASOURCE = "jdbc/nodes"; // context.xml + static final String JNDI_INV_DATASOURCE = "jdbc/inventory"; // context.xml static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml // config keys @@ -112,6 +114,7 @@ public class VaultInitAction extends InitAction { static final String PREVENT_NOT_FOUND_KEY = VAULT_KEY + ".consistency.preventNotFound"; static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; + static final String SINGLE_POOL_KEY = VAULT_KEY + ".singlePool"; static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? @@ -120,7 +123,8 @@ public class VaultInitAction extends InitAction { MultiValuedProperties props; private URI resourceID; private Namespace storageNamespace; - private Map daoConfig; + private Map vosDaoConfig; + private Map invDaoConfig; private String jndiNodePersistence; private String jndiPreauthKeys; @@ -209,6 +213,15 @@ static MultiValuedProperties getConfig() { } else { sb.append("OK"); } + + String sp = mvp.getFirstPropertyValue(SINGLE_POOL_KEY); + sb.append("\n\t").append(SINGLE_POOL_KEY).append(": "); + if (sp == null) { + sb.append("MISSING"); + ok = false; + } else { + sb.append("OK"); + } String ns = mvp.getFirstPropertyValue(STORAGE_NAMESPACE_KEY); sb.append("\n\t").append(STORAGE_NAMESPACE_KEY).append(": "); @@ -227,14 +240,25 @@ static MultiValuedProperties getConfig() { } static Map getDaoConfig(MultiValuedProperties props) { - Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now - ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_DATASOURCE); + ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_VOS_DATASOURCE); ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); return ret; } + + static Map getInvConfig(MultiValuedProperties props) { + boolean usp = Boolean.parseBoolean(props.getFirstPropertyValue(SINGLE_POOL_KEY)); + if (usp) { + return getDaoConfig(props); + } + Map ret = new TreeMap<>(); + ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now + ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_INV_DATASOURCE); + ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); + return ret; + } private void initConfig() { log.info("initConfig: START"); @@ -244,7 +268,8 @@ private void initConfig() { try { this.resourceID = new URI(rid); this.storageNamespace = new Namespace(ns); - this.daoConfig = getDaoConfig(props); + this.vosDaoConfig = getDaoConfig(props); + this.invDaoConfig = getInvConfig(props); log.info("initConfig: OK"); } catch (URISyntaxException ex) { throw new IllegalStateException("invalid config: " + RESOURCE_ID_KEY + " must be a valid URI"); @@ -252,33 +277,45 @@ private void initConfig() { } private void initDatabase() { - log.info("initDatabase: START"); try { - DataSource ds = DBUtil.findJNDIDataSource(JNDI_DATASOURCE); - String schema = (String) daoConfig.get("vosSchema"); + String dsname = (String) vosDaoConfig.get("jndiDataSourceName"); + String schema = (String) vosDaoConfig.get("vosSchema"); + log.info("initDatabase: " + dsname + " " + schema + " START"); + DataSource ds = DBUtil.findJNDIDataSource(dsname); InitDatabaseVOS init = new InitDatabaseVOS(ds, null, schema); init.doInit(); - log.info("initDatabase: " + JNDI_DATASOURCE + " " + schema + " OK"); + log.info("initDatabase: " + dsname + " " + schema + " OK"); + } catch (Exception ex) { + throw new IllegalStateException("check/init vospace database failed", ex); + } + + try { + String dsname = (String) invDaoConfig.get("jndiDataSourceName"); + String schema = (String) invDaoConfig.get("schema"); + log.info("initDatabase: " + dsname + " " + schema + " START"); + DataSource ds = DBUtil.findJNDIDataSource(dsname); + InitDatabase init = new InitDatabase(ds, null, schema); + init.doInit(); + log.info("initDatabase: " + dsname + " " + schema + " OK"); } catch (Exception ex) { - throw new IllegalStateException("check/init database failed", ex); + throw new IllegalStateException("check/init inventory database failed", ex); } } private void initUWSDatabase() { - log.info("initUWSDatabase: START"); try { - // Init UWS database + log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws START"); DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws"); uwsi.doInit(); log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws OK"); - } catch (Exception ex) { throw new RuntimeException("check/init uws database failed", ex); } } private void initNodePersistence() { + log.info("initNodePersistence: START"); jndiNodePersistence = appName + "-" + NodePersistence.class.getName(); try { Context ctx = new InitialContext(); @@ -290,7 +327,7 @@ private void initNodePersistence() { NodePersistence npi = new NodePersistenceImpl(resourceID); ctx.bind(jndiNodePersistence, npi); - log.info("created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); + log.info("initNodePersistence: created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); } catch (Exception ex) { log.error("Failed to create JNDI Key " + jndiNodePersistence, ex); } @@ -301,7 +338,7 @@ private void initKeyPair() { jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); try { PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); - dao.setConfig(daoConfig); + dao.setConfig(vosDaoConfig); PreauthKeyPair keys = dao.get(KEY_PAIR_NAME); if (keys == null) { KeyPair kp = RsaSignatureGenerator.getKeyPair(4096); diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index e5775723b..ebd23f61c 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -15,6 +15,18 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> + + Date: Thu, 18 Jan 2024 10:43:23 -0800 Subject: [PATCH 099/186] log.warn -> debug --- .../src/main/java/org/opencadc/vault/NodePersistenceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 15f3e67e6..b121f23a5 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -655,7 +655,7 @@ public void delete(Node node) throws TransientException { log.debug("commit txn: OK"); if (!singlePool && a != null) { - log.warn("starting artifact transaction"); + log.debug("starting artifact transaction"); atxn.startTransaction(); log.debug("start txn: OK"); @@ -669,7 +669,7 @@ public void delete(Node node) throws TransientException { log.debug("commit artifact txn..."); atxn.commitTransaction(); atxn = null; - log.warn("commit artifact txn: OK"); + log.debug("commit artifact txn: OK"); } } catch (Exception ex) { if (txn.isOpen()) { From 5229b109d7b52f496b32300139245562e32a428d Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 18 Jan 2024 11:27:38 -0800 Subject: [PATCH 100/186] bug fix: add explicit config map for PreauthKeyPairDAO --- .../org/opencadc/vault/NodePersistenceImpl.java | 4 +++- .../java/org/opencadc/vault/VaultInitAction.java | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index b121f23a5..0e352fcc1 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -157,6 +157,7 @@ public class NodePersistenceImpl implements NodePersistence { private final Map nodeDaoConfig; private final Map invDaoConfig; + private final Map kpDaoConfig; private final boolean singlePool; private final ContainerNode root; @@ -178,6 +179,7 @@ public NodePersistenceImpl(URI resourceID) { MultiValuedProperties config = VaultInitAction.getConfig(); this.nodeDaoConfig = VaultInitAction.getDaoConfig(config); this.invDaoConfig = VaultInitAction.getInvConfig(config); + this.kpDaoConfig = VaultInitAction.getKeyPairConfig(config); this.singlePool = nodeDaoConfig.get("jndiDataSourceName").equals(invDaoConfig.get("jndiDataSourceName")); // root node @@ -223,7 +225,7 @@ public Views getViews() { @Override public TransferGenerator getTransferGenerator() { PreauthKeyPairDAO keyDAO = new PreauthKeyPairDAO(); - keyDAO.setConfig(nodeDaoConfig); + keyDAO.setConfig(kpDaoConfig); PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); TokenTool tt = new TokenTool(kp.getPublicKey(), kp.getPrivateKey()); return new VaultTransferGenerator(this, getArtifactDAO(), tt, preventNotFound); diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 7f8a53bf4..4b019e092 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -259,6 +259,14 @@ static Map getInvConfig(MultiValuedProperties props) { ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); return ret; } + + static Map getKeyPairConfig(MultiValuedProperties props) { + Map ret = new TreeMap<>(); + ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now + ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_VOS_DATASOURCE); + ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); + return ret; + } private void initConfig() { log.info("initConfig: START"); @@ -335,10 +343,10 @@ private void initNodePersistence() { private void initKeyPair() { log.info("initKeyPair: START"); - jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); + //jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); try { PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); - dao.setConfig(vosDaoConfig); + dao.setConfig(getKeyPairConfig(props)); PreauthKeyPair keys = dao.get(KEY_PAIR_NAME); if (keys == null) { KeyPair kp = RsaSignatureGenerator.getKeyPair(4096); @@ -359,6 +367,7 @@ private void initKeyPair() { } else { log.info("initKeyPair: re-use existing keys - OK"); } + /* Context ctx = new InitialContext(); try { ctx.unbind(jndiPreauthKeys); @@ -370,6 +379,7 @@ private void initKeyPair() { Object o = ctx.lookup(jndiPreauthKeys); log.info("checking... found: " + jndiPreauthKeys + " = " + o + " in " + ctx); + */ } catch (Exception ex) { throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); } From 2f4d36da45c9cf732d8eea0ed63eeb36e8d1b560 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 19 Jan 2024 10:40:22 -0800 Subject: [PATCH 101/186] minor cleanup in VaultInitAction --- .../java/org/opencadc/vault/VaultInitAction.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 4b019e092..2f8625d36 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -242,9 +242,10 @@ static MultiValuedProperties getConfig() { static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now - ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_VOS_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); - ret.put("vosSchema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); + ret.put("jndiDataSourceName", VaultInitAction.JNDI_VOS_DATASOURCE); + // unused, but inventory "schema" required by cadc-inventory-db + ret.put("schema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("vosSchema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); return ret; } @@ -255,16 +256,16 @@ static Map getInvConfig(MultiValuedProperties props) { } Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now - ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_INV_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.INVENTORY_SCHEMA_KEY)); + ret.put("jndiDataSourceName", JNDI_INV_DATASOURCE); + ret.put("schema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); return ret; } static Map getKeyPairConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now - ret.put("jndiDataSourceName", org.opencadc.vault.VaultInitAction.JNDI_VOS_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(org.opencadc.vault.VaultInitAction.VOSPACE_SCHEMA_KEY)); + ret.put("jndiDataSourceName", JNDI_VOS_DATASOURCE); + ret.put("schema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); return ret; } From 179ff9a5e59c3603e03e9f7072a4f0bff62b124a Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 22 Jan 2024 13:56:43 -0800 Subject: [PATCH 102/186] raven: made preauth key usage optional --- raven/README.md | 17 ++++++++++------- .../java/org/opencadc/raven/ArtifactAction.java | 2 +- .../org/opencadc/raven/RavenInitAction.java | 9 --------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/raven/README.md b/raven/README.md index c121b4191..bd6948c33 100644 --- a/raven/README.md +++ b/raven/README.md @@ -42,29 +42,32 @@ org.opencadc.raven.inventory.schema={schema} # consistency settings org.opencadc.raven.consistency.preventNotFound=true|false - -# url signing key usage -org.opencadc.raven.keys.preauth={true|false} ``` The _preventNotFound_ key can be used to configure `raven` to prevent artifact-not-found errors that might result due to the eventual consistency nature of the system by directly checking for the artifact at _all known_ sites. This feature introduces an overhead for the genuine not-found cases. -The _keys.preauth_ key configures `raven` to use URL-signing. When enabled, `raven` can generate a signed token -and embeds it into the URL; `minoc` services can validate the token and grant access without further permission -checks. With transfer negotiation, the signed URL gets added as an additional "anonymous" URL. + The following optional keys configure raven to use external service(s) to obtain grant information in order -to perform authorization checks: +to perform authorization checks and generate signed URLs: ``` org.opencadc.raven.readGrantProvider={resourceID of a permission granting service} org.opencadc.raven.writeGrantProvider={resourceID of a permission granting service} + +# url signing key usage +org.opencadc.raven.keys.preauth={true|false} ``` The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for operations. Multiple values of the permission granting service resourceID(s) may be provided by including multiple property settings. All services will be consulted but a single positive result is sufficient to grant permission for an action. +The _keys.preauth_ key (default: false) configures `raven` to use URL-signing. When enabled, `raven` can generate a signed token +and embed it into the URL; `minoc` services that are configured to trust a `raven` service will download the public key and can +validate the token and grant access without further permission checks. With transfer negotiation, the signed URL gets added as +an additional "anonymous" URL. + The following optional keys configure raven to prioritize sites returned in transfer negotiation, with higher priority sites first in the list of transfer URL's. Multiple values of _namespace_ may be specified for a single _resourceID_. The _namespace_ value(s) must end with a colon (:) or slash (/) so one namespace cannot accidentally match (be a diff --git a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java index a4c4cf98c..b49032c64 100644 --- a/raven/src/main/java/org/opencadc/raven/ArtifactAction.java +++ b/raven/src/main/java/org/opencadc/raven/ArtifactAction.java @@ -197,7 +197,7 @@ protected ArtifactAction() { this.preauthKeys = Boolean.valueOf(pak); log.debug("Using preauth keys: " + this.preauthKeys); } else { - throw new IllegalStateException("invalid config: missing keys.preauth configuration"); + this.preauthKeys = false; } } diff --git a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java index 95c6f0c05..aaee48675 100644 --- a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java +++ b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java @@ -330,15 +330,6 @@ static MultiValuedProperties getConfig() { sb.append("OK"); } - String preauthKeys = mvp.getFirstPropertyValue(RavenInitAction.PREAUTH_KEY); - sb.append("\n\t").append(RavenInitAction.PREAUTH_KEY).append(": "); - if (preauthKeys == null) { - sb.append("MISSING"); - ok = false; - } else { - sb.append("OK"); - } - if (!ok) { throw new IllegalStateException(sb.toString()); } From 61146cc6b85d2a7225f8c88031ee1c2634df039d Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 22 Jan 2024 14:19:52 -0800 Subject: [PATCH 103/186] fix for disabled preauth key use --- .../inventory/transfer/GetKeyAction.java | 10 +++++- .../transfer/ProtocolsGenerator.java | 34 ++++++++++--------- .../org/opencadc/raven/HeadFilesAction.java | 18 ++++++---- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java index e3396d87b..d42c13355 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java @@ -70,6 +70,7 @@ import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.rest.RestAction; import java.io.OutputStream; +import java.io.PrintWriter; import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingException; @@ -111,7 +112,14 @@ public void doAction() throws Exception { ostream.flush(); } } catch (NamingException ex) { - throw new RuntimeException("BUG: failed to find keys via JNDI", ex); + syncOutput.setHeader("content-type", "test/plain"); + syncOutput.setCode(404); + try (OutputStream ostream = syncOutput.getOutputStream()) { + PrintWriter w = new PrintWriter(ostream); + w.println("not found: key signing disabled"); + w.flush(); + w.close(); + } } } } diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index 5cf4a5329..0307e7c1d 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -499,29 +499,31 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT if (sec == null) { sec = Standards.SECURITY_METHOD_ANON; } - boolean incToken = Standards.SECURITY_METHOD_ANON.equals(sec); + boolean anon = Standards.SECURITY_METHOD_ANON.equals(sec); Interface iface = filesCap.findInterface(sec); log.debug("PUT: " + storageSite + " proto: " + proto + " iface: " + iface); if (iface != null) { URL baseURL = iface.getAccessURL().getURL(); //log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); if (protocolCompat(proto, baseURL)) { - - StringBuilder sb = new StringBuilder(); - sb.append(baseURL.toExternalForm()).append("/"); - if (authToken != null && incToken) { - sb.append(authToken).append("/"); - } - sb.append(artifactURI.toASCIIString()); - Protocol p = new Protocol(proto.getUri()); - if (transfer.version == VOS.VOSPACE_21) { - p.setSecurityMethod(proto.getSecurityMethod()); + // // no plain anon URL for put: !anon or anon+token + boolean gen = (!anon || (anon && authToken != null)); + if (gen) { + StringBuilder sb = new StringBuilder(); + sb.append(baseURL.toExternalForm()).append("/"); + if (authToken != null && anon) { + sb.append(authToken).append("/"); + } + sb.append(artifactURI.toASCIIString()); + Protocol p = new Protocol(proto.getUri()); + if (transfer.version == VOS.VOSPACE_21) { + p.setSecurityMethod(proto.getSecurityMethod()); + } + p.setEndpoint(sb.toString()); + protos.add(p); + log.debug("added: " + p); } - p.setEndpoint(sb.toString()); - protos.add(p); - log.debug("added: " + p); - - // no plain anon URL for put + } else { log.debug("PUT: " + storageSite + "PUT: reject protocol: " + proto + " reason: no compatible URL protocol"); diff --git a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java index a9dcfc657..4b3312eb2 100644 --- a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java +++ b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java @@ -72,9 +72,11 @@ import ca.nrc.cadc.rest.SyncOutput; import java.net.HttpURLConnection; import java.net.URL; +import java.util.Set; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.inventory.StorageSite; import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.transfer.ProtocolsGenerator; import org.opencadc.permissions.ReadGrant; @@ -108,8 +110,10 @@ public void doAction() throws Exception { log.debug("Starting HEAD action for " + artifactURI.toASCIIString()); Artifact artifact = artifactDAO.get(artifactURI); - if (artifact == null) { - if (this.preventNotFound) { + if (artifact == null && preventNotFound) { + StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO); + Set sites = storageSiteDAO.list(); + if (!sites.isEmpty()) { // check known storage sites ProtocolsGenerator pg = new ProtocolsGenerator( this.artifactDAO, this.siteAvailabilities, this.siteRules); @@ -117,14 +121,16 @@ public void doAction() throws Exception { pg.user = this.user; pg.preventNotFound = this.preventNotFound; pg.storageResolver = this.storageResolver; - StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO); + Transfer transfer = new Transfer(artifactURI, Direction.pullFromVoSpace); Protocol proto = new Protocol(VOS.PROTOCOL_HTTPS_GET); proto.setSecurityMethod(Standards.SECURITY_METHOD_ANON); transfer.getProtocols().add(proto); - // TODO: tokenGen is optional so this can fail - String authToken = tokenGen.generateToken(artifactURI, ReadGrant.class, user); - artifact = pg.getUnsyncedArtifact(artifactURI, transfer, storageSiteDAO.list(), authToken); + String authToken = null; + if (tokenGen != null) { + authToken = tokenGen.generateToken(artifactURI, ReadGrant.class, user); + } + artifact = pg.getUnsyncedArtifact(artifactURI, transfer, sites, authToken); } } From 07380e1c19d2bff9107961a6a0cfafde374737bd Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 23 Jan 2024 14:42:03 -0800 Subject: [PATCH 104/186] minoc: support for filename override in the URL path using colon separator --- .../java/org/opencadc/minoc/BasicOpsTest.java | 112 ++++++++++++++++++ .../org/opencadc/minoc/ArtifactAction.java | 56 +++++---- .../java/org/opencadc/minoc/GetAction.java | 4 +- .../java/org/opencadc/minoc/HeadAction.java | 9 +- .../java/org/opencadc/minoc/PostAction.java | 2 +- .../opencadc/minoc/ArtifactActionTest.java | 19 +-- 6 files changed, 160 insertions(+), 42 deletions(-) diff --git a/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java b/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java index d13de5e76..36a6552fd 100644 --- a/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java +++ b/minoc/src/intTest/java/org/opencadc/minoc/BasicOpsTest.java @@ -146,10 +146,12 @@ public void testPutGetUpdateHeadDelete() { long contentLength = get.getContentLength(); String contentType = get.getContentType(); String contentEncoding = get.getContentEncoding(); + String contentDisposition = get.getResponseHeader("content-disposition"); Assert.assertEquals(computeChecksumURI(data), checksumURI); Assert.assertEquals(data.length, contentLength); Assert.assertEquals(type, contentType); Assert.assertEquals(encoding, contentEncoding); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("file.txt")); Date lastModified = get.getLastModified(); Assert.assertNotNull(lastModified); @@ -181,10 +183,12 @@ public void testPutGetUpdateHeadDelete() { contentLength = head.getContentLength(); contentType = head.getContentType(); contentEncoding = head.getContentEncoding(); + contentDisposition = head.getResponseHeader("content-disposition"); Assert.assertEquals(computeChecksumURI(data), checksumURI); Assert.assertEquals(data.length, contentLength); Assert.assertEquals(newType, contentType); Assert.assertEquals(newEncoding, contentEncoding); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("file.txt")); lastModified = head.getLastModified(); Assert.assertNotNull(lastModified); @@ -331,6 +335,114 @@ public void testGetRanges() { } } + @Test + public void testFilenameOverride() { + try { + URI artifactURI = URI.create("cadc:TEST/testFilenameOverride.txt"); + URL artifactURL = new URL(filesURL + "/" + artifactURI.toString()); + + String content = "abcdefghijklmnopqrstuvwxyz"; + String encoding = "test-encoding"; + String type = "text/plain"; + byte[] data = content.getBytes(); + URI expectedChecksum = computeChecksumURI(data); + + // put: no length or checksum + InputStream in = new ByteArrayInputStream(data); + HttpUpload put = new HttpUpload(in, artifactURL); + put.setRequestProperty(HttpTransfer.CONTENT_TYPE, type); + put.setRequestProperty(HttpTransfer.CONTENT_ENCODING, encoding); + put.setDigest(expectedChecksum); + + Subject.doAs(userSubject, new RunnableAction(put)); + log.info("put: " + put.getResponseCode() + " " + put.getThrowable()); + log.info("headers: " + put.getResponseHeader("content-length") + " " + put.getResponseHeader("digest")); + Assert.assertNull(put.getThrowable()); + Assert.assertEquals("Created", 201, put.getResponseCode()); + + // head + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet head = new HttpGet(artifactURL, bos); + head.setHeadOnly(true); + log.info("head: " + artifactURL.toExternalForm()); + Subject.doAs(userSubject, new RunnableAction(head)); + log.info("head: " + head.getResponseCode() + " " + head.getThrowable()); + log.info("headers: " + head.getResponseHeader("content-length") + " " + head.getResponseHeader("digest")); + log.warn("head output: " + bos.toString()); + Assert.assertNull(head.getThrowable()); + URI checksumURI = head.getDigest(); + long contentLength = head.getContentLength(); + String contentType = head.getContentType(); + String contentEncoding = head.getContentEncoding(); + String contentDisposition = head.getResponseHeader("content-disposition"); + Assert.assertEquals(computeChecksumURI(data), checksumURI); + Assert.assertEquals(data.length, contentLength); + Assert.assertEquals(type, contentType); + Assert.assertEquals(encoding, contentEncoding); + log.info("content-disposition: " + contentDisposition); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("testFilenameOverride.txt")); + Date lastModified = head.getLastModified(); + Assert.assertNotNull(lastModified); + + URL foURL = new URL(artifactURL.toExternalForm() + ":fo/alternate.txt"); + head = new HttpGet(foURL, bos); + head.setHeadOnly(true); + log.info("head: " + foURL.toExternalForm()); + Subject.doAs(userSubject, new RunnableAction(head)); + log.info("head: " + head.getResponseCode() + " " + head.getThrowable()); + log.info("headers: " + head.getResponseHeader("content-length") + " " + head.getResponseHeader("digest")); + log.warn("head output: " + bos.toString()); + Assert.assertNull(head.getThrowable()); + checksumURI = head.getDigest(); + contentLength = head.getContentLength(); + contentType = head.getContentType(); + contentEncoding = head.getContentEncoding(); + contentDisposition = head.getResponseHeader("content-disposition"); + Assert.assertEquals(computeChecksumURI(data), checksumURI); + Assert.assertEquals(data.length, contentLength); + Assert.assertEquals(type, contentType); + Assert.assertEquals(encoding, contentEncoding); + log.info("content-disposition: " + contentDisposition); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("alternate.txt")); + Date lastModified2 = head.getLastModified(); + Assert.assertEquals(lastModified, lastModified2); + + // get + bos = new ByteArrayOutputStream(); + log.info("get: " + foURL.toExternalForm()); + HttpGet get = new HttpGet(foURL, bos); + Subject.doAs(userSubject, new RunnableAction(get)); + log.info("get: " + get.getResponseCode() + " " + get.getThrowable()); + log.info("headers: " + get.getResponseHeader("content-length") + " " + get.getResponseHeader("digest")); + log.warn("get output: " + bos.toString()); + Assert.assertNull(get.getThrowable()); + checksumURI = get.getDigest(); + contentLength = get.getContentLength(); + contentType = get.getContentType(); + contentEncoding = get.getContentEncoding(); + contentDisposition = get.getResponseHeader("content-disposition"); + Assert.assertEquals(computeChecksumURI(data), checksumURI); + Assert.assertEquals(data.length, contentLength); + Assert.assertEquals(type, contentType); + Assert.assertEquals(encoding, contentEncoding); + log.info("content-disposition: " + contentDisposition); + Assert.assertTrue(contentDisposition.contains("filename=") && contentDisposition.contains("alternate.txt")); + Date lastModified3 = get.getLastModified(); + Assert.assertEquals(lastModified, lastModified3); + + // delete + HttpDelete delete = new HttpDelete(artifactURL, false); + Subject.doAs(userSubject, new RunnableAction(delete)); + log.info("delete: " + delete.getResponseCode() + " " + delete.getThrowable()); + Assert.assertNull(delete.getThrowable()); + Assert.assertEquals("no content", 204, delete.getResponseCode()); + + } catch (Exception t) { + log.error("unexpected throwable", t); + Assert.fail("unexpected throwable: " + t); + } + } + @Test public void testGetNotFound() { try { diff --git a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java index 5cb3aa59e..02ebb7fdf 100644 --- a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -117,6 +117,9 @@ public abstract class ArtifactAction extends RestAction { // The target artifact URI artifactURI; + // alternmate filename for content-disposition header, usually null + String filenameOverride; + // The (possibly null) authentication token. String authToken; @@ -285,16 +288,29 @@ void parsePath() { String path = this.syncInput.getPath(); log.debug("path: " + path); if (path != null) { - int colonIndex = path.indexOf(":"); - int firstSlashIndex = path.indexOf("/"); - if (colonIndex != -1) { - if (firstSlashIndex < 0 || firstSlashIndex > colonIndex) { - // no auth token--artifact URI is complete path - this.artifactURI = createArtifactURI(path); - } else { - this.artifactURI = createArtifactURI(path.substring(firstSlashIndex + 1)); - this.authToken = path.substring(0, firstSlashIndex); - log.debug("authToken: " + this.authToken); + int colon1 = path.indexOf(":"); + int slash1 = path.indexOf("/"); + if (colon1 != -1) { + if (slash1 >= 0 && slash1 < colon1) { + // auth token + this.authToken = path.substring(0, slash1); + path = path.substring(slash1 + 1); + } + try { + String[] up = path.split(":"); + if (up.length > 1) { + URI auri = new URI(up[0] + ":" + up[1]); + InventoryUtil.validateArtifactURI(ArtifactAction.class, auri); + this.artifactURI = auri; + } + for (int i = 2; i < up.length; i++) { + String[] esp = up[2].split("/"); + if (esp.length == 2 && "fo".equals(esp[0])) { + this.filenameOverride = esp[1]; + } + } + } catch (URISyntaxException | IllegalArgumentException e) { + log.debug("illegal artifact URI: " + path, e); } } } @@ -307,22 +323,4 @@ Artifact getArtifact(URI artifactURI) throws ResourceNotFoundException { } return artifact; } - - /** - * Create a valid artifact uri. - * @param uri The input string. - * @return The artifact uri object. - */ - private URI createArtifactURI(String uri) { - log.debug("artifact URI: " + uri); - URI ret; - try { - ret = new URI(uri); - InventoryUtil.validateArtifactURI(ArtifactAction.class, ret); - } catch (URISyntaxException | IllegalArgumentException e) { - ret = null; - log.debug("illegal artifact URI: " + uri, e); - } - return ret; - } } diff --git a/minoc/src/main/java/org/opencadc/minoc/GetAction.java b/minoc/src/main/java/org/opencadc/minoc/GetAction.java index a74e227de..664881613 100644 --- a/minoc/src/main/java/org/opencadc/minoc/GetAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/GetAction.java @@ -194,7 +194,7 @@ public void doAction() throws Exception { } // default: complete download - HeadAction.setHeaders(artifact, syncOutput); + HeadAction.setHeaders(artifact, filenameOverride, syncOutput); bcos = new ByteCountOutputStream(syncOutput.getOutputStream()); // create tmp StorageLocation with expected checksum so adapter can potentially @@ -238,7 +238,7 @@ public void doAction() throws Exception { private ByteCountOutputStream doByteRangeRequest(Artifact artifact, ByteRange byteRange) throws InterruptedException, IOException, ResourceNotFoundException, ReadException, WriteException, StorageEngageException, TransientException { - HeadAction.setHeaders(artifact, syncOutput); + HeadAction.setHeaders(artifact, filenameOverride, syncOutput); syncOutput.setCode(206); long lastByte = byteRange.getOffset() + byteRange.getLength() - 1; syncOutput.setHeader(CONTENT_RANGE, "bytes " + byteRange.getOffset() + "-" diff --git a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java index 5e16d17a4..7008aef00 100644 --- a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java @@ -131,7 +131,7 @@ public void doAction() throws Exception { artifact = getArtifact(artifactURI); } if (artifact != null) { - setHeaders(artifact, syncOutput); + setHeaders(artifact, filenameOverride, syncOutput); } } @@ -140,7 +140,7 @@ public void doAction() throws Exception { * @param artifact The artifact with metadata * @param syncOutput The target response */ - static void setHeaders(Artifact artifact, SyncOutput syncOutput) { + static void setHeaders(Artifact artifact, String filenameOverride, SyncOutput syncOutput) { syncOutput.setHeader(ARTIFACT_ID_HDR, artifact.getID().toString()); syncOutput.setDigest(artifact.getContentChecksum()); syncOutput.setLastModified(artifact.getContentLastModified()); @@ -149,7 +149,10 @@ static void setHeaders(Artifact artifact, SyncOutput syncOutput) { DateFormat df = DateUtil.getDateFormat(DateUtil.HTTP_DATE_FORMAT, DateUtil.GMT); syncOutput.setHeader("Last-Modified", df.format(artifact.getContentLastModified())); - String filename = InventoryUtil.computeArtifactFilename(artifact.getURI()); + String filename = filenameOverride; + if (filename == null) { + filename = InventoryUtil.computeArtifactFilename(artifact.getURI()); + } syncOutput.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); if (artifact.contentEncoding != null) { diff --git a/minoc/src/main/java/org/opencadc/minoc/PostAction.java b/minoc/src/main/java/org/opencadc/minoc/PostAction.java index 38a07ce63..e6a6cb7eb 100644 --- a/minoc/src/main/java/org/opencadc/minoc/PostAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/PostAction.java @@ -193,7 +193,7 @@ public void doAction() throws Exception { log.debug("commit txn: OK"); syncOutput.setCode(202); // Accepted - HeadAction.setHeaders(existing, syncOutput); + HeadAction.setHeaders(existing, null, syncOutput); syncOutput.setHeader("content-length", 0); } catch (Exception e) { log.error("failed to persist " + artifactURI, e); diff --git a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java index 8b38e6f50..25650c2e8 100644 --- a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java +++ b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java @@ -122,10 +122,16 @@ public void doAction() throws Exception { } private void assertCorrectPath(String path, String expURI, String expToken) { + assertCorrectPath(path, expURI, expToken, null); + } + + private void assertCorrectPath(String path, String expURI, String expToken, String expFilenameOverride) { ArtifactAction action = new TestArtifactAction(path); action.parsePath(); + log.info(path + " -> " + action.artifactURI + " - " + action.authToken + " - " + action.filenameOverride); Assert.assertEquals("artifactURI", URI.create(expURI), action.artifactURI); Assert.assertEquals("authToken", expToken, action.authToken); + Assert.assertEquals("filenameOverride", expFilenameOverride, action.filenameOverride); if (action.artifactURI == null) { Assert.fail("Failed to parse legal path: " + path); } @@ -134,9 +140,7 @@ private void assertCorrectPath(String path, String expURI, String expToken) { private void assertIllegalPath(String path) { ArtifactAction action = new TestArtifactAction(path); action.parsePath(); - if (action.artifactURI != null) { - Assert.fail("Should have failed to parse path: " + path); - } + Assert.assertNull(action.artifactURI); } @Test @@ -147,10 +151,14 @@ public void testParsePath() { assertCorrectPath("token/cadc:TEST/myartifact", "cadc:TEST/myartifact", "token"); assertCorrectPath("cadc:TEST/myartifact", "cadc:TEST/myartifact", null); assertCorrectPath("token/cadc:TEST/myartifact", "cadc:TEST/myartifact", "token"); - assertCorrectPath("mast:long/uri/with/segments/fits.fits", "mast:long/uri/with/segments/fits.fits", null); + assertCorrectPath("mast:long/uri/with/segments/something.fits", "mast:long/uri/with/segments/something.fits", null); assertCorrectPath("token/mast:long/uri/with/segments/fits.fits", "mast:long/uri/with/segments/fits.fits", "token"); assertCorrectPath("token-with-dashes/cadc:TEST/myartifact", "cadc:TEST/myartifact", "token-with-dashes"); + assertCorrectPath("cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", null, "something.fits"); + assertCorrectPath("token/cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", "token", "something.fits"); + + assertIllegalPath(null); assertIllegalPath(""); assertIllegalPath("noschemeinuri"); assertIllegalPath("token/noschemeinuri"); @@ -161,9 +169,6 @@ public void testParsePath() { assertIllegalPath("cadc://:port/path"); assertIllegalPath("artifacts/token1/token2/cadc:FOO/bar"); assertIllegalPath("artifacts/token/cadc:ccda:FOO/bar"); - - assertIllegalPath(null); - } catch (Exception unexpected) { log.error("unexpected exception", unexpected); Assert.fail("unexpected exception: " + unexpected); From ca5b220153877302a035e7c0107cbbfba8c38b6f Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 23 Jan 2024 16:33:02 -0800 Subject: [PATCH 105/186] vault: fix pubkey stored in JNDI; provide filename override to url gen --- .../org/opencadc/vault/VaultInitAction.java | 4 +-- .../vault/VaultTransferGenerator.java | 32 ++++++++----------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 2f8625d36..c253e55fb 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -344,7 +344,7 @@ private void initNodePersistence() { private void initKeyPair() { log.info("initKeyPair: START"); - //jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); + jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName(); try { PreauthKeyPairDAO dao = new PreauthKeyPairDAO(); dao.setConfig(getKeyPairConfig(props)); @@ -368,7 +368,6 @@ private void initKeyPair() { } else { log.info("initKeyPair: re-use existing keys - OK"); } - /* Context ctx = new InitialContext(); try { ctx.unbind(jndiPreauthKeys); @@ -380,7 +379,6 @@ private void initKeyPair() { Object o = ctx.lookup(jndiPreauthKeys); log.info("checking... found: " + jndiPreauthKeys + " = " + o + " in " + ctx); - */ } catch (Exception ex) { throw new RuntimeException("check/init " + KEY_PAIR_NAME + " failed", ex); } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index b75d9ffd8..ec8bc39cc 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -156,33 +156,26 @@ public List getEndpoints(VOSURI target, Transfer transfer, Job job, Li try { Direction dir = transfer.getDirection(); PathResolver ps = new PathResolver(nodePersistence, authorizer, true); - Node n = ps.getNode(target.getParentURI().getPath()); - // assume not null and Container already checked by caller (TransferRunner) - ContainerNode parent = (ContainerNode) n; - Node node = nodePersistence.get(parent, target.getName()); + Node node = ps.getNode(target.getPath()); + if (node == null) { + throw new NodeNotFoundException(target.getPath()); + } Subject currentSubject = AuthenticationUtil.getCurrentSubject(); - if (Direction.pushToVoSpace.equals(dir) && node == null) { - // create new data node?? this currently does not happen because the library - // create DataNode the way that CreateNodeAction would - //ret = handleDataNode(dn, transfer, currentSubject); - throw new RuntimeException("BUG: expected DataNode to be created already: " + target.getPath()); - } else if (node instanceof DataNode) { + if (node instanceof DataNode) { DataNode dn = (DataNode) node; - ret = handleDataNode(dn, transfer, currentSubject); + ret = handleDataNode(dn, target.getName(), transfer, currentSubject); } else { - throw new UnsupportedOperationException(node.getClass().getSimpleName() + " transfer " - + target.getPath()); + throw new UnsupportedOperationException("transfer: " + node.getClass().getSimpleName() + + " at " + target.getPath()); } - } catch (NodeNotFoundException ex) { - throw new FileNotFoundException(target.getPath()); - } catch (LinkingException ex) { - throw new RuntimeException("OOPS: failed to resolve link?", ex); + } finally { + // nothing right now } return ret; } - private List handleDataNode(DataNode node, Transfer trans, Subject s) + private List handleDataNode(DataNode node, String filename, Transfer trans, Subject s) throws IOException, ResourceNotFoundException { log.debug("handleDataNode: " + node); @@ -203,11 +196,12 @@ private List handleDataNode(DataNode node, Transfer trans, Subject s) URI secM = p.getSecurityMethod(); if (secM == null || secM.equals(Standards.SECURITY_METHOD_ANON)) { artifactTrans.getProtocols().add(p); + log.debug("allow protocol: " + p); } } try { - List ret = pg.getProtocols(artifactTrans); + List ret = pg.getProtocols(artifactTrans, filename); log.debug("generated urls: " + ret.size()); for (Protocol p : ret) { log.debug(p.getEndpoint() + " using " + p.getSecurityMethod()); From e46663b4232d69807032a0c43ac5d281a9915a7b Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 23 Jan 2024 16:35:28 -0800 Subject: [PATCH 106/186] cadc-inventory-server allow filename override for pullFromVoSpace --- .../transfer/ProtocolsGenerator.java | 101 ++++++++++-------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index 0307e7c1d..858214c53 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -176,6 +176,10 @@ public boolean getStorageResolverAdded() { } public List getProtocols(Transfer transfer) throws ResourceNotFoundException, IOException { + return getProtocols(transfer, null); + } + + public List getProtocols(Transfer transfer, String filenameOverride) throws ResourceNotFoundException, IOException { String authToken = null; URI artifactURI = transfer.getTargets().get(0); // see PostAction line ~127 if (tokenGen != null) { @@ -189,7 +193,8 @@ public List getProtocols(Transfer transfer) throws ResourceNotFoundExc List protos = null; if (Direction.pullFromVoSpace.equals(transfer.getDirection())) { - protos = doPullFrom(artifactURI, transfer, authToken); + // filename override only on GET + protos = doPullFrom(artifactURI, transfer, authToken, filenameOverride); } else { protos = doPushTo(artifactURI, transfer, authToken); } @@ -324,7 +329,8 @@ private Capability getFilesCapability(StorageSite storageSite) { - List doPullFrom(URI artifactURI, Transfer transfer, String authToken) throws ResourceNotFoundException, IOException { + List doPullFrom(URI artifactURI, Transfer transfer, String authToken, String filenameOverride) + throws ResourceNotFoundException, IOException { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO); Set sites = storageSiteDAO.list(); // this set could be cached @@ -337,7 +343,8 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) artifact = getUnsyncedArtifact(artifactURI, transfer, sites, authToken); } } - + log.debug(artifactURI + " found: " + artifact); + List storageSites = new ArrayList<>(); if (artifact != null) { if (artifact.storageLocation != null) { @@ -357,64 +364,70 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken) } } } - + prioritizePullFromSites(storageSites); + log.debug("available sites: " + storageSites.size()); for (StorageSite storageSite : storageSites) { + log.debug("trying site: " + storageSite.getResourceID() + " allowRead=" + storageSite.getAllowRead()); Capability filesCap = getFilesCapability(storageSite); - if (filesCap != null) { + if (filesCap != null && storageSite.getAllowRead()) { for (Protocol proto : transfer.getProtocols()) { - if (storageSite.getAllowRead()) { - // less generic request for service that implements an API - // HACK: this is filesCap specific in here - if (proto.getUri().equals(filesCap.getStandardID())) { + log.debug("\tprotocol: " + proto); + // less generic request for service that implements an API + // HACK: this is filesCap specific in here + if (proto.getUri().equals(filesCap.getStandardID())) { + Protocol p = new Protocol(proto.getUri()); + p.setEndpoint(storageSite.getResourceID().toASCIIString()); + protos.add(p); + } + URI sec = proto.getSecurityMethod(); + if (sec == null) { + sec = Standards.SECURITY_METHOD_ANON; + } + Interface iface = filesCap.findInterface(sec); + if (iface != null) { + URL baseURL = iface.getAccessURL().getURL(); + log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); + if (protocolCompat(proto, baseURL)) { + StringBuilder sb = new StringBuilder(); + sb.append(baseURL.toExternalForm()).append("/"); + if (authToken != null && Standards.SECURITY_METHOD_ANON.equals(sec)) { + sb.append(authToken).append("/"); + } + sb.append(artifactURI.toASCIIString()); + if (filenameOverride != null) { + sb.append(":fo/").append(filenameOverride); + } Protocol p = new Protocol(proto.getUri()); - p.setEndpoint(storageSite.getResourceID().toASCIIString()); + if (transfer.version == VOS.VOSPACE_21) { + p.setSecurityMethod(proto.getSecurityMethod()); + } + p.setEndpoint(sb.toString()); protos.add(p); - } - URI sec = proto.getSecurityMethod(); - if (sec == null) { - sec = Standards.SECURITY_METHOD_ANON; - } - Interface iface = filesCap.findInterface(sec); - if (iface != null) { - URL baseURL = iface.getAccessURL().getURL(); - log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); - if (protocolCompat(proto, baseURL)) { - StringBuilder sb = new StringBuilder(); + log.debug("added: " + p); + + // add a plain anon URL + if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) { + sb = new StringBuilder(); sb.append(baseURL.toExternalForm()).append("/"); - if (authToken != null && Standards.SECURITY_METHOD_ANON.equals(sec)) { - sb.append(authToken).append("/"); - } sb.append(artifactURI.toASCIIString()); - Protocol p = new Protocol(proto.getUri()); - if (transfer.version == VOS.VOSPACE_21) { - p.setSecurityMethod(proto.getSecurityMethod()); + p = new Protocol(proto.getUri()); + if (filenameOverride != null) { + sb.append(":fo/").append(filenameOverride); } p.setEndpoint(sb.toString()); protos.add(p); log.debug("added: " + p); - - // add a plain anon URL - if (authToken != null && !requirePreauthAnon && Standards.SECURITY_METHOD_ANON.equals(sec)) { - sb = new StringBuilder(); - sb.append(baseURL.toExternalForm()).append("/"); - sb.append(artifactURI.toASCIIString()); - p = new Protocol(proto.getUri()); - p.setEndpoint(sb.toString()); - protos.add(p); - log.debug("added: " + p); - } - } else { - log.debug("reject protocol: " + proto - + " reason: no compatible URL protocol"); } } else { log.debug("reject protocol: " + proto - + " reason: unsupported security method: " + proto.getSecurityMethod()); + + " reason: no compatible URL protocol"); } } else { - log.debug("Storage not allowed read " + storageSite.getName()); + log.debug("reject protocol: " + proto + + " reason: unsupported security method: " + proto.getSecurityMethod()); } + } } } From 9e1f72584af70d8eef19e82ed834f5fd1a0d6b8b Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 24 Jan 2024 16:40:55 -0800 Subject: [PATCH 107/186] minoc: made filename override only work in head/get maybe not require banning colon from path components of artifact.uri --- .../org/opencadc/minoc/ArtifactAction.java | 32 +++++++++++-------- .../java/org/opencadc/minoc/GetAction.java | 2 ++ .../java/org/opencadc/minoc/HeadAction.java | 1 + .../opencadc/minoc/ArtifactActionTest.java | 5 +++ 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java index 02ebb7fdf..7e51264cf 100644 --- a/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/ArtifactAction.java @@ -116,8 +116,10 @@ public abstract class ArtifactAction extends RestAction { // The target artifact URI artifactURI; + String errMsg; // alternmate filename for content-disposition header, usually null + boolean extractFilenameOverride = false; String filenameOverride; // The (possibly null) authentication token. @@ -263,6 +265,10 @@ protected void initAndAuthorize(Class grantClass, boolean allow void init() { if (this.artifactURI == null) { + if (errMsg != null) { + throw new IllegalArgumentException(errMsg); + } + // generic throw new IllegalArgumentException("missing or invalid artifact URI"); } } @@ -292,25 +298,25 @@ void parsePath() { int slash1 = path.indexOf("/"); if (colon1 != -1) { if (slash1 >= 0 && slash1 < colon1) { - // auth token + // auth token in front this.authToken = path.substring(0, slash1); path = path.substring(slash1 + 1); } try { - String[] up = path.split(":"); - if (up.length > 1) { - URI auri = new URI(up[0] + ":" + up[1]); - InventoryUtil.validateArtifactURI(ArtifactAction.class, auri); - this.artifactURI = auri; - } - for (int i = 2; i < up.length; i++) { - String[] esp = up[2].split("/"); - if (esp.length == 2 && "fo".equals(esp[0])) { - this.filenameOverride = esp[1]; - } + int foi = path.indexOf(":fo/"); + if (foi > 0 && extractFilenameOverride) { + // filename override appended + this.filenameOverride = path.substring(foi + 4); + path = path.substring(0, foi); + } else if (foi > 0) { + throw new IllegalArgumentException("detected misuse of :fo/ filename override"); } + URI auri = new URI(path); + InventoryUtil.validateArtifactURI(ArtifactAction.class, auri); + this.artifactURI = auri; } catch (URISyntaxException | IllegalArgumentException e) { - log.debug("illegal artifact URI: " + path, e); + this.errMsg = "illegal artifact URI: " + path + " reason: " + e.getMessage(); + log.debug(errMsg, e); } } } diff --git a/minoc/src/main/java/org/opencadc/minoc/GetAction.java b/minoc/src/main/java/org/opencadc/minoc/GetAction.java index 664881613..e2df6428b 100644 --- a/minoc/src/main/java/org/opencadc/minoc/GetAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/GetAction.java @@ -124,6 +124,7 @@ public class GetAction extends ArtifactAction { // constructor for unit tests with no config/init GetAction(boolean init) { super(init); + this.extractFilenameOverride = true; } /** @@ -131,6 +132,7 @@ public class GetAction extends ArtifactAction { */ public GetAction() { super(); + this.extractFilenameOverride = true; } /** diff --git a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java index 7008aef00..af7ed41c3 100644 --- a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java @@ -93,6 +93,7 @@ public class HeadAction extends ArtifactAction { */ public HeadAction() { super(); + this.extractFilenameOverride = true; } /** diff --git a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java index 25650c2e8..6a34cc19f 100644 --- a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java +++ b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java @@ -127,6 +127,9 @@ private void assertCorrectPath(String path, String expURI, String expToken) { private void assertCorrectPath(String path, String expURI, String expToken, String expFilenameOverride) { ArtifactAction action = new TestArtifactAction(path); + if (expFilenameOverride != null) { + action.extractFilenameOverride = true; + } action.parsePath(); log.info(path + " -> " + action.artifactURI + " - " + action.authToken + " - " + action.filenameOverride); Assert.assertEquals("artifactURI", URI.create(expURI), action.artifactURI); @@ -157,6 +160,8 @@ public void testParsePath() { assertCorrectPath("cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", null, "something.fits"); assertCorrectPath("token/cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", "token", "something.fits"); + + assertCorrectPath("cadc:vault/uuid:/something.fits", "cadc:vault/uuid:/something.fits", null, null); assertIllegalPath(null); assertIllegalPath(""); From c3e515d9f62b575316ef953106315e9cb1254b2c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 26 Jan 2024 09:47:43 -0800 Subject: [PATCH 108/186] make ProtocolsGenerator filter sites and log consistently --- .../transfer/ProtocolsGenerator.java | 137 +++++++++--------- .../transfer/ProtocolsGeneratorTest.java | 15 +- 2 files changed, 83 insertions(+), 69 deletions(-) diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java index 858214c53..d8fca2f08 100644 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java +++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java @@ -163,12 +163,8 @@ public class ProtocolsGenerator { public ProtocolsGenerator(ArtifactDAO artifactDAO, Map siteAvailabilities, Map siteRules) { this.artifactDAO = artifactDAO; this.deletedArtifactEventDAO = new DeletedArtifactEventDAO(this.artifactDAO); - this.user = user; - this.tokenGen = tokenGen; this.siteAvailabilities = siteAvailabilities; this.siteRules = siteRules; - this.preventNotFound = preventNotFound; - this.storageResolver = storageResolver; } public boolean getStorageResolverAdded() { @@ -195,8 +191,11 @@ public List getProtocols(Transfer transfer, String filenameOverride) t if (Direction.pullFromVoSpace.equals(transfer.getDirection())) { // filename override only on GET protos = doPullFrom(artifactURI, transfer, authToken, filenameOverride); - } else { + } else if (Direction.pushToVoSpace.equals(transfer.getDirection())) { protos = doPushTo(artifactURI, transfer, authToken); + } else { + throw new UnsupportedOperationException("unexpected transfer direction: " + transfer.getDirection().getValue()); + } return protos; } @@ -274,16 +273,6 @@ public Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set storageSites) { - // contains the algorithm for prioritizing storage sites to pull from. - - // was: prefer read/write sites to put less load on a read-only "seeder" site during migration - //storageSites.sort((site1, site2) -> Boolean.compare(!site1.getAllowWrite(), !site2.getAllowWrite())); - - // random - Collections.shuffle(storageSites); - } - Artifact getRemoteArtifact(URL location, URI artifactURI) { try { HttpGet head = new HttpGet(location, true); @@ -327,8 +316,23 @@ private Capability getFilesCapability(StorageSite storageSite) { return filesCap; } + // contains the algorithm for prioritizing storage sites to get file + static List prioritizePullFromSites(List storageSites) { + // filter out non-readble + List ret = new ArrayList<>(storageSites.size()); + for (StorageSite s : storageSites) { + if (s.getAllowRead()) { + ret.add(s); + } else { + log.debug("storage site is not readable: " + s.getResourceID()); + } + } + + // random + Collections.shuffle(ret); + return ret; + } - List doPullFrom(URI artifactURI, Transfer transfer, String authToken, String filenameOverride) throws ResourceNotFoundException, IOException { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO); @@ -365,9 +369,9 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken, } } - prioritizePullFromSites(storageSites); - log.debug("available sites: " + storageSites.size()); - for (StorageSite storageSite : storageSites) { + List readableSites = prioritizePullFromSites(storageSites); + log.debug("pullFrom: known sites " + storageSites.size() + " -> readableSites " + readableSites.size()); + for (StorageSite storageSite : readableSites) { log.debug("trying site: " + storageSite.getResourceID() + " allowRead=" + storageSite.getAllowRead()); Capability filesCap = getFilesCapability(storageSite); if (filesCap != null && storageSite.getAllowRead()) { @@ -459,13 +463,16 @@ List doPullFrom(URI artifactURI, Transfer transfer, String authToken, return protos; } + // the algorithm for prioritizing storage sites to put file static SortedSet prioritizePushToSites(Set storageSites, URI artifactURI, Map siteRules) { PrioritizingStorageSiteComparator comparator = new PrioritizingStorageSiteComparator(siteRules, artifactURI, null); TreeSet orderedSet = new TreeSet<>(comparator); - for (StorageSite storageSite : storageSites) { - if (storageSite.getAllowWrite()) { - orderedSet.add(storageSite); + for (StorageSite s : storageSites) { + if (s.getAllowWrite()) { + orderedSet.add(s); + } else { + log.debug("storage site is not writable: " + s.getResourceID()); } } return orderedSet; @@ -477,8 +484,10 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT Set storageSites = storageSiteDAO.list(); // this set could be cached List protos = new ArrayList<>(); - SortedSet orderedSites = prioritizePushToSites(storageSites, artifactURI, this.siteRules); + // prioritize also filters out non-writable sites + Set orderedSites = prioritizePushToSites(storageSites, artifactURI, this.siteRules); // produce URLs for all writable sites + log.debug("pushTo: known sites " + storageSites.size() + " -> writableSites " + orderedSites.size()); for (StorageSite storageSite : orderedSites) { // check if site is currently offline if (!isAvailable(storageSite.getResourceID())) { @@ -486,7 +495,7 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT continue; } - //log.warn("PUT: " + storageSite); + log.debug("pushTo: trying site " + storageSite.getResourceID()); Capability filesCap = null; try { Capabilities caps = regClient.getCapabilities(storageSite.getResourceID()); @@ -499,52 +508,50 @@ private List doPushTo(URI artifactURI, Transfer transfer, String authT } if (filesCap != null) { for (Protocol proto : transfer.getProtocols()) { - //log.warn("PUT: " + storageSite + " proto: " + proto); - if (storageSite.getAllowWrite()) { - // less generic request for service that implements - // HACK: this is filesCap specific in here - if (proto.getUri().equals(filesCap.getStandardID())) { - Protocol p = new Protocol(proto.getUri()); - p.setEndpoint(storageSite.getResourceID().toASCIIString()); - protos.add(p); - } - URI sec = proto.getSecurityMethod(); - if (sec == null) { - sec = Standards.SECURITY_METHOD_ANON; - } - boolean anon = Standards.SECURITY_METHOD_ANON.equals(sec); - Interface iface = filesCap.findInterface(sec); - log.debug("PUT: " + storageSite + " proto: " + proto + " iface: " + iface); - if (iface != null) { - URL baseURL = iface.getAccessURL().getURL(); - //log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); - if (protocolCompat(proto, baseURL)) { - // // no plain anon URL for put: !anon or anon+token - boolean gen = (!anon || (anon && authToken != null)); - if (gen) { - StringBuilder sb = new StringBuilder(); - sb.append(baseURL.toExternalForm()).append("/"); - if (authToken != null && anon) { - sb.append(authToken).append("/"); - } - sb.append(artifactURI.toASCIIString()); - Protocol p = new Protocol(proto.getUri()); - if (transfer.version == VOS.VOSPACE_21) { - p.setSecurityMethod(proto.getSecurityMethod()); - } - p.setEndpoint(sb.toString()); - protos.add(p); - log.debug("added: " + p); + log.debug("pushTo: " + storageSite + " proto: " + proto); + // less generic request for service that implements + // HACK: this is filesCap specific in here + if (proto.getUri().equals(filesCap.getStandardID())) { + Protocol p = new Protocol(proto.getUri()); + p.setEndpoint(storageSite.getResourceID().toASCIIString()); + protos.add(p); + } + URI sec = proto.getSecurityMethod(); + if (sec == null) { + sec = Standards.SECURITY_METHOD_ANON; + } + boolean anon = Standards.SECURITY_METHOD_ANON.equals(sec); + Interface iface = filesCap.findInterface(sec); + log.debug("pushTo: " + storageSite + " proto: " + proto + " iface: " + iface); + if (iface != null) { + URL baseURL = iface.getAccessURL().getURL(); + //log.debug("base url for site " + storageSite.getResourceID() + ": " + baseURL); + if (protocolCompat(proto, baseURL)) { + // // no plain anon URL for put: !anon or anon+token + boolean gen = (!anon || (anon && authToken != null)); + if (gen) { + StringBuilder sb = new StringBuilder(); + sb.append(baseURL.toExternalForm()).append("/"); + if (authToken != null && anon) { + sb.append(authToken).append("/"); + } + sb.append(artifactURI.toASCIIString()); + Protocol p = new Protocol(proto.getUri()); + if (transfer.version == VOS.VOSPACE_21) { + p.setSecurityMethod(proto.getSecurityMethod()); } - - } else { - log.debug("PUT: " + storageSite + "PUT: reject protocol: " + proto - + " reason: no compatible URL protocol"); + p.setEndpoint(sb.toString()); + protos.add(p); + log.debug("added: " + p); } + } else { log.debug("PUT: " + storageSite + "PUT: reject protocol: " + proto - + " reason: unsupported security method: " + proto.getSecurityMethod()); + + " reason: no compatible URL protocol"); } + } else { + log.debug("PUT: " + storageSite + "PUT: reject protocol: " + proto + + " reason: unsupported security method: " + proto.getSecurityMethod()); } } } diff --git a/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java b/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java index 2cc21f3f9..190aad14c 100644 --- a/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java +++ b/cadc-inventory-server/src/test/java/org/opencadc/inventory/transfer/ProtocolsGeneratorTest.java @@ -73,6 +73,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; @@ -99,10 +100,16 @@ public void testPrioritizePullFromSites() throws Exception { for (int i = 0; i < 10; i++) { sites.add(new StorageSite(URI.create("ivo://site" + i), "site1" + i, true, rd.nextBoolean())); } - ProtocolsGenerator.prioritizePullFromSites(sites); - for (StorageSite s : sites) { - log.info("found: " + s.getID() + " aka " + s.getResourceID()); - } + List result1 = ProtocolsGenerator.prioritizePullFromSites(sites); + Assert.assertEquals(sites.size(), result1.size()); + Assert.assertTrue(result1.containsAll(sites)); + + List result2 = ProtocolsGenerator.prioritizePullFromSites(sites); + Assert.assertEquals(sites.size(), result2.size()); + Assert.assertTrue(result2.containsAll(sites)); + + // test random order + Assert.assertNotEquals(result1, result2); } @Test From cdada9a30985d4067f013ce81f8afaa332cd7817 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 26 Jan 2024 09:48:15 -0800 Subject: [PATCH 109/186] minoc: use trust preauth config when determining readable and writable --- .../src/main/java/org/opencadc/minoc/MinocInitAction.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java index 03274739f..86304dcc2 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java @@ -196,9 +196,13 @@ private void initStorageSite() { if (name.charAt(0) == '/') { name = name.substring(1); } + + // possibly temporary hack: advertise readable and writable if this service + // is configured to accept preauth tokens + boolean trustPreauth = !config.getTrustedServices().isEmpty(); - boolean allowRead = !config.getReadGrantServices().isEmpty(); - boolean allowWrite = !config.getWriteGrantServices().isEmpty(); + boolean allowRead = trustPreauth || !config.getReadGrantServices().isEmpty(); + boolean allowWrite = trustPreauth || !config.getWriteGrantServices().isEmpty(); StorageSite self = null; if (curlist.isEmpty()) { From afe0eec2b94760d7da0213834bea6603b894c2a9 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 26 Jan 2024 09:48:55 -0800 Subject: [PATCH 110/186] vault: adapt to PathResolver changes; fix dao config usage --- .../main/java/org/opencadc/vault/NodePersistenceImpl.java | 2 +- .../java/org/opencadc/vault/VaultTransferGenerator.java | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0e352fcc1..0669c1055 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -239,7 +239,7 @@ private NodeDAO getDAO() { private ArtifactDAO getArtifactDAO() { ArtifactDAO instance = new ArtifactDAO(true); // origin==true? - instance.setConfig(nodeDaoConfig); + instance.setConfig(invDaoConfig); return instance; } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index ec8bc39cc..cb3ed97a0 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -74,7 +74,6 @@ import ca.nrc.cadc.uws.Job; import ca.nrc.cadc.uws.Parameter; import ca.nrc.cadc.vosi.Availability; -import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -91,9 +90,7 @@ import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; import org.opencadc.permissions.TokenTool; -import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.LinkingException; import org.opencadc.vospace.Node; import org.opencadc.vospace.NodeNotFoundException; import org.opencadc.vospace.VOSURI; @@ -155,8 +152,8 @@ public List getEndpoints(VOSURI target, Transfer transfer, Job job, Li List ret = null; try { Direction dir = transfer.getDirection(); - PathResolver ps = new PathResolver(nodePersistence, authorizer, true); - Node node = ps.getNode(target.getPath()); + PathResolver ps = new PathResolver(nodePersistence, authorizer); + Node node = ps.getNode(target.getPath(), true); if (node == null) { throw new NodeNotFoundException(target.getPath()); } From fe6c2c7cf1beb56ecc3be2a609a8c912ee750d9d Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 26 Jan 2024 13:59:42 -0800 Subject: [PATCH 111/186] update vault README --- .../org/opencadc/inventory/InventoryUtil.java | 3 ++- vault/README.md | 18 +++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java index a302179a3..adc5065f6 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java @@ -384,6 +384,7 @@ public static void assertValidPathComponent(Class caller, String name, String te boolean slash = (test.indexOf('/') >= 0); boolean escape = (test.indexOf('\\') >= 0); boolean percent = (test.indexOf('%') >= 0); + boolean colon = (test.indexOf(":") >= 0); boolean semic = (test.indexOf(';') >= 0); boolean amp = (test.indexOf('&') >= 0); boolean dollar = (test.indexOf('$') >= 0); @@ -398,7 +399,7 @@ public static void assertValidPathComponent(Class caller, String name, String te } throw new IllegalArgumentException(s + name + ": " + test + " reason: path component may not contain space ( ), slash (/), escape (\\), percent (%)," - + " semi-colon (;), ampersand (&), or dollar ($), question (?), or square brackets ([])"); + + " colon (:), semi-colon (;), ampersand (&), dollar ($), question (?), or square brackets ([])"); } } diff --git a/vault/README.md b/vault/README.md index 081fbefa7..02e701eab 100644 --- a/vault/README.md +++ b/vault/README.md @@ -62,8 +62,9 @@ in the JDBC URL and the schema name is specified in the minoc.properties (below) initialize the database will show up in logs and in the VOSI-availability output. The _inventory_ content may be in the same database as the _nodes_, in a different database in the same server, or in a different server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. Note: it is a good -idea to set `maxActive` to a valid integer (e.g. 0) when using a single pool; this avoids an ugly but -meaningless stack trace in the logs at startup. +idea to set `maxActive` to a valid integer (e.g. 1 because the tomcat connection pool doesn't like 0 and +decides to make it 100 instead) when using a single pool; this avoids an ugly but meaningless stack trace +in the logs at startup. The _uws_ account owns and manages (create, alter, drop) uws database objects in the `uws` schema and manages all the content (insert, update, delete). The database is specified in the JDBC URLFailure to connect or initialize the @@ -101,20 +102,19 @@ _all known_ sites. It only makes sense to enable this when `vault` is running in `raven` and/or `fenwick` instances syncing artifact metadata. This feature introduces an overhead for the genuine not-found cases: transfer negotiation to GET the file that was never PUT. -The _inventory.schema_ name is the name of the database schema that contains the inventory database objects. The -account nominally requires read-only (select) permission on those objects. This currently must be "inventory" due -to configuration limitations in luskan. +The _inventory.schema_ name is the name of the database schema used for all inventory database objects. This +currently must be "inventory" due to configuration limitations in luskan. -The _vospace.schema_ name is the name of the database schema used for all created database objects (tables, indices, etc). Note that with a single connection pool, the two schemas must currently be in the same database. -TODO: augment config to support separate inventory and vospace pools. +The _vospace.schema_ name is the name of the database schema used for all vospace database objects. Note that +with a single connection pool, the two schemas must be in the same database. The _singlePool_ key configures `vault` to use a single pool (the _nodes_ pool) for both vospace and inventory operations. The inventory and vospace content must be in the same database for this to work. When configured to use a single pool, delete node operations can delete a DataNode and the associated Artifact and create the DeletedArtifactEvent in a single transaction. When configured to use separate pools, the delete Artifact and create DeletedArtifactEvent are done in a separate transaction and if that fails the Artifact will be left behind and -orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` allows -the content to be stored in two separate databases or servers. +orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` +allows the content to be stored in two separate databases or servers. The _root.owner_ owns the root node and has full read and write permission in the root container, so it can create and delete container nodes at the root and assign container node properties that are normally read-only From ab0972fd931e9ac0f1db52e1a581770b7c52a126 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 2 Feb 2024 13:19:13 -0800 Subject: [PATCH 112/186] ban colon (:) in artifact uri path components --- .../src/main/java/org/opencadc/inventory/InventoryUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java index adc5065f6..15553c656 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java @@ -384,7 +384,7 @@ public static void assertValidPathComponent(Class caller, String name, String te boolean slash = (test.indexOf('/') >= 0); boolean escape = (test.indexOf('\\') >= 0); boolean percent = (test.indexOf('%') >= 0); - boolean colon = (test.indexOf(":") >= 0); + boolean colon = (test.indexOf(':') >= 0); boolean semic = (test.indexOf(';') >= 0); boolean amp = (test.indexOf('&') >= 0); boolean dollar = (test.indexOf('$') >= 0); @@ -392,7 +392,7 @@ public static void assertValidPathComponent(Class caller, String name, String te boolean sqopen = (test.indexOf('[') >= 0); boolean sqclose = (test.indexOf(']') >= 0); - if (space || slash || escape || percent || semic || amp || dollar || question || sqopen || sqclose) { + if (space || slash || escape || percent || colon || semic || amp || dollar || question || sqopen || sqclose) { String s = "invalid "; if (caller != null) { s += caller.getSimpleName() + "."; From d13dfdeb5671b7b0b76a01c6da0f3efc833c25c0 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 2 Feb 2024 13:20:53 -0800 Subject: [PATCH 113/186] tweaks to vault-quota/Design --- vault-quota/Design.md | 44 +++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index b2b43b920..af9b5cc13 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -5,8 +5,8 @@ The definitive source of content-length (file size) of a DataNode comes from the In the case of a `vault` service co-located with a single storage site (`minoc`), the new Artifact is visible in the database as soon as the PUT to `minoc` is completed. In the case of a `vault` service co-located with a global SI, the new -Artifact is visible in the database once it is synced from the site of the PUT to -`minoc` to the global database by `fenwick` (or worst case: `ratik`). +Artifact is visible in the database once it is synced from the site of the PUT to +the global database by `fenwick` (or worst case: `ratik`). ## TODO The design below only takes into account incremental propagation of space used @@ -61,6 +61,9 @@ but there is nothing there right now. ## validation +### ContainerNode vs child nodes discrepancies +TODO: figure out how to validate ContainerNode sizes vs sum(child sizes) in a live system + ### DataNode vs Artifact discrepancies These can be validated in parallel by multiple threads, subdivide work by bucket. @@ -73,37 +76,38 @@ else: ?? discrepancy 2: DataNode exists but Artifact does not explanation: DataNode created, Artifact never (successfully) put -evidence: dataNode.size == 0 -action: none +evidence: +action: set nodeSize = 0 discrepancy 3: DataNode exists but Artifact does not explanation: deleted or lost Artifact evidence: DataNode.size != 0 (deleted vs lost: DeletedArtifactEvent exists) -action: fix DataNode.size +action: fix nodeSize -discrepancy 4: DataNode.size != Artifact.contentLength +discrepancy 4: nodeSize != Artifact.contentLength explanation: pending/missed Artifact event action: fix DataNode and propagate delta to parent ContainerNode (same as incremental) ``` -This could be accomplished with a single query on on inventory.Artifact full outer join -vospace.Node to get all the pairs. The more generic approach would be to do a merge join -of two iterators: - -Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); -Iterator niter = nodeDAO.iterator(vaultNamespace, bucket); - -The more generic dual iterator approach could be made to work if the inventory and vospace -content are in different PG database or server - TBD. +The most generic implementation is a merge join of two iterators (see ratik, tantar): +``` +Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); // artifact.uri order +Iterator niter = nodeDAO.iterator(vaultNamespace, bucket); // storageID order +``` ## database changes required note: all field and column names TBD -* add `size` and `delta` fields to ContainerNode (transient) -* add `size` field to DataNode (transient) -* add `size` to the `vospace.Node` table +note: fields in Node classes probably not transient but TBD +* add `nodeSize` and `delta` fields to ContainerNode +* add `nodeSize` field to DataNode (no size props in LinkNode!) +* add `nodeSize` to the `vospace.Node` table * add `delta` to the `vospace.Node` table +* add `storageBucket` to DataNode * add `storageBucket` to `vospace.Node` table (validation) -* incremental sync query/iterator (ArtifactDAO?) -* lookup DataNode by storageID (ArtifactDAO?) +## cadc-inventory-db API required +* incremental sync query/iterator: ArtifactDAO.iterator(Namespace ns, String uriBucketPrefix, Date minLastModified)? +* lookup DataNode by storageID: NodeDAO.getDataNode(URI storageID)? +* validate-by-bucket: use ArtifactDAO.iterator(String uriBucketPrefix, boolean ordered, Namespace ns) +* validate-by-bucket: NodeDAO.dataNodeIterator(String storageBucketPrefix, boolean ordered) * indices to support new queries From 0a5345d8c067c07102e2d9bd5e87ffe5ad0fa481 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 2 Feb 2024 13:23:53 -0800 Subject: [PATCH 114/186] cadc-inventory-0.10.0 --- cadc-inventory/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cadc-inventory/build.gradle b/cadc-inventory/build.gradle index 1019d3295..16f69ad90 100644 --- a/cadc-inventory/build.gradle +++ b/cadc-inventory/build.gradle @@ -14,7 +14,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.9.4' +version = '0.10.0' description = 'OpenCADC Storage Inventory core library' def git_url = 'https://github.com/opencadc/storage-inventory' From fa98d3c717ad838f95c1b239528bae5e376d1a44 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Mon, 5 Feb 2024 14:49:11 -0800 Subject: [PATCH 115/186] Added the files end point --- .../java/org/opencadc/minoc/HeadAction.java | 2 +- vault/build.gradle | 6 +- .../opencadc/vault/NodePersistenceImpl.java | 8 +- .../vault/VaultTransferGenerator.java | 20 ++--- vault/src/main/webapp/WEB-INF/web.xml | 27 +++++++ vault/src/main/webapp/capabilities.xml | 10 +++ vault/src/main/webapp/service.yaml | 74 +++++++++++++++++-- 7 files changed, 123 insertions(+), 24 deletions(-) diff --git a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java index af7ed41c3..5cec323e0 100644 --- a/minoc/src/main/java/org/opencadc/minoc/HeadAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/HeadAction.java @@ -154,7 +154,7 @@ static void setHeaders(Artifact artifact, String filenameOverride, SyncOutput sy if (filename == null) { filename = InventoryUtil.computeArtifactFilename(artifact.getURI()); } - syncOutput.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); + syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); if (artifact.contentEncoding != null) { syncOutput.setHeader("Content-Encoding", artifact.contentEncoding); diff --git a/vault/build.gradle b/vault/build.gradle index bba3646dc..116e9208a 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -36,13 +36,13 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' - compile 'org.opencadc:cadc-vos-server:[2.0.3,)' + compile 'org.opencadc:cadc-vos-server:[2.0.10,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' compile 'org.opencadc:cadc-uws-server:[1.2.19,)' compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' compile 'org.opencadc:cadc-cdp:[1.2.3,)' - compile 'org.opencadc:cadc-registry:[1.7.4,)' + compile 'org.opencadc:cadc-registry:[1.7.6,)' compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' compile 'org.opencadc:cadc-inventory-server:[0.3,1.0)' @@ -54,7 +54,7 @@ dependencies { runtime 'org.opencadc:cadc-gms:[1.0.5,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' - intTestCompile 'org.opencadc:cadc-test-vos:[2.1,)' + intTestCompile 'org.opencadc:cadc-test-vos:[2.1.7,)' } configurations { diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0669c1055..bbc1d2f5d 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -100,7 +100,6 @@ import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.PreauthKeyPairDAO; -import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.permissions.TokenTool; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; @@ -307,9 +306,8 @@ public Node get(ContainerNode parent, String name) throws TransientException { DataNode dn = (DataNode) ret; ArtifactDAO artifactDAO = getArtifactDAO(); Artifact a = artifactDAO.get(dn.storageID); + DateFormat df = NodeWriter.getDateFormat(); if (a != null) { - DateFormat df = NodeWriter.getDateFormat(); - Date d = ret.getLastModified(); Date cd = null; if (ret.getLastModified().before(a.getLastModified())) { @@ -336,6 +334,10 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (a.contentType != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); } + } else { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(ret.getLastModified()))); + // default size to 0 + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "0")); } } return ret; diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index cb3ed97a0..2f2a913b6 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -71,7 +71,6 @@ import ca.nrc.cadc.auth.IdentityManager; import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.reg.Standards; -import ca.nrc.cadc.uws.Job; import ca.nrc.cadc.uws.Parameter; import ca.nrc.cadc.vosi.Availability; import java.io.IOException; @@ -97,7 +96,6 @@ import org.opencadc.vospace.server.PathResolver; import org.opencadc.vospace.server.auth.VOSpaceAuthorizer; import org.opencadc.vospace.server.transfers.TransferGenerator; -import org.opencadc.vospace.transfer.Direction; import org.opencadc.vospace.transfer.Protocol; import org.opencadc.vospace.transfer.Transfer; @@ -114,8 +112,8 @@ public class VaultTransferGenerator implements TransferGenerator { private final TokenTool tokenTool; private final boolean preventNotFound; - private Map siteRules = new HashMap<>(); - private Map siteAvailabilities; + private final Map siteRules = new HashMap<>(); + private final Map siteAvailabilities; @SuppressWarnings("unchecked") public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { @@ -141,7 +139,7 @@ public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO a } @Override - public List getEndpoints(VOSURI target, Transfer transfer, Job job, List additionalParams) throws Exception { + public List getEndpoints(VOSURI target, Transfer transfer, List additionalParams) throws Exception { log.debug("getEndpoints: " + target); if (target == null) { throw new IllegalArgumentException("target is required"); @@ -149,19 +147,17 @@ public List getEndpoints(VOSURI target, Transfer transfer, Job job, Li if (transfer == null) { throw new IllegalArgumentException("transfer is required"); } - List ret = null; + List ret; try { - Direction dir = transfer.getDirection(); PathResolver ps = new PathResolver(nodePersistence, authorizer); Node node = ps.getNode(target.getPath(), true); if (node == null) { throw new NodeNotFoundException(target.getPath()); } - Subject currentSubject = AuthenticationUtil.getCurrentSubject(); if (node instanceof DataNode) { DataNode dn = (DataNode) node; - ret = handleDataNode(dn, target.getName(), transfer, currentSubject); + ret = handleDataNode(dn, target.getName(), transfer); } else { throw new UnsupportedOperationException("transfer: " + node.getClass().getSimpleName() + " at " + target.getPath()); @@ -172,8 +168,8 @@ public List getEndpoints(VOSURI target, Transfer transfer, Job job, Li return ret; } - private List handleDataNode(DataNode node, String filename, Transfer trans, Subject s) - throws IOException, ResourceNotFoundException { + private List handleDataNode(DataNode node, String filename, Transfer trans) + throws IOException { log.debug("handleDataNode: " + node); IdentityManager im = AuthenticationUtil.getIdentityManager(); @@ -205,7 +201,7 @@ private List handleDataNode(DataNode node, String filename, Transfer t } return ret; } catch (ResourceNotFoundException ex) { - return new ArrayList(); + return new ArrayList<>(); } } } diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index cdfc34518..b8ce77f24 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -63,6 +63,28 @@ 2 + + + FilesServlet + ca.nrc.cadc.rest.RestServlet + + augmentSubject + true + + + init + org.opencadc.vault.VaultInitAction + + + get + org.opencadc.vault.files.GetAction + + + head + org.opencadc.vault.files.HeadAction + + 2 + PubKeyServlet @@ -249,6 +271,11 @@ /nodes/* + + FilesServlet + /files/* + + RecursiveDeleteNodeServlet /async-delete/* diff --git a/vault/src/main/webapp/capabilities.xml b/vault/src/main/webapp/capabilities.xml index bd833eb73..221e35e05 100644 --- a/vault/src/main/webapp/capabilities.xml +++ b/vault/src/main/webapp/capabilities.xml @@ -41,6 +41,16 @@ + + + https://replace.me.com/vault/files + + + + + + + https://replace.me.com/vault/async-delete diff --git a/vault/src/main/webapp/service.yaml b/vault/src/main/webapp/service.yaml index 27bcda435..3be524653 100644 --- a/vault/src/main/webapp/service.yaml +++ b/vault/src/main/webapp/service.yaml @@ -17,6 +17,49 @@ schemes: - https basePath: /vault paths: + /files/{filePath}: + + get: + description: | + Get the specified file. + tags: + - Files + responses: + '200': + description: Successful response + '400': + description: If the user requested a container node (directory). + '403': + description: If the user does not have permission. + '404': + description: If the file or part of the path to the file could not be found. + '500': + description: Internal error + '503': + description: Service busy + default: + description: Unexpected error + head: + description: | + Get the metadata of the specified file. + tags: + - Files + responses: + '200': + description: Successful response + '400': + description: If the user requested a container node (directory). + '403': + description: If the user does not have permission. + '404': + description: If the file or part of the path to the file could not be found. + '500': + description: Internal error + '503': + description: Service busy + default: + description: Unexpected error + /nodes/{nodePath}: put: description: | @@ -96,11 +139,32 @@ paths: schema: $ref: '#/definitions/Error' parameters: - - name: nodePath - in: path - description: The path for the node - required: true - type: string + - name: detail + in: query + description: set the node detail level + required: false + type: string + enum: + - min + - max + - properties + - name: uri + in: query + description: for container nodes, the uri of a child node of the container on which to start the list of children. + required: false + type: string + - name: limit + in: query + description: for container nodes, the number of children to return. + required: false + type: string + - name: view + in: query + description: for data nodes, a specific view + required: false + type: string + enum: + - data post: description: | Set the property values for a specific Node From 3fc90a6c29f3315234aae1d12596c7ebf9e17182 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 8 Feb 2024 16:11:43 -0800 Subject: [PATCH 116/186] Ready for code review --- vault/build.gradle | 2 +- .../java/org/opencadc/vault/FilesTest.java | 95 ++++++++++ .../org/opencadc/vault/files/GetAction.java | 124 +++++++++++++ .../org/opencadc/vault/files/HeadAction.java | 170 ++++++++++++++++++ vault/src/main/webapp/WEB-INF/web.xml | 7 +- vault/src/main/webapp/index.html | 165 +++++++++++++++++ vault/src/main/webapp/service.yaml | 123 ++++++++++++- 7 files changed, 679 insertions(+), 7 deletions(-) create mode 100644 vault/src/intTest/java/org/opencadc/vault/FilesTest.java create mode 100644 vault/src/main/java/org/opencadc/vault/files/GetAction.java create mode 100644 vault/src/main/java/org/opencadc/vault/files/HeadAction.java create mode 100644 vault/src/main/webapp/index.html diff --git a/vault/build.gradle b/vault/build.gradle index 116e9208a..70d596939 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -36,7 +36,7 @@ dependencies { compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' compile 'org.opencadc:cadc-vos:[2.0,)' - compile 'org.opencadc:cadc-vos-server:[2.0.10,)' + compile 'org.opencadc:cadc-vos-server:[2.0.9,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' compile 'org.opencadc:cadc-uws-server:[1.2.19,)' diff --git a/vault/src/intTest/java/org/opencadc/vault/FilesTest.java b/vault/src/intTest/java/org/opencadc/vault/FilesTest.java new file mode 100644 index 000000000..c0a64bcd5 --- /dev/null +++ b/vault/src/intTest/java/org/opencadc/vault/FilesTest.java @@ -0,0 +1,95 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024i. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.net.URI; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * Test the nodes endpoint. + * + * @author pdowler + */ +public class FilesTest extends org.opencadc.conformance.vos.FilesTest { + private static final Logger log = Logger.getLogger(FilesTest.class); + + static { + Log4jInit.setLevel("org.opencadc.conformance.vos", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); + } + + private static File ADMIN_CERT = FileUtil.getFileFromResource("vault-test.pem", FilesTest.class); + + public FilesTest() { + super(URI.create("ivo://opencadc.org/vault"), ADMIN_CERT); + } +} diff --git a/vault/src/main/java/org/opencadc/vault/files/GetAction.java b/vault/src/main/java/org/opencadc/vault/files/GetAction.java new file mode 100644 index 000000000..b8363eea4 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/files/GetAction.java @@ -0,0 +1,124 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2024. (c) 2024. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + ************************************************************************ + */ + +package org.opencadc.vault.files; + +import ca.nrc.cadc.net.TransientException; +import java.net.HttpURLConnection; +import java.util.List; +import org.apache.log4j.Logger; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.NodeProperty; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.VOSURI; +import org.opencadc.vospace.server.Utils; +import org.opencadc.vospace.server.transfers.TransferGenerator; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; + +/** + * Class to redirect to a storage URL from which the content of a DataNode can be downloaded + * @author adriand + */ +public class GetAction extends HeadAction { + protected static Logger log = Logger.getLogger(GetAction.class); + + public GetAction() { + super(); + } + + @Override + public void doAction() throws Exception { + DataNode node = resolveAndSetMetadata(); + + for (NodeProperty prop : node.getProperties()) { + if (prop.getKey().equals(VOS.PROPERTY_URI_CONTENTLENGTH) && prop.getValue().equals("0")) { + // empty file + syncOutput.setCode(HttpURLConnection.HTTP_NO_CONTENT); + return; + } + } + + VOSURI targetURI = localServiceURI.getURI(node); + Transfer pullTransfer = new Transfer(targetURI.getURI(), Direction.pullFromVoSpace); + pullTransfer.version = VOS.VOSPACE_21; + pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth + + TransferGenerator tg = nodePersistence.getTransferGenerator(); + List protos = tg.getEndpoints(targetURI, pullTransfer, null); + if (protos.isEmpty()) { + throw new TransientException("No location found for file " + Utils.getPath(node)); + } + Protocol proto = protos.get(0); + String loc = proto.getEndpoint(); + log.debug("Location: " + loc); + syncOutput.setHeader("Location", loc); + syncOutput.setCode(HttpURLConnection.HTTP_SEE_OTHER); + } + +} diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java new file mode 100644 index 000000000..ffe5cb5cc --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -0,0 +1,170 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2024. (c) 2024. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + ************************************************************************ + */ + +package org.opencadc.vault.files; + +import ca.nrc.cadc.rest.InlineContentHandler; +import ca.nrc.cadc.rest.RestAction; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Date; +import javax.naming.Context; +import javax.naming.InitialContext; +import org.apache.log4j.Logger; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.VOSURI; +import org.opencadc.vospace.io.NodeWriter; +import org.opencadc.vospace.server.LocalServiceURI; +import org.opencadc.vospace.server.NodePersistence; +import org.opencadc.vospace.server.PathResolver; +import org.opencadc.vospace.server.Utils; +import org.opencadc.vospace.server.auth.VOSpaceAuthorizer; + +/** + * Class to handle a HEAD request to a DataNode + * @author adriand + */ +public class HeadAction extends RestAction { + protected static Logger log = Logger.getLogger(HeadAction.class); + + protected VOSpaceAuthorizer voSpaceAuthorizer; + protected NodePersistence nodePersistence; + protected LocalServiceURI localServiceURI; + + public HeadAction() { + super(); + } + + @Override + protected final InlineContentHandler getInlineContentHandler() { + return null; + } + + @Override + public void initAction() throws Exception { + String jndiNodePersistence = super.appName + "-" + NodePersistence.class.getName(); + try { + Context ctx = new InitialContext(); + this.nodePersistence = ((NodePersistence) ctx.lookup(jndiNodePersistence)); + this.voSpaceAuthorizer = new VOSpaceAuthorizer(nodePersistence); + localServiceURI = new LocalServiceURI(nodePersistence.getResourceID()); + } catch (Exception oops) { + throw new RuntimeException("BUG: NodePersistence implementation not found with JNDI key " + jndiNodePersistence, oops); + } + + checkReadable(); + } + + @Override + public void doAction() throws Exception { + resolveAndSetMetadata(); + } + + DataNode resolveAndSetMetadata() throws Exception { + PathResolver pathResolver = new PathResolver(nodePersistence, voSpaceAuthorizer); + String filePath = syncInput.getPath(); + Node node = pathResolver.getNode(filePath, true); + + if (node == null) { + throw new IllegalArgumentException("Target not found: " + filePath); + } + + if (!(node instanceof DataNode)) { + throw new IllegalArgumentException("Resolved target is not a data node: " + Utils.getPath(node)); + } + + VOSURI nodeURI = localServiceURI.getURI(node); + + log.debug("node path resolved: " + node.getName()); + log.debug("node type: " + node.getClass().getCanonicalName()); + syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + nodeURI.getName() + "\""); + syncOutput.setHeader("Content-Type", node.getPropertyValue(VOS.PROPERTY_URI_TYPE)); + syncOutput.setHeader("Content-Encoding", node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING)); + syncOutput.setHeader("Content-Length", node.getPropertyValue(VOS.PROPERTY_URI_CONTENTLENGTH)); + if (node.getPropertyValue(VOS.PROPERTY_URI_DATE) != null) { + Date lastMod = NodeWriter.getDateFormat().parse(node.getPropertyValue(VOS.PROPERTY_URI_DATE)); + syncOutput.setLastModified(lastMod); + } + + String contentMD5 = node.getPropertyValue(VOS.PROPERTY_URI_CONTENTMD5); + if (contentMD5 != null) { + try { + URI md5 = new URI("md5:" + contentMD5); + syncOutput.setDigest(md5); + } catch (URISyntaxException ex) { + log.error("found invalid checksum attribute " + contentMD5 + " on node " + nodeURI); + // yes, just skip: users can set attributes so hard to tell if this is a bug or + // user mistake + } + } + syncOutput.setCode(200); + return (DataNode)node; + } + +} diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index b8ce77f24..37d76f544 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -6,6 +6,9 @@ vault + + index.html + logControl @@ -319,9 +322,5 @@ /logControl - - index.html - - diff --git a/vault/src/main/webapp/index.html b/vault/src/main/webapp/index.html new file mode 100644 index 000000000..0fb2c056b --- /dev/null +++ b/vault/src/main/webapp/index.html @@ -0,0 +1,165 @@ + + + + + vault API + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 
+
+ + diff --git a/vault/src/main/webapp/service.yaml b/vault/src/main/webapp/service.yaml index 3be524653..4b0268c45 100644 --- a/vault/src/main/webapp/service.yaml +++ b/vault/src/main/webapp/service.yaml @@ -202,12 +202,97 @@ paths: description: The path for the node required: true type: string - /recursiveDelete: + /transfers: + post: + description: | + Post a transfer request document to the asynchronous VOSpace UWS Job endpoint. This is the first step in performing any transfer for data objects: + + - uploading a data object + - downloading a data object + - moving a data object + - copying a data object + tags: + - Transfering data + consumes: + - text/xml + responses: + '200': + description: Successful response + '201': + description: Successful response + '403': + description: If the user does not have permission. + '404': + description: If the source node could not be found. + '409': + description: If the destination node already exists. + '500': + description: Internal error + '503': + description: Service busy + default: + description: Unexpeced error + parameters: + - name: Transfer + in: body + description: The transfer negotiation document + required: true + schema: + $ref: '#/definitions/Transfer' + /synctrans: + post: + description: | + Post a via query parameters a transfer request to the synchronous VOSpace UWS Job endpoint. This is the first step in performing any transfer for data objects: + + - uploading a data object + - downloading a data object + - moving a data object + - copying a data object + tags: + - Transfering data + consumes: + - text/xml + responses: + '200': + description: Successful response + '201': + description: Successful response + '403': + description: If the user does not have permission. + '404': + description: If the source node could not be found. + '409': + description: If the destination node already exists. + '500': + description: Internal error + '503': + description: Service busy + default: + description: Unexpeced error + parameters: + - name: target + in: query + description: The target parameter + required: true + type: string + format: uri + - name: direction + in: query + description: The direction of the transfer + required: true + type: string + - name: protocol + in: query + description: The protocol to use for the transfer + required: true + type: string + format: uri + /async-delete: post: description: | Post a recursive delete command. This is an IVOA UWS end point. tags: - - Recursive delete + - Recursive operations (async) consumes: - None responses: @@ -236,6 +321,40 @@ paths: required: true type: string format: uri + /async-setprops: + post: + description: | + Post a recursive set properties command. This is an IVOA UWS end point. + tags: + - Recursive operations (async) + consumes: + - None + responses: + '200': + description: Successful response + '201': + description: Successful response + '403': + description: If the user does not have permission. + '404': + description: If the source node could not be found. + '409': + description: If the destination node already exists. + '500': + description: Internal error + '503': + description: Service busy + default: + description: Unexpeced error + schema: + $ref: '#/definitions/Error' + parameters: + - name: nodeURI + in: query + description: The base node (typically a container) to recursively set properties to + required: true + type: string + format: uri /availability: get: tags: From cf1f6f422ee3f58d6cfa3a265543acbe3d825bb5 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 8 Feb 2024 16:44:16 -0800 Subject: [PATCH 117/186] Skipped incomplete test --- minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java index 6a34cc19f..08f41456d 100644 --- a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java +++ b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java @@ -161,7 +161,7 @@ public void testParsePath() { assertCorrectPath("cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", null, "something.fits"); assertCorrectPath("token/cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", "token", "something.fits"); - assertCorrectPath("cadc:vault/uuid:/something.fits", "cadc:vault/uuid:/something.fits", null, null); + //assertCorrectPath("cadc:vault/uuid:/something.fits", "cadc:vault/uuid:/something.fits", null, null); assertIllegalPath(null); assertIllegalPath(""); From 268b55c5a571e5d74af7c74ec3d8e2ed59e92b68 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Mon, 12 Feb 2024 14:11:21 -0800 Subject: [PATCH 118/186] Changes after code review --- vault/build.gradle | 2 +- .../main/java/org/opencadc/vault/NodePersistenceImpl.java | 1 - .../src/main/java/org/opencadc/vault/files/HeadAction.java | 7 +++---- vault/src/main/webapp/WEB-INF/web.xml | 6 +----- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/vault/build.gradle b/vault/build.gradle index 70d596939..17becafa5 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -35,7 +35,7 @@ dependencies { compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' - compile 'org.opencadc:cadc-vos:[2.0,)' + compile 'org.opencadc:cadc-vos:[2.0.3,)' compile 'org.opencadc:cadc-vos-server:[2.0.9,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index bbc1d2f5d..855bc6d30 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -335,7 +335,6 @@ public Node get(ContainerNode parent, String name) throws TransientException { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); } } else { - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_DATE, df.format(ret.getLastModified()))); // default size to 0 ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "0")); } diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java index ffe5cb5cc..6344ef72c 100644 --- a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -67,6 +67,7 @@ package org.opencadc.vault.files; +import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.rest.RestAction; import java.net.URI; @@ -132,7 +133,7 @@ DataNode resolveAndSetMetadata() throws Exception { Node node = pathResolver.getNode(filePath, true); if (node == null) { - throw new IllegalArgumentException("Target not found: " + filePath); + throw new ResourceNotFoundException("Target not found: " + filePath); } if (!(node instanceof DataNode)) { @@ -158,9 +159,7 @@ DataNode resolveAndSetMetadata() throws Exception { URI md5 = new URI("md5:" + contentMD5); syncOutput.setDigest(md5); } catch (URISyntaxException ex) { - log.error("found invalid checksum attribute " + contentMD5 + " on node " + nodeURI); - // yes, just skip: users can set attributes so hard to tell if this is a bug or - // user mistake + throw new RuntimeException("BUG: invalid " + VOS.PROPERTY_URI_CONTENTMD5 + " value " + contentMD5); } } syncOutput.setCode(200); diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index 37d76f544..2c9d33816 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -74,10 +74,6 @@ augmentSubject true - - init - org.opencadc.vault.VaultInitAction - get org.opencadc.vault.files.GetAction @@ -86,7 +82,7 @@ head org.opencadc.vault.files.HeadAction - 2 + 3 From 62c5b196de2087291a242612a1bc88302ec069fc Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 12 Feb 2024 15:15:28 -0800 Subject: [PATCH 119/186] fix cadc-test-vos dependency --- vault/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault/build.gradle b/vault/build.gradle index 17becafa5..a11549995 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -54,7 +54,7 @@ dependencies { runtime 'org.opencadc:cadc-gms:[1.0.5,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' - intTestCompile 'org.opencadc:cadc-test-vos:[2.1.7,)' + intTestCompile 'org.opencadc:cadc-test-vos:[2.1.6,)' } configurations { From 67956298ff8509990a485dac7a8f5a2128ba8bab Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 12 Feb 2024 15:23:12 -0800 Subject: [PATCH 120/186] merge --- baldur/README.md | 3 +++ .../java/org/opencadc/inventory/Entity.java | 9 ++++--- .../org/opencadc/inventory/InventoryUtil.java | 17 +++--------- .../org/opencadc/inventory/StorageSite.java | 9 ++++++- luskan/README.md | 4 +++ minoc/README.md | 26 +++++++++++++++---- .../opencadc/minoc/ArtifactActionTest.java | 7 +++-- raven/README.md | 3 +++ .../opencadc/vault/NodePersistenceImpl.java | 9 +++++++ 9 files changed, 61 insertions(+), 26 deletions(-) diff --git a/baldur/README.md b/baldur/README.md index 641ee0425..93a8b26fa 100644 --- a/baldur/README.md +++ b/baldur/README.md @@ -112,6 +112,9 @@ When more than one entry matches an artifact URI, the grants are combined as fol * the members of any of the groups in all matching readOnlyGroup lists are allowed to read * the members of any of the groups in all matching readWriteGroup lists are allowed to read and write +### cadc-log.properties (optional) +See cadc-log for common +dynamic logging control. ## integration testing diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java b/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java index 8cdda73a9..d334635d8 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java @@ -75,12 +75,15 @@ * @author pdowler */ public abstract class Entity extends org.opencadc.persist.Entity { - + // DO NOT CHANGE + private static final boolean ENTITY_TRUNCATE_DATES = false; + private static final boolean ENTITY_DIGEST_FIELD_NAMES = false; + public Entity() { - super(false); + super(ENTITY_TRUNCATE_DATES, ENTITY_DIGEST_FIELD_NAMES); } public Entity(UUID id) { - super(id, false); + super(id, ENTITY_TRUNCATE_DATES, ENTITY_DIGEST_FIELD_NAMES); } } diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java index 15553c656..c3c3704ab 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/InventoryUtil.java @@ -68,6 +68,7 @@ package org.opencadc.inventory; import ca.nrc.cadc.util.HexUtil; +import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; @@ -129,28 +130,18 @@ public static Boolean isRemoteWinner(Artifact local, Artifact remote) { public static String computeBucket(URI uri, int length) { try { MessageDigest md = MessageDigest.getInstance("SHA-1"); - byte[] bytes = new DummyEntity().primitiveValueToBytes(uri, "Artifact.uri", md.getAlgorithm()); + byte[] bytes = uri.toASCIIString().trim().getBytes("UTF-8"); md.update(bytes); byte[] sha = md.digest(); String hex = HexUtil.toHex(sha); return hex.substring(0, length); } catch (NoSuchAlgorithmException ex) { throw new RuntimeException("BUG: failed to get instance of SHA-1", ex); + } catch (UnsupportedEncodingException ex) { + throw new RuntimeException("BUG: failed to encode String in UTF-8", ex); } } - // make primitiveValueToBytes usable in computeBucket above - private static class DummyEntity extends org.opencadc.inventory.Entity { - DummyEntity() { - super(); - } - - @Override - protected byte[] primitiveValueToBytes(Object o, String name, String digestAlg) { - return super.primitiveValueToBytes(o, name, digestAlg); - } - } - /** * Compute the filename of an artifact URI. * @param uri The uri to parse diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java b/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java index 13dad0efa..a112e0c63 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java @@ -69,6 +69,8 @@ import java.net.URI; import java.util.Objects; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.UUID; import org.apache.log4j.Logger; @@ -84,10 +86,15 @@ public class StorageSite extends Entity implements Comparable { private static final Logger log = Logger.getLogger(StorageSite.class); private URI resourceID; - private String name; + private String name; // deprecate? private boolean allowRead; private boolean allowWrite; + // TODO: resourceID of trusted services - intended from config + private SortedSet trustedSigners = new TreeSet<>(); + // TODO: checksum of the pubkey of trusted services - actual from pub key retrieval + private SortedSet trustedPubKeyChecksums = new TreeSet<>(); + /** * Create a new StorageSite. * diff --git a/luskan/README.md b/luskan/README.md index a436ec89f..dc49e7497 100644 --- a/luskan/README.md +++ b/luskan/README.md @@ -103,6 +103,10 @@ org.opencadc.luskan.uwsRollover = 180 ``` Assuming instances are restarted regularly, this would cause rollover approximatelty once every 6 months. +### cadc-log.properties (optional) +See cadc-log for common +dynamic logging control. + ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/minoc/README.md b/minoc/README.md index c6db67305..8af912e59 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -81,10 +81,14 @@ org.opencadc.minoc.trust.preauth = {resourceID} org.opencadc.minoc.readGrantProvider={resourceID of a permission granting service} org.opencadc.minoc.writeGrantProvider={resourceID of a permission granting service} +# override the implied readable and writable state +org.opencadc.minoc.readable = true | false +org.opencadc.minoc.writable = true | false + # configure StorageAdapter delete behaviour org.opencadc.minoc.recoverableNamespace = {namespace} ``` -The optional _trust.preauth_ key(s) configure `minoc` to trust an external service to have performed +The optional _trust.preauth_ key(s) configure `minoc` to trust external service(s) to have performed authorization checks. Such services may include a signed token in the URL and `minoc` will validate the request using a public key retrieved from the service instead of performing authorization checks itself. Example: @@ -95,11 +99,17 @@ org.opencadc.minoc.trust.preauth = ivo://example.net/raven # trust a SI VOSpace service org.opencadc.minoc.trust.preauth = ivo://example.net/vault ``` +Setting _trust.preauth_ one or more times also implies _readable_ and _writable_ are _true_. + +The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to +get grants (permissions) for operations. Multiple values of the granting service resourceID(s) may be provided +by including multiple property settings (one per line). All services will be consulted but a single positive +result is sufficient to grant permission for an action. Setting these values also sets the implied _readable_ +and _writable_ is _true_ respectively. -The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for -operations. Multiple values of the granting service resourceID(s) may be provided by including multiple property -settings (one per line). All services will be consulted but a single positive result is sufficient to grant permission for an -action. +The optional _readable_ and _writable_ keys configure minoc explicitly rather than relying on one or more of +the above trust or grant provider settings. For example, this allows one to configure a read-only minoc that +trusts other services to do the authorization checks. The optional _recoverableNamespace_ key causes `minoc` to configure the storage adapter so that deletions preserve the file content in a recoverable state. This generally means that storage space remains in use @@ -131,7 +141,13 @@ org.opencadc.minoc.authenticateOnly=true With `authenticateOnly=true`, any authenticated user will be able to read/write/delete files and anonymous users will be able to read files. +### cadc-log.properties (optional) +See cadc-log for common +dynamic logging control. + ### minoc-availability.properties (optional) +WARN: This config file name is going to change to a common one for consistency across multiple services. + The minoc-availability.properties file specifies which users have the authority to change the availability state of the minoc service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. Example: diff --git a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java index 08f41456d..cf9807c66 100644 --- a/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java +++ b/minoc/src/test/java/org/opencadc/minoc/ArtifactActionTest.java @@ -161,8 +161,6 @@ public void testParsePath() { assertCorrectPath("cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", null, "something.fits"); assertCorrectPath("token/cadc:vault/uuid:fo/something.fits", "cadc:vault/uuid", "token", "something.fits"); - //assertCorrectPath("cadc:vault/uuid:/something.fits", "cadc:vault/uuid:/something.fits", null, null); - assertIllegalPath(null); assertIllegalPath(""); assertIllegalPath("noschemeinuri"); @@ -172,8 +170,9 @@ public void testParsePath() { assertIllegalPath("cadc:path#fragment?query"); assertIllegalPath("cadc://host/path"); assertIllegalPath("cadc://:port/path"); - assertIllegalPath("artifacts/token1/token2/cadc:FOO/bar"); - assertIllegalPath("artifacts/token/cadc:ccda:FOO/bar"); + assertIllegalPath("artifacts/token1/token2/cadc:FOO/bar"); // sketchy scheme + assertIllegalPath("cadc:ccda:FOO/bar"); // sketchy extra colons + assertIllegalPath("cadc:vault/uuid:/something.fits"); // extra colons in path component } catch (Exception unexpected) { log.error("unexpected exception", unexpected); Assert.fail("unexpected exception: " + unexpected); diff --git a/raven/README.md b/raven/README.md index bd6948c33..e4ea4f0d9 100644 --- a/raven/README.md +++ b/raven/README.md @@ -108,6 +108,9 @@ org.opencadc.raven.authenticateOnly=true When _authenticateOnly_ is `true`, any authenticated user will be able to read/write/delete files and anonymous users will be able to read files. +### cadc-log.properties (optional) +See cadc-log for common +dynamic logging control. ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 855bc6d30..dec2229b1 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -565,6 +565,15 @@ public void move(Node node, ContainerNode dest, String newName) { if (node.parent == null || dest.parent == null) { throw new IllegalArgumentException("args must both be peristent nodes before move"); } + // try to detect attempt to dsconnect from path to root: node is a parent of dest + ContainerNode cur = dest; + while (!cur.getID().equals(root.getID())) { + cur = cur.parent; + if (cur.getID().equals(node.getID())) { + throw new IllegalArgumentException("invalid destination for move: " + node.getID() + " -> " + dest.getID()); + } + + } Subject caller = AuthenticationUtil.getCurrentSubject(); node.owner = caller; From adbc699be58da0b91a791cf72a66d6ee39c84210 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Tue, 13 Feb 2024 10:51:19 -0800 Subject: [PATCH 121/186] Required changes to make production vostools work --- .../main/java/org/opencadc/vault/NodePersistenceImpl.java | 2 +- .../java/org/opencadc/vault/VaultTransferGenerator.java | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 855bc6d30..d250693b4 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -218,7 +218,7 @@ private Subject getRootOwner(MultiValuedProperties mvp, IdentityManager im) { @Override public Views getViews() { - throw new UnsupportedOperationException(); + return new Views(); } @Override diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 2f2a913b6..9386ac339 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -186,11 +186,9 @@ private List handleDataNode(DataNode node, String filename, Transfer t Transfer artifactTrans = new Transfer(node.storageID, trans.getDirection()); for (Protocol p : trans.getProtocols()) { log.debug("requested protocol: " + p); - URI secM = p.getSecurityMethod(); - if (secM == null || secM.equals(Standards.SECURITY_METHOD_ANON)) { - artifactTrans.getProtocols().add(p); - log.debug("allow protocol: " + p); - } + p.setSecurityMethod(Standards.SECURITY_METHOD_ANON); + artifactTrans.getProtocols().add(p); + log.debug("allow only anon method for " + p); } try { From 5495a660d9b17008cb6b403d434050bca8fb332c Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Tue, 13 Feb 2024 15:16:23 -0800 Subject: [PATCH 122/186] Reworked --- .../opencadc/vault/VaultTransferGenerator.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 9386ac339..97064afb8 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -77,8 +77,10 @@ import java.net.URI; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingException; @@ -184,11 +186,18 @@ private List handleDataNode(DataNode node, String filename, Transfer t pg.preventNotFound = preventNotFound; Transfer artifactTrans = new Transfer(node.storageID, trans.getDirection()); + Set protoURIs = new HashSet<>(); + // storage nodes only work with pre-auth URLs. Return those regardless of the security method + // requested by the user for (Protocol p : trans.getProtocols()) { log.debug("requested protocol: " + p); - p.setSecurityMethod(Standards.SECURITY_METHOD_ANON); - artifactTrans.getProtocols().add(p); - log.debug("allow only anon method for " + p); + if (!protoURIs.contains(p.getUri())) { + Protocol anonProto = new Protocol(p.getUri()); + anonProto.setSecurityMethod(Standards.SECURITY_METHOD_ANON); + artifactTrans.getProtocols().add(p); + protoURIs.add(p.getUri()); + log.debug("Added anon protocol for " + p.getUri()); + } } try { From fe4a210ff7de1cccf11073f3b3c2b20b784e9f9a Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 14 Feb 2024 14:49:02 -0800 Subject: [PATCH 123/186] adapt base Entity for library ctor changes minoc: implement readable and writable config to override state implied by trust --- cadc-inventory/build.gradle | 2 +- .../java/org/opencadc/inventory/Entity.java | 7 +- .../org/opencadc/inventory/EntityTest.java | 84 +++++++++++++++++++ cadc-inventory/src/test/resources/dummy | 1 - .../src/test/resources/sample-artifact.tsv | 2 + .../src/test/resources/sample-iris.tsv | 2 + minoc/README.md | 9 +- .../java/org/opencadc/minoc/MinocConfig.java | 55 +++++++++++- .../org/opencadc/minoc/MinocInitAction.java | 11 +-- 9 files changed, 154 insertions(+), 19 deletions(-) delete mode 100644 cadc-inventory/src/test/resources/dummy create mode 100644 cadc-inventory/src/test/resources/sample-artifact.tsv create mode 100644 cadc-inventory/src/test/resources/sample-iris.tsv diff --git a/cadc-inventory/build.gradle b/cadc-inventory/build.gradle index 16f69ad90..55f43e500 100644 --- a/cadc-inventory/build.gradle +++ b/cadc-inventory/build.gradle @@ -20,7 +20,7 @@ description = 'OpenCADC Storage Inventory core library' def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { - compile 'org.opencadc:cadc-util:[1.9.5,2.0)' + compile 'org.opencadc:cadc-util:[1.10.6,2.0)' testCompile 'junit:junit:[4.0,)' } diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java b/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java index d334635d8..ae7c8b535 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/Entity.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -75,8 +75,11 @@ * @author pdowler */ public abstract class Entity extends org.opencadc.persist.Entity { - // DO NOT CHANGE + // Entity metaChecksum algorithm setup: DO NOT CHANGE private static final boolean ENTITY_TRUNCATE_DATES = false; + // this was the default behaviour when SI was deployed + // operationally and the model is not currently vulnerable + // to "value shifting" bugs so it is still OK private static final boolean ENTITY_DIGEST_FIELD_NAMES = false; public Entity() { diff --git a/cadc-inventory/src/test/java/org/opencadc/inventory/EntityTest.java b/cadc-inventory/src/test/java/org/opencadc/inventory/EntityTest.java index 4bd396e50..938e2dbee 100644 --- a/cadc-inventory/src/test/java/org/opencadc/inventory/EntityTest.java +++ b/cadc-inventory/src/test/java/org/opencadc/inventory/EntityTest.java @@ -68,7 +68,12 @@ package org.opencadc.inventory; import ca.nrc.cadc.date.DateUtil; +import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.io.FileReader; +import java.io.LineNumberReader; +import java.io.PrintWriter; import java.net.URI; import java.security.MessageDigest; import java.text.DateFormat; @@ -217,6 +222,85 @@ public void testArtifactTransientState() { } } + @Test + public void testStableArtifactChecksum() { + URI uri = URI.create("cadc:FOO/bar"); + URI contentChecksum = URI.create("md5:d41d8cd98f00b204e9800998ecf8427e"); + Date contentLastModified = new Date(); + Long contentLength = 1024L; + + try { + + if (false) { + // generate a sample artifact from current code + Artifact cur = new Artifact(uri, contentChecksum, contentLastModified, contentLength); + cur.contentEncoding = "gzip"; + cur.contentType = "text/plain"; + log.info("created: " + cur); + final URI mcs1 = cur.computeMetaChecksum(MessageDigest.getInstance("MD5")); + + StringBuilder sb = new StringBuilder(); + sb.append("uri\tcontentChecksum\tcontentLastModified\tcontentLength\tcontentEncoding\tcontentType\tid\tmetaChecksum\n"); + sb.append(toTSV(cur, mcs1)); + + File out = new File("build/tmp/sample-artifact.tsv"); + PrintWriter w = new PrintWriter(out); + w.println(sb.toString()); + w.close(); + log.info("new sample artifact: " + out.getPath()); + } + + // check that meta checksum of previous samples is stable + final DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + for (String fname : new String[] {"sample-artifact.tsv", "sample-iris.tsv" }) { + File in = FileUtil.getFileFromResource(fname, EntityTest.class); + log.info("checking: " + in.getPath()); + + LineNumberReader r = new LineNumberReader(new FileReader(in)); + String line = r.readLine(); + log.info("header: " + line); + line = r.readLine(); + String[] ss = line.split("[\t]"); + log.info("IN:\n" + line); + + URI suri = URI.create(ss[0]); + URI ccs = URI.create(ss[1]); + Date clm = df.parse(ss[2]); + Long clen = Long.parseLong(ss[3]); + String cenc = ss[4]; + String ctype = ss[5]; + UUID id = UUID.fromString(ss[6]); + Artifact actual = new Artifact(id, suri, ccs, clm, clen); + actual.contentEncoding = cenc; + actual.contentType = ctype; + URI metaChecksum = URI.create(ss[7]); + + URI recomp = actual.computeMetaChecksum(MessageDigest.getInstance("MD5")); + log.info("RE:\n" + toTSV(actual, recomp)); + + Assert.assertEquals(in.getName(), metaChecksum, recomp); + } + + } catch (Exception ex) { + log.error("unexpected exception", ex); + Assert.fail("unexpected exception: " + ex); + } + } + + private String toTSV(Artifact cur, URI mcs) { + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + StringBuilder sb = new StringBuilder(); + sb.append(cur.getURI().toASCIIString()).append("\t"); + sb.append(cur.getContentChecksum().toASCIIString()).append("\t"); + sb.append(df.format(cur.getContentLastModified())).append("\t"); + sb.append(cur.getContentLength()).append("\t"); + sb.append(cur.contentEncoding).append("\t"); + sb.append(cur.contentType).append("\t"); + sb.append(cur.getID().toString()).append("\t"); + sb.append(mcs.toASCIIString()); + return sb.toString(); + } + @Test public void testDeletedArtifactEvent() { try { diff --git a/cadc-inventory/src/test/resources/dummy b/cadc-inventory/src/test/resources/dummy deleted file mode 100644 index e92bbff87..000000000 --- a/cadc-inventory/src/test/resources/dummy +++ /dev/null @@ -1 +0,0 @@ -dummy file so gradle will make dir build/resources/test on build diff --git a/cadc-inventory/src/test/resources/sample-artifact.tsv b/cadc-inventory/src/test/resources/sample-artifact.tsv new file mode 100644 index 000000000..9b5d9621c --- /dev/null +++ b/cadc-inventory/src/test/resources/sample-artifact.tsv @@ -0,0 +1,2 @@ +uri contentChecksum contentLastModified contentLength contentEncoding contentType id metaChecksum +cadc:FOO/bar md5:d41d8cd98f00b204e9800998ecf8427e 2024-02-14T19:02:31.459 1024 gzip text/plain 9b053ecf-4f3f-45f0-934b-8b32be42b6bf md5:450e68e0536f9c0cf78452382c9db9f4 diff --git a/cadc-inventory/src/test/resources/sample-iris.tsv b/cadc-inventory/src/test/resources/sample-iris.tsv new file mode 100644 index 000000000..72b1feaac --- /dev/null +++ b/cadc-inventory/src/test/resources/sample-iris.tsv @@ -0,0 +1,2 @@ +uri contentChecksum contentLastModified contentLength contentEncoding contentType id metaChecksum +cadc:IRIS/I001B1H0.fits md5:b6ead425ae84289246e4528bbdd7da9a 2006-07-25T16:15:19.000 1008000 application/fits 9b53914f-1465-4440-94fa-2871b8532fca md5:daf96da09f157ff65272118ebb2ec589 diff --git a/minoc/README.md b/minoc/README.md index 8af912e59..0f27e2b08 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -82,8 +82,8 @@ org.opencadc.minoc.readGrantProvider={resourceID of a permission granting servic org.opencadc.minoc.writeGrantProvider={resourceID of a permission granting service} # override the implied readable and writable state -org.opencadc.minoc.readable = true | false -org.opencadc.minoc.writable = true | false +org.opencadc.minoc.readable = true|false +org.opencadc.minoc.writable = true|false # configure StorageAdapter delete behaviour org.opencadc.minoc.recoverableNamespace = {namespace} @@ -108,8 +108,8 @@ result is sufficient to grant permission for an action. Setting these values als and _writable_ is _true_ respectively. The optional _readable_ and _writable_ keys configure minoc explicitly rather than relying on one or more of -the above trust or grant provider settings. For example, this allows one to configure a read-only minoc that -trusts other services to do the authorization checks. +the above trust or grant provider settings. For example, this allows one to configure a read-only minoc +(_writable_ = false) that trusts other services to do the authorization checks. The optional _recoverableNamespace_ key causes `minoc` to configure the storage adapter so that deletions preserve the file content in a recoverable state. This generally means that storage space remains in use @@ -131,7 +131,6 @@ prefixes will be recoverable. Others (e.g. `test:FOO/bar`) will be permanently d Note: Since artifact and stored object deletion can also be performed by the `tantar` file validation tool, all instances of `minoc` and `tantar` that use the same inventory and storage adapter should use the same _recoverableNamespace_ configuration so that preservation and recovery (from mistakes) is consistent. - --- **For developer testing only:** To disable authorization checking (via `readGrantProvider` or `writeGrantProvider` services), add the following configuration entry to minoc.properties: diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java index fff166449..53d492bf1 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java @@ -72,6 +72,7 @@ import ca.nrc.cadc.net.HttpTransfer; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; import java.io.ByteArrayOutputStream; @@ -107,6 +108,8 @@ public class MinocConfig { static final String TRUST_KEY = MINOC_KEY + ".trust.preauth"; static final String READ_GRANTS_KEY = MINOC_KEY + ".readGrantProvider"; static final String WRITE_GRANTS_KEY = MINOC_KEY + ".writeGrantProvider"; + static final String READABLE_KEY = MINOC_KEY + ".readable"; + static final String WRITABLE_KEY = MINOC_KEY + ".writable"; static final String RECOVERABLE_NS_KEY = MINOC_KEY + ".recoverableNamespace"; static final String DEV_AUTH_ONLY_KEY = MINOC_KEY + ".authenticateOnly"; @@ -116,6 +119,8 @@ public class MinocConfig { private boolean trustedServiceKeySync = false; private final List readGrantServices = new ArrayList<>(); private final List writeGrantServices = new ArrayList<>(); + private final boolean readable; + private final boolean writable; private final List recoverableNamespaces = new ArrayList<>(); final boolean authenticateOnly; @@ -190,6 +195,20 @@ public MinocConfig() { } } } + + // optional + String sread = configProperties.getFirstPropertyValue(MinocConfig.READABLE_KEY); + if (sread != null) { + this.readable = Boolean.parseBoolean(sread); + } else { + this.readable = !readGrantServices.isEmpty() || !trustedServices.isEmpty(); + } + String swrite = configProperties.getFirstPropertyValue(MinocConfig.WRITABLE_KEY); + if (swrite != null) { + this.writable = Boolean.parseBoolean(swrite); + } else { + this.writable = !writeGrantServices.isEmpty() || !trustedServices.isEmpty(); + } } private void validateConfigProps(MultiValuedProperties mvp) { @@ -206,7 +225,7 @@ private void validateConfigProps(MultiValuedProperties mvp) { } else { sb.append("OK"); } - + String sac = mvp.getFirstPropertyValue(SA_KEY); sb.append("\n\t").append(SA_KEY).append(": "); if (sac == null) { @@ -285,6 +304,30 @@ private void validateConfigProps(MultiValuedProperties mvp) { } } + // optional + String sread = mvp.getFirstPropertyValue(MinocConfig.READABLE_KEY); + sb.append("\n\t" + READABLE_KEY + ": "); + if (sread != null) { + if ("true".equals(sread) || "false".equals(sread)) { + sb.append(" OK"); + } else { + sb.append(" INVALID"); + ok = false; + } + } + + // optional + String swrite = mvp.getFirstPropertyValue(MinocConfig.WRITABLE_KEY); + sb.append("\n\t" + WRITABLE_KEY + ": "); + if (sread != null) { + if ("true".equals(swrite) || "false".equals(swrite)) { + sb.append(" OK"); + } else { + sb.append(" INVALID"); + ok = false; + } + } + // optional List rawRecNS = mvp.getProperty(RECOVERABLE_NS_KEY); if (rawRecNS != null) { @@ -299,7 +342,7 @@ private void validateConfigProps(MultiValuedProperties mvp) { } if (!ok) { - throw new IllegalStateException(sb.toString()); + throw new InvalidConfigException(sb.toString()); } } @@ -349,6 +392,14 @@ private void syncKeys() { } } + public boolean isReadable() { + return readable; + } + + public boolean isWritable() { + return writable; + } + public List getReadGrantServices() { return readGrantServices; } diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java index 86304dcc2..2ca64bbbb 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -197,12 +197,8 @@ private void initStorageSite() { name = name.substring(1); } - // possibly temporary hack: advertise readable and writable if this service - // is configured to accept preauth tokens - boolean trustPreauth = !config.getTrustedServices().isEmpty(); - - boolean allowRead = trustPreauth || !config.getReadGrantServices().isEmpty(); - boolean allowWrite = trustPreauth || !config.getWriteGrantServices().isEmpty(); + boolean allowRead = config.isReadable(); + boolean allowWrite = config.isWritable(); StorageSite self = null; if (curlist.isEmpty()) { @@ -214,7 +210,6 @@ private void initStorageSite() { self.setName(name); self.setAllowRead(allowRead); self.setAllowWrite(allowWrite); - } else { throw new IllegalStateException("BUG: found " + curlist.size() + " StorageSite entries; expected 0 or 1"); } From 2f39d9e4ba30db595be00fa1c8f66f682dd9b893 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 15 Feb 2024 16:41:54 -0800 Subject: [PATCH 124/186] cadc-inventory-db: add instanceID to HarvestState improve handling of generic Class-table mapping and SQLGenerator ctors rename the SI InitDatabase class to InitDatabaseSI --- .../inventory/db/ArtifactDAOTest.java | 17 +-- .../inventory/db/EntityEventDAOTest.java | 34 +++--- .../inventory/db/HarvestStateDAOTest.java | 30 ++++- .../db/ObsoleteStorageLocationDAOTest.java | 9 +- .../inventory/db/PreauthKeyPairDAOTest.java | 7 +- .../inventory/db/StorageSiteDAOTest.java | 7 +- .../org/opencadc/vospace/db/NodeDAOTest.java | 3 +- .../opencadc/inventory/db/AbstractDAO.java | 12 +- .../opencadc/inventory/db/HarvestState.java | 10 +- .../inventory/db/HarvestStateDAO.java | 8 +- .../opencadc/inventory/db/SQLGenerator.java | 111 ++++++++++-------- ...{InitDatabase.java => InitDatabaseSI.java} | 16 +-- .../opencadc/inventory/db/version/Main.java | 2 +- .../opencadc/vospace/db/InitDatabaseVOS.java | 20 ++-- ...vestState.sql => generic.HarvestState.sql} | 1 + ...elVersion.sql => generic.ModelVersion.sql} | 0 .../main/resources/inventory.upgrade-0.15.sql | 2 + .../src/main/resources/vos.ModelVersion.sql | 9 -- ...Event.sql => vospace.DeletedNodeEvent.sql} | 0 .../{vos.Node.sql => vospace.Node.sql} | 0 20 files changed, 178 insertions(+), 120 deletions(-) rename cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/{InitDatabase.java => InitDatabaseSI.java} (92%) rename cadc-inventory-db/src/main/resources/{inventory.HarvestState.sql => generic.HarvestState.sql} (94%) rename cadc-inventory-db/src/main/resources/{inventory.ModelVersion.sql => generic.ModelVersion.sql} (100%) create mode 100644 cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql delete mode 100644 cadc-inventory-db/src/main/resources/vos.ModelVersion.sql rename cadc-inventory-db/src/main/resources/{vos.DeletedNodeEvent.sql => vospace.DeletedNodeEvent.sql} (100%) rename cadc-inventory-db/src/main/resources/{vos.Node.sql => vospace.Node.sql} (100%) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java index 6591d3af7..bad7928cc 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java @@ -97,7 +97,7 @@ import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StoredArtifactComparator; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; /** * @@ -126,22 +126,23 @@ public ArtifactDAOTest() throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); - + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); + originDAO = new ArtifactDAO(); originDAO.setConfig(config); - + nonOriginDAO = new ArtifactDAO(false); nonOriginDAO.setConfig(config); - + DBUtil.createJNDIDataSource("jdbc/ArtifactDAOTest-alt", cc); Map altConfig = new TreeMap(); altConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); altConfig.put("jndiDataSourceName", "jdbc/ArtifactDAOTest-alt"); altConfig.put("database", TestUtil.DATABASE); - altConfig.put("schema", TestUtil.SCHEMA); + altConfig.put("invSchema", TestUtil.SCHEMA); + altConfig.put("genSchema", TestUtil.SCHEMA); altDAO.setConfig(altConfig); - } catch (Exception ex) { log.error("setup failed", ex); throw ex; @@ -151,7 +152,7 @@ public ArtifactDAOTest() throws Exception { @Before public void init_cleanup() throws Exception { log.info("init database..."); - InitDatabase init = new InitDatabase(originDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(originDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); init.doInit(); log.info("init database... OK"); diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/EntityEventDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/EntityEventDAOTest.java index f6e40cf68..2dbba4be6 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/EntityEventDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/EntityEventDAOTest.java @@ -87,7 +87,7 @@ import org.opencadc.inventory.DeletedStorageLocationEvent; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.StorageLocationEvent; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; /** * @@ -106,25 +106,31 @@ public class EntityEventDAOTest { StorageLocationEventDAO slDAO = new StorageLocationEventDAO(); public EntityEventDAOTest() throws Exception { - DBConfig dbrc = new DBConfig(); - ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); - DBUtil.createJNDIDataSource("jdbc/EntityEventDAOTest", cc); - - Map config = new TreeMap(); - config.put(SQLGenerator.class.getName(), SQLGenerator.class); - config.put("jndiDataSourceName", "jdbc/EntityEventDAOTest"); - config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); - daeDAO.setConfig(config); - dslDAO.setConfig(config); - slDAO.setConfig(config); + try { + DBConfig dbrc = new DBConfig(); + ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); + DBUtil.createJNDIDataSource("jdbc/EntityEventDAOTest", cc); + + Map config = new TreeMap(); + config.put(SQLGenerator.class.getName(), SQLGenerator.class); + config.put("jndiDataSourceName", "jdbc/EntityEventDAOTest"); + config.put("database", TestUtil.DATABASE); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); + daeDAO.setConfig(config); + dslDAO.setConfig(config); + slDAO.setConfig(config); + } catch (Exception ex) { + log.error("setup failed", ex); + throw ex; + } } @Before public void setup() throws Exception { log.info("init database..."); - InitDatabase init = new InitDatabase(daeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(daeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); init.doInit(); log.info("init database... OK"); diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/HarvestStateDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/HarvestStateDAOTest.java index 7802893c8..e7a78cbc3 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/HarvestStateDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/HarvestStateDAOTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -84,7 +84,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; /** * @@ -112,7 +112,8 @@ public HarvestStateDAOTest() throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/HarvestStateDAOTest"); config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); dao.setConfig(config); } catch (Exception ex) { log.error("setup failed", ex); @@ -123,7 +124,7 @@ public HarvestStateDAOTest() throws Exception { @Before public void setup() throws Exception { log.info("init database..."); - InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); init.doInit(); log.info("init database... OK"); @@ -136,6 +137,11 @@ public void setup() throws Exception { log.info("clearing old content... OK"); } + @Test + public void noop() { + log.info("no-op - just setup"); + } + @Test public void testPutGetUpdateDelete() { try { @@ -172,6 +178,7 @@ public void testPutGetUpdateDelete() { // update hs1.curLastModified = new Date(); hs1.curID = UUID.randomUUID(); + hs1.instanceID = UUID.randomUUID(); dao.put(hs1); //log.warn("SLEEPING for lock diagnostics: 20 sec"); @@ -181,12 +188,25 @@ public void testPutGetUpdateDelete() { HarvestState hs2 = dao.get(hs1.getID()); log.info("found: " + hs2); - Assert.assertNotNull("find by uuid", hs1); + Assert.assertNotNull("find by uuid", hs2); Assert.assertNotEquals(expected.getLastModified(), hs2.getLastModified()); Assert.assertNotEquals(expected.getMetaChecksum(), hs2.getMetaChecksum()); Assert.assertEquals("round trip metachecksum", hs1.getMetaChecksum(), hs2.getMetaChecksum()); Assert.assertEquals("curLastModified", hs1.curLastModified.getTime(), hs2.curLastModified.getTime()); Assert.assertEquals("curID", hs1.curID, hs2.curID); + Assert.assertEquals("instanceID", hs1.instanceID, hs2.instanceID); + + // force update with no state change + final Date prevT = hs2.getLastModified(); + Thread.sleep(100L); + dao.put(hs2, true); + + HarvestState tup = dao.get(hs2.getID()); + Assert.assertNotNull(tup); + URI tupCS = tup.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals("meta: no change", tup.getMetaChecksum(), tupCS); + Assert.assertTrue("timestamp: update", tup.getLastModified().after(prevT)); + log.info("force update OK: " + tup); // clear tracking state hs1.curLastModified = null; diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ObsoleteStorageLocationDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ObsoleteStorageLocationDAOTest.java index c5d818b11..7f1c4c229 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ObsoleteStorageLocationDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ObsoleteStorageLocationDAOTest.java @@ -67,7 +67,6 @@ package org.opencadc.inventory.db; -import org.opencadc.inventory.ObsoleteStorageLocation; import ca.nrc.cadc.db.ConnectionConfig; import ca.nrc.cadc.db.DBConfig; import ca.nrc.cadc.db.DBUtil; @@ -83,8 +82,9 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.opencadc.inventory.ObsoleteStorageLocation; import org.opencadc.inventory.StorageLocation; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; /** * @@ -110,7 +110,8 @@ public ObsoleteStorageLocationDAOTest() throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/ArtifactDAOTest"); config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); dao.setConfig(config); } catch (Exception ex) { log.error("setup failed", ex); @@ -123,7 +124,7 @@ public void setup() throws Exception { log.info("init database..."); - InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); init.doInit(); log.info("init database... OK"); diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java index b07aedcc3..2f6e86180 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/PreauthKeyPairDAOTest.java @@ -86,7 +86,7 @@ import org.junit.Before; import org.junit.Test; import org.opencadc.inventory.PreauthKeyPair; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; /** * @@ -111,7 +111,8 @@ public PreauthKeyPairDAOTest()throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/PreauthKeyPairDAOTest"); config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); dao.setConfig(config); } @@ -120,7 +121,7 @@ public void setup() throws Exception { log.info("init database..."); - InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); init.doInit(); log.info("init database... OK"); diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/StorageSiteDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/StorageSiteDAOTest.java index 2c80b595a..ce679f1fc 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/StorageSiteDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/StorageSiteDAOTest.java @@ -84,7 +84,7 @@ import org.junit.Test; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.StorageSite; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; /** * @@ -109,7 +109,8 @@ public StorageSiteDAOTest() throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/StorageSiteDAOTest"); config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); dao.setConfig(config); } @@ -118,7 +119,7 @@ public void setup() throws Exception { log.info("init database..."); - InitDatabase init = new InitDatabase(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(dao.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); init.doInit(); log.info("init database... OK"); diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index c24341c6e..4a776c116 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -124,7 +124,8 @@ public NodeDAOTest() throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/NodeDAOTest"); config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.VOS_SCHEMA); config.put("vosSchema", TestUtil.VOS_SCHEMA); this.nodeDAO = new NodeDAO(); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java index 237668ba9..6100383b0 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/AbstractDAO.java @@ -190,8 +190,9 @@ public Map getParams() { Map ret = new TreeMap(); ret.put("jndiDataSourceName", String.class); ret.put("database", String.class); - ret.put("schema", String.class); - ret.put("vosSchema", String.class); // optional + ret.put("invSchema", String.class); + ret.put("genSchema", String.class); + ret.put("vosSchema", String.class); ret.put(SQLGenerator.class.getName(), Class.class); return ret; } @@ -224,11 +225,12 @@ public void setConfig(Map config) { } String database = (String) config.get("database"); - String schema = (String) config.get("schema"); + String invSchema = (String) config.get("invSchema"); + String genSchema = (String) config.get("genSchema"); String vosSchema = (String) config.get("vosSchema"); try { - Constructor ctor = genClass.getConstructor(String.class, String.class, String.class); - this.gen = (SQLGenerator) ctor.newInstance(database, schema, vosSchema); + Constructor ctor = genClass.getConstructor(String.class, String.class, String.class, String.class); + this.gen = (SQLGenerator) ctor.newInstance(database, invSchema, genSchema, vosSchema); } catch (Exception ex) { throw new RuntimeException("failed to instantiate SQLGenerator: " + genClass.getName(), ex); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java index 210030c66..04a0b7248 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2020. (c) 2020. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -70,7 +70,6 @@ import java.net.URI; import java.util.Date; import java.util.UUID; - import org.apache.log4j.Logger; import org.opencadc.inventory.Entity; import org.opencadc.inventory.InventoryUtil; @@ -95,6 +94,12 @@ public class HarvestState extends Entity { */ public UUID curID; + /** + * The ID of the current running instance. This is optional and only used by applications + * that share workload between instances. + */ + public UUID instanceID; + public HarvestState(String name, URI resourceID) { super(); InventoryUtil.assertNotNull(HarvestState.class, "name", name); @@ -117,6 +122,7 @@ public String getName() { } public void setName(String name) { + InventoryUtil.assertNotNull(HarvestState.class, "name", name); this.name = name; } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java index d7dd81e2b..5df563354 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java @@ -176,12 +176,16 @@ public HarvestState get(String name, URI resourceID) { @Override public void put(HarvestState val) { - if (curBufferCount < updateBufferCount) { + put(val, false); + } + + public void put(HarvestState val, boolean forceTimestampUpdate) { + if (curBufferCount < updateBufferCount && !forceTimestampUpdate) { log.debug("buffering: " + curBufferCount + " < " + updateBufferCount + " " + val); curBufferCount++; bufferedState = val; } else { - super.put(val); + super.put(val, false, forceTimestampUpdate); curBufferCount = 0; bufferedState = null; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 58f913c13..e1dffdfb1 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -123,46 +123,52 @@ public class SQLGenerator { private final Map columnMap = new TreeMap<>(new ClassComp()); protected final String database; // currently not used in SQL - protected final String schema; // may be null + protected final String invSchema; + protected final String genSchema; protected final String vosSchema; /** * Constructor. The database name is currently not used in any generated SQL; code assumes * that the DataSource is connected to the right database already and cross-database statements - * are not supported. The schema name is used to qualify table names (if set). The optional table - * name prefix is pre-pended to the table name (after the optional {schema}.) and is provided as - * a work around in shared (usually test) databases where schema name is a username and table - * names may collide with other content. Normal "production" use would specify the schema only. + * are not supported. * - * @param database database name (may be null) - * @param schema schema name (may be null) + * @param database database name: not used; future-proof + * @param invSchema inventory schema name (required, implies genSchema = invSchema) + * @param genSchema generic schema name for internal tables (PreauthKeys, HarvestState) - optional */ - public SQLGenerator(String database, String schema) { - this(database, schema, null); + public SQLGenerator(String database, String invSchema, String genSchema) { + this(database, invSchema, genSchema, null); } - public SQLGenerator(String database, String schema, String vosSchema) { + /** + * Constructor. The database name is currently not used in any generated SQL; code assumes + * that the DataSource is connected to the right database already and cross-database statements + * are not supported. The genSchema is optional (required for HarvestState and PreauthKeyPair). + * The vosSchema is optional (used for Node and DeletedNodeEvent). + * + * @param database database name: not used; future-proof + * @param invSchema inventory schema name - required + * @param genSchema generic schema name - optional + * @param vosSchema vospace schema name - optional + */ + public SQLGenerator(String database, String invSchema, String genSchema, String vosSchema) { this.database = database; - this.schema = schema; + InventoryUtil.assertNotNull(SQLGenerator.class, "invSchema", invSchema); + this.invSchema = invSchema; + this.genSchema = genSchema; this.vosSchema = vosSchema; init(); } protected void init() { - String pref = ""; - if (schema != null) { - pref = schema + "."; - } // inventory model - this.tableMap.put(Artifact.class, pref + Artifact.class.getSimpleName()); - this.tableMap.put(StorageSite.class, pref + StorageSite.class.getSimpleName()); - this.tableMap.put(DeletedArtifactEvent.class, pref + DeletedArtifactEvent.class.getSimpleName()); - this.tableMap.put(DeletedStorageLocationEvent.class, pref + DeletedStorageLocationEvent.class.getSimpleName()); - this.tableMap.put(StorageLocationEvent.class, pref + StorageLocationEvent.class.getSimpleName()); + this.tableMap.put(Artifact.class, invSchema + "." + Artifact.class.getSimpleName()); + this.tableMap.put(StorageSite.class, invSchema + "." + StorageSite.class.getSimpleName()); + this.tableMap.put(DeletedArtifactEvent.class, invSchema + "." + DeletedArtifactEvent.class.getSimpleName()); + this.tableMap.put(DeletedStorageLocationEvent.class, invSchema + "." + DeletedStorageLocationEvent.class.getSimpleName()); + this.tableMap.put(StorageLocationEvent.class, invSchema + "." + StorageLocationEvent.class.getSimpleName()); // internal - this.tableMap.put(ObsoleteStorageLocation.class, pref + ObsoleteStorageLocation.class.getSimpleName()); - this.tableMap.put(HarvestState.class, pref + HarvestState.class.getSimpleName()); - this.tableMap.put(PreauthKeyPair.class, pref + PreauthKeyPair.class.getSimpleName()); + this.tableMap.put(ObsoleteStorageLocation.class, invSchema + "." + ObsoleteStorageLocation.class.getSimpleName()); String[] cols = new String[] { "uri", // first column is logical key @@ -208,33 +214,39 @@ protected void init() { }; this.columnMap.put(ObsoleteStorageLocation.class, cols); - cols = new String[] { - "name", - "resourceID", - "curLastModified", - "curID", - "lastModified", - "metaChecksum", - "id" // last column is always PK - }; - this.columnMap.put(HarvestState.class, cols); + log.debug("genSchema: " + genSchema); + if (genSchema != null) { + // generic support + this.tableMap.put(HarvestState.class, genSchema + "." + HarvestState.class.getSimpleName()); + this.tableMap.put(PreauthKeyPair.class, genSchema + "." + PreauthKeyPair.class.getSimpleName()); + cols = new String[] { + "name", + "resourceID", + "curLastModified", + "curID", + "instanceID", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(HarvestState.class, cols); - cols = new String[] { - "name", - "publicKey", - "privateKey", - "lastModified", - "metaChecksum", - "id" // last column is always PK - }; - this.columnMap.put(PreauthKeyPair.class, cols); + cols = new String[] { + "name", + "publicKey", + "privateKey", + "lastModified", + "metaChecksum", + "id" // last column is always PK + }; + this.columnMap.put(PreauthKeyPair.class, cols); + } // optional vospace log.debug("vosSchema: " + vosSchema); if (vosSchema != null) { - pref = vosSchema + "."; - tableMap.put(Node.class, pref + Node.class.getSimpleName()); - tableMap.put(DeletedNodeEvent.class, pref + DeletedNodeEvent.class.getSimpleName()); + tableMap.put(Node.class, vosSchema + "." + Node.class.getSimpleName()); + tableMap.put(DeletedNodeEvent.class, vosSchema + "." + DeletedNodeEvent.class.getSimpleName()); cols = new String[] { "parentID", @@ -1347,6 +1359,11 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } else { prep.setNull(col++, Types.OTHER); } + if (value.instanceID != null) { + prep.setObject(col++, value.instanceID); + } else { + prep.setNull(col++, Types.OTHER); + } prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); prep.setString(col++, value.getMetaChecksum().toASCIIString()); @@ -1964,6 +1981,7 @@ public HarvestState extractData(ResultSet rs) throws SQLException, DataAccessExc final URI resourecID = Util.getURI(rs, col++); final Date curLastModified = Util.getDate(rs, col++, utc); final UUID curID = Util.getUUID(rs, col++); + final UUID instanceID = Util.getUUID(rs, col++); final Date lastModified = Util.getDate(rs, col++, utc); final URI metaChecksum = Util.getURI(rs, col++); @@ -1972,6 +1990,7 @@ public HarvestState extractData(ResultSet rs) throws SQLException, DataAccessExc HarvestState ret = new HarvestState(id, name, resourecID); ret.curLastModified = curLastModified; ret.curID = curID; + ret.instanceID = instanceID; InventoryUtil.assignLastModified(ret, lastModified); InventoryUtil.assignMetaChecksum(ret, metaChecksum); return ret; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java similarity index 92% rename from cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java rename to cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java index 46ba83171..6e810b8f3 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -67,6 +67,7 @@ package org.opencadc.inventory.db.version; +import ca.nrc.cadc.db.version.InitDatabase; import java.net.URL; import javax.sql.DataSource; import org.apache.log4j.Logger; @@ -75,8 +76,8 @@ * * @author pdowler */ -public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase { - private static final Logger log = Logger.getLogger(InitDatabase.class); +public class InitDatabaseSI extends InitDatabase { + private static final Logger log = Logger.getLogger(InitDatabaseSI.class); public static final String MODEL_NAME = "storage-inventory"; public static final String MODEL_VERSION = "0.15"; @@ -84,24 +85,25 @@ public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase { //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; static String[] CREATE_SQL = new String[] { - "inventory.ModelVersion.sql", + "generic.ModelVersion.sql", "inventory.Artifact.sql", "inventory.StorageSite.sql", "inventory.ObsoleteStorageLocation.sql", "inventory.DeletedArtifactEvent.sql", "inventory.DeletedStorageLocationEvent.sql", "inventory.StorageLocationEvent.sql", - "inventory.HarvestState.sql", + "generic.HarvestState.sql", "generic.PreauthKeyPair.sql", "generic.permissions.sql" }; static String[] UPGRADE_SQL = new String[] { + "inventory.upgrade-0.15.sql", "generic.PreauthKeyPair.sql", "generic.permissions.sql" }; - public InitDatabase(DataSource ds, String database, String schema) { + public InitDatabaseSI(DataSource ds, String database, String schema) { super(ds, database, schema, MODEL_NAME, MODEL_VERSION, PREV_MODEL_VERSION); for (String s : CREATE_SQL) { createSQL.add(s); @@ -114,6 +116,6 @@ public InitDatabase(DataSource ds, String database, String schema) { @Override protected URL findSQL(String fname) { // SQL files are stored inside the jar file - return InitDatabase.class.getClassLoader().getResource(fname); + return InitDatabaseSI.class.getClassLoader().getResource(fname); } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java index 5eddc5360..4d18205f3 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java @@ -145,7 +145,7 @@ public void run() { DataSource ds = DBUtil.getDataSource(cc); log.info("target: " + server + " " + database + " " + schema); - InitDatabase init = new InitDatabase(ds, database, schema); + InitDatabaseSI init = new InitDatabaseSI(ds, database, schema); boolean result = init.doInit(); if (result) { log.info("init: complete"); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java index ab32ba1cb..5ac9910e7 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -70,7 +70,7 @@ import java.net.URL; import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; /** * @@ -80,20 +80,20 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); public static final String MODEL_NAME = "vospace-inventory"; - public static final String MODEL_VERSION = "0.2"; - public static final String PREV_MODEL_VERSION = "0.1"; - //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; + public static final String MODEL_VERSION = "0.3"; + public static final String PREV_MODEL_VERSION = "0.2"; static String[] CREATE_SQL = new String[] { - "vos.ModelVersion.sql", - "vos.Node.sql", - "vos.DeletedNodeEvent.sql", + "generic.ModelVersion.sql", + "vospace.Node.sql", + "vospace.DeletedNodeEvent.sql", + "generic.HarvestState.sql", "generic.PreauthKeyPair.sql", "generic.permissions.sql" }; static String[] UPGRADE_SQL = new String[] { - "generic.PreauthKeyPair.sql", + "generic.HarvestState.sql", "generic.permissions.sql" }; @@ -110,6 +110,6 @@ public InitDatabaseVOS(DataSource ds, String database, String schema) { @Override protected URL findSQL(String fname) { // SQL files are stored inside the jar file - return InitDatabase.class.getClassLoader().getResource(fname); + return InitDatabaseSI.class.getClassLoader().getResource(fname); } } diff --git a/cadc-inventory-db/src/main/resources/inventory.HarvestState.sql b/cadc-inventory-db/src/main/resources/generic.HarvestState.sql similarity index 94% rename from cadc-inventory-db/src/main/resources/inventory.HarvestState.sql rename to cadc-inventory-db/src/main/resources/generic.HarvestState.sql index bccc28f9f..4a52dfcda 100644 --- a/cadc-inventory-db/src/main/resources/inventory.HarvestState.sql +++ b/cadc-inventory-db/src/main/resources/generic.HarvestState.sql @@ -3,6 +3,7 @@ create table .HarvestState ( resourceID varchar(128), curLastModified timestamp, curID uuid, + instanceID uuid, lastModified timestamp not null, metaChecksum varchar(136) not null, diff --git a/cadc-inventory-db/src/main/resources/inventory.ModelVersion.sql b/cadc-inventory-db/src/main/resources/generic.ModelVersion.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/inventory.ModelVersion.sql rename to cadc-inventory-db/src/main/resources/generic.ModelVersion.sql diff --git a/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql new file mode 100644 index 000000000..2505aaac1 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql @@ -0,0 +1,2 @@ + +alter table .HarvestState add column instanceID uuid; diff --git a/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql b/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql deleted file mode 100644 index ca9126697..000000000 --- a/cadc-inventory-db/src/main/resources/vos.ModelVersion.sql +++ /dev/null @@ -1,9 +0,0 @@ - -create table .ModelVersion -( - model varchar(32) not null primary key, - version varchar(32) not null, - lastModified timestamp not null -) -; - diff --git a/cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql b/cadc-inventory-db/src/main/resources/vospace.DeletedNodeEvent.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/vos.DeletedNodeEvent.sql rename to cadc-inventory-db/src/main/resources/vospace.DeletedNodeEvent.sql diff --git a/cadc-inventory-db/src/main/resources/vos.Node.sql b/cadc-inventory-db/src/main/resources/vospace.Node.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/vos.Node.sql rename to cadc-inventory-db/src/main/resources/vospace.Node.sql From ef87e130b1b20c8c9a8052a1672d454c9d418ffb Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 15 Feb 2024 16:46:35 -0800 Subject: [PATCH 125/186] luskan: use the DelegatingStorageManager --- luskan/README.md | 9 +++------ luskan/src/main/resources/PluginFactory.properties | 6 ++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/luskan/README.md b/luskan/README.md index dc49e7497..0d4dd2093 100644 --- a/luskan/README.md +++ b/luskan/README.md @@ -63,12 +63,9 @@ See cadc-reg ### cadc-tap-tmp.properties Temporary storage of async results is now handled by the -[cadc-tap-tmp](https://github.com/opencadc/tap/tree/master/cadc-tap-tmp) library. This -library should be configured as follows: -``` -org.opencadc.tap.tmp.TempStorageManager.baseURL = https://{server name}/{luskan path}/results -org.opencadc.tap.tmp.TempStorageManager.baseStorageDir = {local directory} -``` +[cadc-tap-tmp](https://github.com/opencadc/tap/tree/master/cadc-tap-tmp) library. `luskan` is configured +internally to use the `DelegatingStorageManager` to the config file must also specify the storage manager +to use. ### luskan.properties ``` diff --git a/luskan/src/main/resources/PluginFactory.properties b/luskan/src/main/resources/PluginFactory.properties index 3fca85cde..f8f75cc5a 100644 --- a/luskan/src/main/resources/PluginFactory.properties +++ b/luskan/src/main/resources/PluginFactory.properties @@ -19,10 +19,8 @@ ca.nrc.cadc.tap.db.DatabaseDataType=ca.nrc.cadc.tap.pg.PostgresDataTypeMapper ca.nrc.cadc.tap.writer.format.FormatFactory = org.opencadc.luskan.tap.FormatFactoryImpl -## currently need to pick this at compile -## TODO: modify cadc-tap-server to find this config file at runtime -ca.nrc.cadc.tap.ResultStore = org.opencadc.tap.tmp.TempStorageManager -#ca.nrc.cadc.tap.ResultStore = org.opencadc.tap.tmp.HttpStorageManager +# use cadc-tap-tmp +ca.nrc.cadc.tap.ResultStore = org.opencadc.tap.tmp.DelegatingStorageManager ca.nrc.cadc.tap.schema.TapSchemaDAO = org.opencadc.luskan.tap.TapSchemaDAOImpl From 438284e645e42a2a9ab593289deaf7dc6019f979 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 15 Feb 2024 16:47:43 -0800 Subject: [PATCH 126/186] minoc: adapt to db refactoring --- .../java/org/opencadc/minoc/ReplaceArtifactTest.java | 2 +- minoc/src/main/java/org/opencadc/minoc/MinocConfig.java | 4 +++- minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java b/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java index 2baae3bf0..0216cb74b 100644 --- a/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java +++ b/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java @@ -120,7 +120,7 @@ public ReplaceArtifactTest() throws Exception { Map config = new TreeMap<>(); config.put("jndiDataSourceName", "jdbc/minoc"); config.put(SQLGenerator.class.getName(), SQLGenerator.class); - config.put("schema", "inventory"); + config.put("invSchema", "inventory"); this.dao = new ArtifactDAO(); dao.setConfig(config); diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java index 53d492bf1..d4b27d9b9 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java @@ -433,7 +433,9 @@ public Map getDaoConfig() { Class clz = Class.forName(cname); ret.put(SQLGenerator.class.getName(), clz); ret.put("jndiDataSourceName", JNDI_DATASOURCE); - ret.put("schema", configProperties.getFirstPropertyValue(SCHEMA_KEY)); + ret.put("invSchema", configProperties.getFirstPropertyValue(SCHEMA_KEY)); + ret.put("genSchema", configProperties.getFirstPropertyValue(SCHEMA_KEY)); + //config.put("vosSchema", null); //config.put("database", null); return ret; } catch (ClassNotFoundException ex) { diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java index 2ca64bbbb..b8bdd677d 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocInitAction.java @@ -89,7 +89,7 @@ import org.opencadc.inventory.StorageSite; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.storage.StorageAdapter; /** @@ -173,8 +173,8 @@ private void initDatabase() { Map daoConfig = config.getDaoConfig(); DataSource ds = DBUtil.findJNDIDataSource(MinocConfig.JNDI_DATASOURCE); String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); - InitDatabase init = new InitDatabase(ds, database, schema); + String schema = (String) daoConfig.get("invSchema"); + InitDatabaseSI init = new InitDatabaseSI(ds, database, schema); init.doInit(); log.info("initDatabase: " + MinocConfig.JNDI_DATASOURCE + " " + schema + " OK"); } catch (Exception ex) { From 82279991e02ade708f2a5336f6efee6cf79ad917 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 15 Feb 2024 16:48:18 -0800 Subject: [PATCH 127/186] raven: fix content-dispisition header, adapt to db refactoring --- raven/src/intTest/java/org/opencadc/raven/FilesTest.java | 6 +++--- .../intTest/java/org/opencadc/raven/NegotiationTest.java | 4 ++-- raven/src/intTest/java/org/opencadc/raven/RavenTest.java | 3 ++- .../main/java/org/opencadc/raven/HeadFilesAction.java | 2 +- .../main/java/org/opencadc/raven/RavenInitAction.java | 9 +++++---- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/raven/src/intTest/java/org/opencadc/raven/FilesTest.java b/raven/src/intTest/java/org/opencadc/raven/FilesTest.java index 3b213b051..2e02e7453 100644 --- a/raven/src/intTest/java/org/opencadc/raven/FilesTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/FilesTest.java @@ -100,7 +100,7 @@ import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.ProtocolsGenerator; import org.opencadc.vospace.VOS; import org.opencadc.vospace.transfer.Direction; @@ -133,7 +133,7 @@ public FilesTest() throws Exception { artifactDAO.setConfig(config); this.siteDAO = new StorageSiteDAO(artifactDAO); - InitDatabase init = new InitDatabase(artifactDAO.getDataSource(), DATABASE, SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(artifactDAO.getDataSource(), DATABASE, SCHEMA); init.doInit(); } @@ -414,7 +414,7 @@ public void checkHeadResult(HttpGet request, URI artifactURI, long size, URI che Assert.assertEquals("HEAD response code", 200, request.getResponseCode()); Assert.assertEquals("File length", size, Long.valueOf(request.getResponseHeader("Content-Length")).longValue()); Assert.assertNotNull("File last-modified", request.getResponseHeader("Last-Modified")); - Assert.assertEquals("File name", "attachment; filename=\"" + InventoryUtil.computeArtifactFilename(artifactURI) + "\"", + Assert.assertEquals("File name", "inline; filename=\"" + InventoryUtil.computeArtifactFilename(artifactURI) + "\"", request.getResponseHeader("Content-Disposition")); Assert.assertEquals("File digest", checksum, request.getDigest()); Assert.assertEquals("File type", contentType, request.getResponseHeader("Content-Type")); diff --git a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java index a9b3742f5..c1191493d 100644 --- a/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java @@ -99,7 +99,7 @@ import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.ProtocolsGenerator; import org.opencadc.permissions.TokenTool; import org.opencadc.permissions.WriteGrant; @@ -131,7 +131,7 @@ public NegotiationTest() throws Exception { artifactDAO.setConfig(config); this.siteDAO = new StorageSiteDAO(artifactDAO); - InitDatabase init = new InitDatabase(artifactDAO.getDataSource(), DATABASE, SCHEMA); + InitDatabaseSI init = new InitDatabaseSI(artifactDAO.getDataSource(), DATABASE, SCHEMA); init.doInit(); } diff --git a/raven/src/intTest/java/org/opencadc/raven/RavenTest.java b/raven/src/intTest/java/org/opencadc/raven/RavenTest.java index 86239f968..cf48f07ab 100644 --- a/raven/src/intTest/java/org/opencadc/raven/RavenTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/RavenTest.java @@ -143,7 +143,8 @@ public RavenTest() throws Exception { config = new TreeMap(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/inventory"); - config.put("schema", SCHEMA); + config.put("invSchema", SCHEMA); + //config.put("genSchema", SCHEMA); } catch (Exception ex) { log.error("setup failed", ex); diff --git a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java index 4b3312eb2..69cadf54d 100644 --- a/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java +++ b/raven/src/main/java/org/opencadc/raven/HeadFilesAction.java @@ -168,7 +168,7 @@ public static void setHeaders(Artifact artifact, SyncOutput syncOutput) { syncOutput.setLastModified(artifact.getContentLastModified()); syncOutput.setHeader("Content-Length", artifact.getContentLength()); String filename = InventoryUtil.computeArtifactFilename(artifact.getURI()); - syncOutput.setHeader("Content-Disposition", "attachment; filename=\"" + filename + "\""); + syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); if (artifact.contentEncoding != null) { syncOutput.setHeader("Content-Encoding", artifact.contentEncoding); } diff --git a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java index aaee48675..53bffab91 100644 --- a/raven/src/main/java/org/opencadc/raven/RavenInitAction.java +++ b/raven/src/main/java/org/opencadc/raven/RavenInitAction.java @@ -92,7 +92,7 @@ import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.inventory.transfer.StorageSiteRule; import org.springframework.dao.DataIntegrityViolationException; @@ -168,9 +168,9 @@ private void initDatabase() { Map daoConfig = getDaoConfig(props, JNDI_ADMIN_DATASOURCE); String jndiDataSourceName = (String) daoConfig.get("jndiDataSourceName"); String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); + String schema = (String) daoConfig.get("invSchema"); DataSource ds = DBUtil.findJNDIDataSource(jndiDataSourceName); - InitDatabase init = new InitDatabase(ds, database, schema); + InitDatabaseSI init = new InitDatabaseSI(ds, database, schema); init.doInit(); log.info("initDatabase: " + jndiDataSourceName + " " + schema + " OK"); } catch (Exception ex) { @@ -344,7 +344,8 @@ static Map getDaoConfig(MultiValuedProperties props, String pool) Class clz = Class.forName(cname); ret.put(SQLGenerator.class.getName(), clz); ret.put("jndiDataSourceName", pool); - ret.put("schema", props.getFirstPropertyValue(RavenInitAction.SCHEMA_KEY)); + ret.put("invSchema", props.getFirstPropertyValue(RavenInitAction.SCHEMA_KEY)); + ret.put("genSchema", props.getFirstPropertyValue(RavenInitAction.SCHEMA_KEY)); //config.put("database", null); return ret; } catch (ClassNotFoundException ex) { From 460269e46bb6c5886f8ba80465637461f575908e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 15 Feb 2024 16:48:53 -0800 Subject: [PATCH 128/186] vault: adapt to db refactoring --- .../java/org/opencadc/vault/TransferTest.java | 2 +- .../org/opencadc/vault/VaultInitAction.java | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java index 37c236bb9..070b7aad3 100644 --- a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java @@ -122,7 +122,7 @@ public void checkGlobal() throws Exception { Map config = new TreeMap(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/inventory"); - config.put("schema", SCHEMA); + config.put("invSchema", SCHEMA); StorageSiteDAO dao = new StorageSiteDAO(false); dao.setConfig(config); diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index c253e55fb..6e62a021d 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -88,7 +88,7 @@ import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; @@ -243,8 +243,8 @@ static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now ret.put("jndiDataSourceName", VaultInitAction.JNDI_VOS_DATASOURCE); - // unused, but inventory "schema" required by cadc-inventory-db - ret.put("schema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("genSchema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); // for complete init ret.put("vosSchema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); return ret; } @@ -257,18 +257,23 @@ static Map getInvConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now ret.put("jndiDataSourceName", JNDI_INV_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("genSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // for complete init return ret; } static Map getKeyPairConfig(MultiValuedProperties props) { + return getDaoConfig(props); + /* Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now ret.put("jndiDataSourceName", JNDI_VOS_DATASOURCE); - ret.put("schema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); + ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // requied but unused + ret.put("genSchema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); return ret; + */ } - + private void initConfig() { log.info("initConfig: START"); this.props = getConfig(); @@ -300,10 +305,10 @@ private void initDatabase() { try { String dsname = (String) invDaoConfig.get("jndiDataSourceName"); - String schema = (String) invDaoConfig.get("schema"); + String schema = (String) invDaoConfig.get("invSchema"); log.info("initDatabase: " + dsname + " " + schema + " START"); DataSource ds = DBUtil.findJNDIDataSource(dsname); - InitDatabase init = new InitDatabase(ds, null, schema); + InitDatabaseSI init = new InitDatabaseSI(ds, null, schema); init.doInit(); log.info("initDatabase: " + dsname + " " + schema + " OK"); } catch (Exception ex) { From 70ecf9a887e71441fce8ce9ff028300e71910c83 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 16 Feb 2024 10:27:54 -0800 Subject: [PATCH 129/186] made genSchema required because it is needed for correct init --- .../main/java/org/opencadc/inventory/db/SQLGenerator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index e1dffdfb1..16d07a4c7 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -153,10 +153,11 @@ public SQLGenerator(String database, String invSchema, String genSchema) { */ public SQLGenerator(String database, String invSchema, String genSchema, String vosSchema) { this.database = database; - InventoryUtil.assertNotNull(SQLGenerator.class, "invSchema", invSchema); + InventoryUtil.assertNotNull(SQLGenerator.class, "invSchema", invSchema); // required for all uses this.invSchema = invSchema; + InventoryUtil.assertNotNull(SQLGenerator.class, "genSchema", genSchema); // required for correct init this.genSchema = genSchema; - this.vosSchema = vosSchema; + this.vosSchema = vosSchema; // only required for vospace init(); } From e9907248c4b942c8cbb6e30d0a2a739746457a2c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 16 Feb 2024 14:39:09 -0800 Subject: [PATCH 130/186] fenwick, ratik: adapt to refactoring in cadc-inventory-db --- fenwick/build.gradle | 4 ++-- .../opencadc/fenwick/InventoryEnvironment.java | 18 ++++++++++-------- .../opencadc/fenwick/LuskanEnvironment.java | 13 +++++++------ .../opencadc/fenwick/InventoryHarvester.java | 8 ++++---- .../main/java/org/opencadc/fenwick/Main.java | 3 ++- ratik/build.gradle | 4 ++-- .../opencadc/ratik/InventoryEnvironment.java | 9 +++++---- .../org/opencadc/ratik/LuskanEnvironment.java | 15 +++++++-------- .../org/opencadc/ratik/InventoryValidator.java | 11 ++++------- .../src/main/java/org/opencadc/ratik/Main.java | 6 ++++-- 10 files changed, 47 insertions(+), 44 deletions(-) diff --git a/fenwick/build.gradle b/fenwick/build.gradle index f61aa5320..eb9cce1e1 100644 --- a/fenwick/build.gradle +++ b/fenwick/build.gradle @@ -17,8 +17,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // temporarily limit this lib because cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15)' + // cadc-inventory-db-0.15.0 is from the vos2 feature branch + compile 'org.opencadc:cadc-inventory-db:[0.15,)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.14,1.2)' // 1.2 upper bound is correct #reasons diff --git a/fenwick/src/intTest/java/org/opencadc/fenwick/InventoryEnvironment.java b/fenwick/src/intTest/java/org/opencadc/fenwick/InventoryEnvironment.java index 6902637ae..71568adc6 100644 --- a/fenwick/src/intTest/java/org/opencadc/fenwick/InventoryEnvironment.java +++ b/fenwick/src/intTest/java/org/opencadc/fenwick/InventoryEnvironment.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2020. (c) 2020. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -81,7 +81,7 @@ import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageLocationEventDAO; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.springframework.jdbc.core.JdbcTemplate; public class InventoryEnvironment { @@ -102,8 +102,9 @@ public InventoryEnvironment() throws Exception { final DBConfig dbrc = new DBConfig(); connectionConfig = dbrc.getConnectionConfig(TestUtil.INVENTORY_SERVER, TestUtil.INVENTORY_DATABASE); daoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); - daoConfig.put("database", TestUtil.INVENTORY_DATABASE); - daoConfig.put("schema", TestUtil.INVENTORY_SCHEMA); + //daoConfig.put("database", TestUtil.INVENTORY_DATABASE); + daoConfig.put("invSchema", TestUtil.INVENTORY_SCHEMA); + daoConfig.put("genSchema", TestUtil.INVENTORY_SCHEMA); // connectionConfig and daoConfig used by InventoryHarvester to create it's own datasource Map testConfig = new TreeMap<>(); @@ -113,8 +114,9 @@ public InventoryEnvironment() throws Exception { testConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); testConfig.put("jndiDataSourceName", jndiPath); - testConfig.put("database", TestUtil.INVENTORY_DATABASE); - testConfig.put("schema", TestUtil.INVENTORY_SCHEMA); + //testConfig.put("database", TestUtil.INVENTORY_DATABASE); + testConfig.put("invSchema", TestUtil.INVENTORY_SCHEMA); + testConfig.put("genSchema", TestUtil.INVENTORY_SCHEMA); storageSiteDAO.setConfig(testConfig); artifactDAO.setConfig(testConfig); @@ -123,9 +125,9 @@ public InventoryEnvironment() throws Exception { deletedStorageLocationEventDAO.setConfig(testConfig); harvestStateDAO.setConfig(testConfig); - new InitDatabase(DBUtil.findJNDIDataSource(jndiPath), + new InitDatabaseSI(DBUtil.findJNDIDataSource(jndiPath), (String) daoConfig.get("database"), - (String) daoConfig.get("schema")).doInit(); + (String) daoConfig.get("invSchema")).doInit(); } void cleanTestEnvironment() throws Exception { diff --git a/fenwick/src/intTest/java/org/opencadc/fenwick/LuskanEnvironment.java b/fenwick/src/intTest/java/org/opencadc/fenwick/LuskanEnvironment.java index ae9f1c25f..ef882cc20 100644 --- a/fenwick/src/intTest/java/org/opencadc/fenwick/LuskanEnvironment.java +++ b/fenwick/src/intTest/java/org/opencadc/fenwick/LuskanEnvironment.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2020. (c) 2020. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -80,7 +80,7 @@ import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageLocationEventDAO; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.springframework.jdbc.core.JdbcTemplate; @@ -106,8 +106,9 @@ public LuskanEnvironment() throws Exception { final Map daoConfig = new TreeMap<>(); daoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); daoConfig.put("jndiDataSourceName", jndiPath); - daoConfig.put("database", TestUtil.LUSKAN_DATABASE); - daoConfig.put("schema", TestUtil.LUSKAN_SCHEMA); + //daoConfig.put("database", TestUtil.LUSKAN_DATABASE); + daoConfig.put("invSchema", TestUtil.LUSKAN_SCHEMA); + daoConfig.put("genSchema", TestUtil.LUSKAN_SCHEMA); storageSiteDAO.setConfig(daoConfig); artifactDAO.setConfig(daoConfig); @@ -115,9 +116,9 @@ public LuskanEnvironment() throws Exception { deletedArtifactEventDAO.setConfig(daoConfig); deletedStorageLocationEventDAO.setConfig(daoConfig); - new InitDatabase(DBUtil.findJNDIDataSource(jndiPath), + new InitDatabaseSI(DBUtil.findJNDIDataSource(jndiPath), (String) daoConfig.get("database"), - (String) daoConfig.get("schema")).doInit(); + (String) daoConfig.get("invSchema")).doInit(); } void cleanTestEnvironment() throws Exception { diff --git a/fenwick/src/main/java/org/opencadc/fenwick/InventoryHarvester.java b/fenwick/src/main/java/org/opencadc/fenwick/InventoryHarvester.java index 157c9dba9..fc827b964 100644 --- a/fenwick/src/main/java/org/opencadc/fenwick/InventoryHarvester.java +++ b/fenwick/src/main/java/org/opencadc/fenwick/InventoryHarvester.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2022. (c) 2022. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -86,7 +86,7 @@ import org.opencadc.inventory.StorageSite; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.util.ArtifactSelector; /** @@ -155,9 +155,9 @@ public InventoryHarvester(Map daoConfig, ConnectionConfig connec this.artifactDAO = new ArtifactDAO(storageSiteDAO); String database = (String) dconf.get("database"); - String schema = (String) dconf.get("schema"); + String schema = (String) dconf.get("invSchema"); DataSource ds = DBUtil.findJNDIDataSource(dsname); - InitDatabase init = new InitDatabase(ds, database, schema); + InitDatabaseSI init = new InitDatabaseSI(ds, database, schema); init.doInit(); log.info("initDatabase: " + schema + " OK"); diff --git a/fenwick/src/main/java/org/opencadc/fenwick/Main.java b/fenwick/src/main/java/org/opencadc/fenwick/Main.java index a1dcfea12..15670ed78 100644 --- a/fenwick/src/main/java/org/opencadc/fenwick/Main.java +++ b/fenwick/src/main/java/org/opencadc/fenwick/Main.java @@ -160,7 +160,8 @@ public static void main(final String[] args) { final String password = props.getFirstPropertyValue(DB_PASSWORD_CONFIG_KEY); final String dbUrl = props.getFirstPropertyValue(DB_URL_CONFIG_KEY); - daoConfig.put("schema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); + daoConfig.put("invSchema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); + daoConfig.put("genSchema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); final ConnectionConfig cc = new ConnectionConfig(null, null, username, password, "org.postgresql.Driver", dbUrl); diff --git a/ratik/build.gradle b/ratik/build.gradle index feaeb51bd..6eef59f99 100644 --- a/ratik/build.gradle +++ b/ratik/build.gradle @@ -17,8 +17,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // temporarily limit this lib because cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.14.6,0.15)' + // cadc-inventory-db-0.15.0 is from the vos2 feature branch + compile 'org.opencadc:cadc-inventory-db:[0.15,)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.15,2.0)' diff --git a/ratik/src/intTest/java/org/opencadc/ratik/InventoryEnvironment.java b/ratik/src/intTest/java/org/opencadc/ratik/InventoryEnvironment.java index 8fdd021d7..95820b9e2 100644 --- a/ratik/src/intTest/java/org/opencadc/ratik/InventoryEnvironment.java +++ b/ratik/src/intTest/java/org/opencadc/ratik/InventoryEnvironment.java @@ -78,7 +78,7 @@ import org.opencadc.inventory.db.DeletedStorageLocationEventDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.springframework.jdbc.core.JdbcTemplate; public class InventoryEnvironment { @@ -101,7 +101,8 @@ public InventoryEnvironment() throws Exception { daoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); daoConfig.put("database", TestUtil.INVENTORY_DATABASE); - daoConfig.put("schema", TestUtil.INVENTORY_SCHEMA); + daoConfig.put("invSchema", TestUtil.INVENTORY_SCHEMA); + daoConfig.put("genSchema", TestUtil.INVENTORY_SCHEMA); daoConfig.put("jndiDataSourceName", jndiPath); storageSiteDAO.setConfig(daoConfig); @@ -110,9 +111,9 @@ public InventoryEnvironment() throws Exception { deletedArtifactEventDAO.setConfig(daoConfig); deletedStorageLocationEventDAO.setConfig(daoConfig); - new InitDatabase(DBUtil.findJNDIDataSource(jndiPath), + new InitDatabaseSI(DBUtil.findJNDIDataSource(jndiPath), (String) daoConfig.get("database"), - (String) daoConfig.get("schema")).doInit(); + (String) daoConfig.get("invSchema")).doInit(); daoConfig.remove("jndiDataSourceName"); } diff --git a/ratik/src/intTest/java/org/opencadc/ratik/LuskanEnvironment.java b/ratik/src/intTest/java/org/opencadc/ratik/LuskanEnvironment.java index 965391993..b15026e02 100644 --- a/ratik/src/intTest/java/org/opencadc/ratik/LuskanEnvironment.java +++ b/ratik/src/intTest/java/org/opencadc/ratik/LuskanEnvironment.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2020. (c) 2020. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -71,19 +71,17 @@ import ca.nrc.cadc.db.ConnectionConfig; import ca.nrc.cadc.db.DBConfig; import ca.nrc.cadc.db.DBUtil; - import java.net.URI; import java.util.Map; import java.util.TreeMap; - import org.apache.log4j.Logger; import org.opencadc.inventory.StorageSite; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedArtifactEventDAO; import org.opencadc.inventory.db.DeletedStorageLocationEventDAO; -import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.springframework.jdbc.core.JdbcTemplate; public class LuskanEnvironment { @@ -109,7 +107,8 @@ public LuskanEnvironment() throws Exception { daoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); daoConfig.put("jndiDataSourceName", jndiPath); daoConfig.put("database", TestUtil.LUSKAN_DATABASE); - daoConfig.put("schema", TestUtil.LUSKAN_SCHEMA); + daoConfig.put("invSchema", TestUtil.LUSKAN_SCHEMA); + daoConfig.put("genSchema", TestUtil.LUSKAN_SCHEMA); storageSiteDAO.setConfig(daoConfig); artifactDAO.setConfig(daoConfig); @@ -118,9 +117,9 @@ public LuskanEnvironment() throws Exception { globalArtifactDAO.setConfig(daoConfig); - new InitDatabase(DBUtil.findJNDIDataSource(jndiPath), + new InitDatabaseSI(DBUtil.findJNDIDataSource(jndiPath), (String) daoConfig.get("database"), - (String) daoConfig.get("schema")).doInit(); + (String) daoConfig.get("invSchema")).doInit(); } void cleanTestEnvironment() throws Exception { diff --git a/ratik/src/main/java/org/opencadc/ratik/InventoryValidator.java b/ratik/src/main/java/org/opencadc/ratik/InventoryValidator.java index f8dcc78c5..db4b17291 100644 --- a/ratik/src/main/java/org/opencadc/ratik/InventoryValidator.java +++ b/ratik/src/main/java/org/opencadc/ratik/InventoryValidator.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2023. (c) 2023. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -79,7 +79,6 @@ import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.BucketSelector; import ca.nrc.cadc.util.StringUtil; - import java.io.File; import java.io.IOException; import java.net.URI; @@ -95,18 +94,16 @@ import java.util.Map; import java.util.TreeMap; import java.util.UUID; - import javax.naming.NamingException; import javax.security.auth.Subject; import javax.sql.DataSource; - import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.StorageSite; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.StorageSiteDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.query.ArtifactRowMapper; import org.opencadc.inventory.util.ArtifactSelector; import org.opencadc.tap.TapClient; @@ -185,9 +182,9 @@ public InventoryValidator(ConnectionConfig connectionConfig, Map try { String jndiDataSourceName = (String) daoConfig.get("jndiDataSourceName"); String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); + String schema = (String) daoConfig.get("invSchema"); DataSource ds = DBUtil.findJNDIDataSource(jndiDataSourceName); - InitDatabase init = new InitDatabase(ds, database, schema); + InitDatabaseSI init = new InitDatabaseSI(ds, database, schema); init.doInit(); log.info(String.format("initDatabase: %s %s", jndiDataSourceName, schema)); } catch (Exception ex) { diff --git a/ratik/src/main/java/org/opencadc/ratik/Main.java b/ratik/src/main/java/org/opencadc/ratik/Main.java index e4fb592ab..c3d2afdd8 100644 --- a/ratik/src/main/java/org/opencadc/ratik/Main.java +++ b/ratik/src/main/java/org/opencadc/ratik/Main.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2020. (c) 2020. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -158,7 +158,9 @@ public static void main(final String[] args) { final String password = props.getFirstPropertyValue(DB_PASSWORD_CONFIG_KEY); final String dbUrl = props.getFirstPropertyValue(DB_URL_CONFIG_KEY); - daoConfig.put("schema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); + //daoConfig.put("database",...); + daoConfig.put("invSchema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); + daoConfig.put("genSchema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); final ConnectionConfig cc = new ConnectionConfig(null, null, username, password, "org.postgresql.Driver", dbUrl); From fb484a60d2d37b456151dd4d4d8b43e463440a9b Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 16 Feb 2024 14:42:21 -0800 Subject: [PATCH 131/186] cadc-inventory-server: remove unused InitDatabaseAction --- .../inventory/server/InitDatabaseAction.java | 116 ------------------ 1 file changed, 116 deletions(-) delete mode 100644 cadc-inventory-server/src/main/java/org/opencadc/inventory/server/InitDatabaseAction.java diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/server/InitDatabaseAction.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/server/InitDatabaseAction.java deleted file mode 100644 index 6236091a3..000000000 --- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/server/InitDatabaseAction.java +++ /dev/null @@ -1,116 +0,0 @@ -/* -************************************************************************ -******************* CANADIAN ASTRONOMY DATA CENTRE ******************* -************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -* -* (c) 2020. (c) 2020. -* Government of Canada Gouvernement du Canada -* National Research Council Conseil national de recherches -* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -* All rights reserved Tous droits réservés -* -* NRC disclaims any warranties, Le CNRC dénie toute garantie -* expressed, implied, or énoncée, implicite ou légale, -* statutory, of any kind with de quelque nature que ce -* respect to the software, soit, concernant le logiciel, -* including without limitation y compris sans restriction -* any warranty of merchantability toute garantie de valeur -* or fitness for a particular marchande ou de pertinence -* purpose. NRC shall not be pour un usage particulier. -* liable in any event for any Le CNRC ne pourra en aucun cas -* damages, whether direct or être tenu responsable de tout -* indirect, special or general, dommage, direct ou indirect, -* consequential or incidental, particulier ou général, -* arising from the use of the accessoire ou fortuit, résultant -* software. Neither the name de l'utilisation du logiciel. Ni -* of the National Research le nom du Conseil National de -* Council of Canada nor the Recherches du Canada ni les noms -* names of its contributors may de ses participants ne peuvent -* be used to endorse or promote être utilisés pour approuver ou -* products derived from this promouvoir les produits dérivés -* software without specific prior de ce logiciel sans autorisation -* written permission. préalable et particulière -* par écrit. -* -* This file is part of the Ce fichier fait partie du projet -* OpenCADC project. OpenCADC. -* -* OpenCADC is free software: OpenCADC est un logiciel libre ; -* you can redistribute it and/or vous pouvez le redistribuer ou le -* modify it under the terms of modifier suivant les termes de -* the GNU Affero General Public la “GNU Affero General Public -* License as published by the License” telle que publiée -* Free Software Foundation, par la Free Software Foundation -* either version 3 of the : soit la version 3 de cette -* License, or (at your option) licence, soit (à votre gré) -* any later version. toute version ultérieure. -* -* OpenCADC is distributed in the OpenCADC est distribué -* hope that it will be useful, dans l’espoir qu’il vous -* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -* without even the implied GARANTIE : sans même la garantie -* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence -* General Public License for Générale Publique GNU Affero -* more details. pour plus de détails. -* -* You should have received Vous devriez avoir reçu une -* a copy of the GNU Affero copie de la Licence Générale -* General Public License along Publique GNU Affero avec -* with OpenCADC. If not, see OpenCADC ; si ce n’est -* . pas le cas, consultez : -* . -* -************************************************************************ -*/ - -package org.opencadc.inventory.server; - -import ca.nrc.cadc.db.DBUtil; -import ca.nrc.cadc.rest.InitAction; -import java.util.Map; -import java.util.TreeMap; -import javax.sql.DataSource; -import org.apache.log4j.Logger; -import org.opencadc.inventory.db.version.InitDatabase; - -/** - * Base class for storage service database initialisation. - * - * @author pdowler - */ -public abstract class InitDatabaseAction extends InitAction { - private static final Logger log = Logger.getLogger(InitDatabaseAction.class); - - protected final Map daoConfig = new TreeMap<>(); - - protected InitDatabaseAction() { - } - - @Override - public void doInit() { - initDaoConfig(); - initDatabase(); - } - - /** - * Add content to the (protected) daoConfig map. - */ - protected abstract void initDaoConfig(); - - private void initDatabase() { - log.info("initDatabase: START"); - try { - String jndiDataSourceName = (String) daoConfig.get("jndiDataSourceName"); - String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); - DataSource ds = DBUtil.findJNDIDataSource(jndiDataSourceName); - InitDatabase init = new InitDatabase(ds, database, schema); - init.doInit(); - log.info("initDatabase: " + jndiDataSourceName + " " + schema + " OK"); - } catch (Exception ex) { - throw new IllegalStateException("check/init database failed", ex); - } - } -} From 1b692bc6e13a1eb7ab5c2952872616615d0bd504 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 16 Feb 2024 14:59:43 -0800 Subject: [PATCH 132/186] critwall: adapt to cadc-inventory-db refactoring port to cadc-vos-2.0 --- critwall/build.gradle | 6 +++--- .../org/opencadc/critwall/FileSyncJobTest.java | 7 ++++--- .../java/org/opencadc/critwall/FileSyncTest.java | 7 ++++--- .../java/org/opencadc/critwall/FileSync.java | 8 ++++---- .../java/org/opencadc/critwall/FileSyncJob.java | 16 ++++++++-------- .../main/java/org/opencadc/critwall/Main.java | 3 ++- 6 files changed, 25 insertions(+), 22 deletions(-) diff --git a/critwall/build.gradle b/critwall/build.gradle index 14aba0e89..9b9a74cd9 100644 --- a/critwall/build.gradle +++ b/critwall/build.gradle @@ -21,12 +21,12 @@ mainClassName = 'org.opencadc.critwall.Main' dependencies { compile 'org.opencadc:cadc-storage-adapter:[0.8,1.0)' compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' + compile 'org.opencadc:cadc-inventory:[0.10,)' // cadc-inventory-db-0.15 is in the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15.0)' + compile 'org.opencadc:cadc-inventory-db:[0.15.0,)' compile 'org.opencadc:cadc-registry:[1.7,2.0)' compile 'org.opencadc:cadc-vosi:[1.3.6,2.0)' - compile 'org.opencadc:cadc-vos:[1.2,2.0)' + compile 'org.opencadc:cadc-vos:[2.0,)' runtime 'org.opencadc:cadc-storage-adapter-fs:[0.7,)' runtime 'org.opencadc:cadc-storage-adapter-swift:[0.6,)' diff --git a/critwall/src/intTest/java/org/opencadc/critwall/FileSyncJobTest.java b/critwall/src/intTest/java/org/opencadc/critwall/FileSyncJobTest.java index ed3a8edcb..54ddfeee3 100644 --- a/critwall/src/intTest/java/org/opencadc/critwall/FileSyncJobTest.java +++ b/critwall/src/intTest/java/org/opencadc/critwall/FileSyncJobTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2021. (c) 2021. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -133,8 +133,9 @@ public FileSyncJobTest() throws Exception { Map config = new TreeMap(); config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/FileSyncJobTest"); - config.put("database", TestUtil.DATABASE); - config.put("schema", TestUtil.SCHEMA); + //config.put("database", TestUtil.DATABASE); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); dao.setConfig(config); String testDir = TEST_ROOT + File.separator + "testValidJob"; diff --git a/critwall/src/intTest/java/org/opencadc/critwall/FileSyncTest.java b/critwall/src/intTest/java/org/opencadc/critwall/FileSyncTest.java index e9a24e095..170a828ac 100644 --- a/critwall/src/intTest/java/org/opencadc/critwall/FileSyncTest.java +++ b/critwall/src/intTest/java/org/opencadc/critwall/FileSyncTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2020. (c) 2020. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -133,8 +133,9 @@ public FileSyncTest() throws Exception { daoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); daoConfig.put("jndiDataSourceName", "jdbc/FileSyncTest"); - daoConfig.put("database", TestUtil.DATABASE); - daoConfig.put("schema", TestUtil.SCHEMA); + //daoConfig.put("database", TestUtil.DATABASE); + daoConfig.put("invSchema", TestUtil.SCHEMA); + daoConfig.put("genSchema", TestUtil.SCHEMA); dao.setConfig(daoConfig); String testDir = TEST_ROOT + File.separator + "testFileSync"; diff --git a/critwall/src/main/java/org/opencadc/critwall/FileSync.java b/critwall/src/main/java/org/opencadc/critwall/FileSync.java index 797f55571..231564e9c 100644 --- a/critwall/src/main/java/org/opencadc/critwall/FileSync.java +++ b/critwall/src/main/java/org/opencadc/critwall/FileSync.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2020. (c) 2020. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -91,7 +91,7 @@ import org.opencadc.inventory.Artifact; import org.opencadc.inventory.InventoryUtil; import org.opencadc.inventory.db.ArtifactDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.storage.StorageAdapter; @@ -170,9 +170,9 @@ public FileSync(Map daoConfig, ConnectionConfig connectionConfig try { String database = null; // unused (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); + String schema = (String) daoConfig.get("invSchema"); DataSource ds = ca.nrc.cadc.db.DBUtil.findJNDIDataSource(fileSyncDS); - InitDatabase init = new InitDatabase(ds, database, schema); + InitDatabaseSI init = new InitDatabaseSI(ds, database, schema); init.doInit(); log.info("initDatabase: " + schema + " OK"); } catch (Exception ex) { diff --git a/critwall/src/main/java/org/opencadc/critwall/FileSyncJob.java b/critwall/src/main/java/org/opencadc/critwall/FileSyncJob.java index a83e9381b..20c6c3732 100644 --- a/critwall/src/main/java/org/opencadc/critwall/FileSyncJob.java +++ b/critwall/src/main/java/org/opencadc/critwall/FileSyncJob.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2022. (c) 2022. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -84,13 +84,6 @@ import ca.nrc.cadc.net.TransientException; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.reg.client.RegistryClient; -import ca.nrc.cadc.vos.Direction; -import ca.nrc.cadc.vos.Protocol; -import ca.nrc.cadc.vos.Transfer; -import ca.nrc.cadc.vos.TransferParsingException; -import ca.nrc.cadc.vos.TransferReader; -import ca.nrc.cadc.vos.TransferWriter; -import ca.nrc.cadc.vos.VOS; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.MalformedURLException; @@ -118,6 +111,13 @@ import org.opencadc.inventory.storage.StorageAdapter; import org.opencadc.inventory.storage.StorageEngageException; import org.opencadc.inventory.storage.StorageMetadata; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; +import org.opencadc.vospace.transfer.TransferParsingException; +import org.opencadc.vospace.transfer.TransferReader; +import org.opencadc.vospace.transfer.TransferWriter; /** * Single file sync instance. diff --git a/critwall/src/main/java/org/opencadc/critwall/Main.java b/critwall/src/main/java/org/opencadc/critwall/Main.java index f9af65789..dc4c7a8fd 100644 --- a/critwall/src/main/java/org/opencadc/critwall/Main.java +++ b/critwall/src/main/java/org/opencadc/critwall/Main.java @@ -187,7 +187,8 @@ public static void main(String[] args) { // populate/assign values to pass to FileSync Map daoConfig = new TreeMap<>(); - daoConfig.put("schema", schema); + daoConfig.put("invSchema", schema); + daoConfig.put("genSchema", schema); // needed for correct init try { daoConfig.put(SQLGENERATOR_CONFIG_KEY, Class.forName(generatorName)); From f197591055f70a414ba0707b94fceda883f99094 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 16 Feb 2024 15:03:38 -0800 Subject: [PATCH 133/186] tantar: update for cadc-inventory-db refactoring --- tantar/build.gradle | 4 ++-- .../src/intTest/java/org/opencadc/tantar/TantarTest.java | 3 ++- .../main/java/org/opencadc/tantar/BucketValidator.java | 8 ++++---- tantar/src/main/java/org/opencadc/tantar/Main.java | 5 +++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tantar/build.gradle b/tantar/build.gradle index c59d7786e..c99751a61 100644 --- a/tantar/build.gradle +++ b/tantar/build.gradle @@ -21,8 +21,8 @@ mainClassName = 'org.opencadc.tantar.Main' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' compile 'org.opencadc:cadc-log:[1.1.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,1.0)' + compile 'org.opencadc:cadc-inventory:[0.10,)' + compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-storage-adapter:[0.11.1,1.0)' diff --git a/tantar/src/intTest/java/org/opencadc/tantar/TantarTest.java b/tantar/src/intTest/java/org/opencadc/tantar/TantarTest.java index 41c290ed1..d4c362da9 100644 --- a/tantar/src/intTest/java/org/opencadc/tantar/TantarTest.java +++ b/tantar/src/intTest/java/org/opencadc/tantar/TantarTest.java @@ -142,7 +142,8 @@ protected TantarTest(ResolutionPolicy policy, boolean includeRecoverable) throws Map daoConfig = new TreeMap<>(); daoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); - daoConfig.put("schema", "inventory"); + daoConfig.put("invSchema", "inventory"); + daoConfig.put("genSchema", "inventory"); this.validator = new BucketValidator(daoConfig, cc, preservingAdapter, policy, "0-f", false); diff --git a/tantar/src/main/java/org/opencadc/tantar/BucketValidator.java b/tantar/src/main/java/org/opencadc/tantar/BucketValidator.java index 291c623e1..91498c464 100644 --- a/tantar/src/main/java/org/opencadc/tantar/BucketValidator.java +++ b/tantar/src/main/java/org/opencadc/tantar/BucketValidator.java @@ -4,7 +4,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2022. (c) 2022. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -95,7 +95,7 @@ import org.opencadc.inventory.db.DeletedStorageLocationEventDAO; import org.opencadc.inventory.db.ObsoleteStorageLocationDAO; import org.opencadc.inventory.db.StorageLocationEventDAO; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.storage.StorageAdapter; import org.opencadc.inventory.storage.StorageEngageException; import org.opencadc.inventory.storage.StorageMetadata; @@ -193,9 +193,9 @@ public BucketValidator(Map daoConfig, ConnectionConfig connectio try { String database = (String) daoConfig.get("database"); - String schema = (String) daoConfig.get("schema"); + String schema = (String) daoConfig.get("invSchema"); DataSource ds = ca.nrc.cadc.db.DBUtil.findJNDIDataSource("jdbc/inventory"); - InitDatabase init = new InitDatabase(ds, database, schema); + InitDatabaseSI init = new InitDatabaseSI(ds, database, schema); init.doInit(); log.info("initDatabase: " + schema + " OK"); } catch (Exception ex) { diff --git a/tantar/src/main/java/org/opencadc/tantar/Main.java b/tantar/src/main/java/org/opencadc/tantar/Main.java index 228d3d438..a326e2832 100644 --- a/tantar/src/main/java/org/opencadc/tantar/Main.java +++ b/tantar/src/main/java/org/opencadc/tantar/Main.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * - * (c) 2020. (c) 2020. + * (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -193,7 +193,8 @@ public static void main(final String[] args) { final String jdbcDriverClassname = "org.postgresql.Driver"; final String schemaName = props.getFirstPropertyValue(DB_SCHEMA_KEY); if (StringUtil.hasLength(schemaName)) { - daoConfig.put("schema", schemaName); + daoConfig.put("invSchema", schemaName); + daoConfig.put("genSchema", schemaName); } else { throw new InvalidConfigException("required config property missing: " + DB_SCHEMA_KEY); } From bd7e186e2b0365e6a85deb0b252eea48a01cdaf2 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 16 Feb 2024 15:11:52 -0800 Subject: [PATCH 134/186] ringhold: update for cadc-inventory-db refactoring --- ringhold/build.gradle | 4 ++-- .../org/opencadc/ringhold/InventoryValidatorTest.java | 8 +++++--- ringhold/src/main/java/org/opencadc/ringhold/Main.java | 6 ++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ringhold/build.gradle b/ringhold/build.gradle index e205a6e76..e29f2e50c 100644 --- a/ringhold/build.gradle +++ b/ringhold/build.gradle @@ -16,8 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.6,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14,1.0)' + compile 'org.opencadc:cadc-inventory:[0.10,)' + compile 'org.opencadc:cadc-inventory-db:[0.15,1.0)' testCompile 'junit:junit:[4.12,5.0)' } diff --git a/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java b/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java index 37fffaaa4..0a809c6fb 100644 --- a/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java +++ b/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java @@ -72,6 +72,7 @@ import ca.nrc.cadc.db.ConnectionConfig; import ca.nrc.cadc.db.DBConfig; import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.db.version.InitDatabase; import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.HexUtil; import ca.nrc.cadc.util.Log4jInit; @@ -101,7 +102,7 @@ import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedStorageLocationEventDAO; import org.opencadc.inventory.db.SQLGenerator; -import org.opencadc.inventory.db.version.InitDatabase; +import org.opencadc.inventory.db.version.InitDatabaseSI; import org.springframework.jdbc.core.JdbcTemplate; /** @@ -168,7 +169,7 @@ public InventoryValidatorTest() throws Exception { try { DataSource dataSource = DBUtil.findJNDIDataSource(jndiPath); - InitDatabase init = new InitDatabase(dataSource, INVENTORY_DATABASE, INVENTORY_SCHEMA); + InitDatabase init = new InitDatabaseSI(dataSource, INVENTORY_DATABASE, INVENTORY_SCHEMA); init.doInit(); log.debug("initDatabase: " + jndiPath + " " + INVENTORY_SCHEMA + " OK"); } catch (Exception ex) { @@ -178,7 +179,8 @@ public InventoryValidatorTest() throws Exception { daoConfig.put(SQLGenerator.class.getName(), SQLGenerator.class); daoConfig.put("jndiDataSourceName", jndiPath); daoConfig.put("database", INVENTORY_DATABASE); - daoConfig.put("schema", INVENTORY_SCHEMA); + daoConfig.put("invSchema", INVENTORY_SCHEMA); + daoConfig.put("genSchema", INVENTORY_SCHEMA); artifactDAO.setConfig(daoConfig); deletedStorageLocationEventDAO.setConfig(daoConfig); diff --git a/ringhold/src/main/java/org/opencadc/ringhold/Main.java b/ringhold/src/main/java/org/opencadc/ringhold/Main.java index e09a3410c..b774ba256 100644 --- a/ringhold/src/main/java/org/opencadc/ringhold/Main.java +++ b/ringhold/src/main/java/org/opencadc/ringhold/Main.java @@ -151,13 +151,15 @@ public static void main(final String[] args) { } final Map daoConfig = new TreeMap<>(); - daoConfig.put("schema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); + daoConfig.put("invSchema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); + daoConfig.put("genSchema", props.getFirstPropertyValue(DB_SCHEMA_CONFIG_KEY)); daoConfig.put("jndiDataSourceName", "jdbc/inventory-txn"); final String configuredSQLGenerator = props.getFirstPropertyValue(SQLGENERATOR_CONFIG_KEY); daoConfig.put(SQLGENERATOR_CONFIG_KEY, Class.forName(configuredSQLGenerator)); final Map iterConfig = new TreeMap<>(); - iterConfig.put("schema", daoConfig.get("schema")); + iterConfig.put("invSchema", daoConfig.get("schema")); + iterConfig.put("genSchema", daoConfig.get("schema")); iterConfig.put("jndiDataSourceName", "jdbc/inventory-iter"); iterConfig.put(SQLGENERATOR_CONFIG_KEY, Class.forName(configuredSQLGenerator)); From 479b0b7a3b63b22914be5f3ebbfe5df652d771fc Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 19 Feb 2024 10:49:37 -0800 Subject: [PATCH 135/186] update vault-quota/Design --- vault-quota/Design.md | 103 +++++++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 32 deletions(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index af9b5cc13..e5ef4a6ea 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -23,9 +23,9 @@ for each new Artifact: query for DataNode (storageID = artifact.uri) if Artifact.contentLength != Node.size: start txn + lock parent lock datanode compute delta - lock parent apply delta to parent.delta set dataNode.size update HarvestState @@ -41,73 +41,112 @@ ContainerNode.delta since DataNode(s) never have a delta. query for ContainerNode with non-zero delta for each ContainerNode: start txn + lock parent lock containernode re-check delta - lock parent apply delta to parent.delta apply delta containernode.size, set containernode.delta=0 commit txn ``` -The above sequence finds candidate propagations, locks (order: child-then-parent as above), +The above sequence finds candidate propagations, locks (order: parent-child), and applies the propagation. This moves the outstanding delta up the tree one level. If the sequence acts on multiple child containers before the parent, the delta(s) naturally -_merge_ and there are fewer larger delta propagations in the upper part of the tree. It would -be optimal to do propagations depth-first but it doesn't seem practical to forcibly accomplish -that ordering. +_merge_ and there are fewer larger delta propagations in the upper part of the tree. -Container size propagation will be implemented as a single sequence (thread). We could add -something to the vospace.Node table to support subdividing work and enable multiple threads, -but there is nothing there right now. +The most generic implementation is to iterate over container nodes: +``` +Iterator iter = nodeDAO.containerIterator(boolean nonZeroDelta); +``` +It would be optimal to do propagations from the bottom upwards in order to "merge" them, +but it doesn't seem practical to forcibly accomplish that ordering. Container size propagation +will be implemented as a single sequence (thread). We could add something to the vospace.Node +table to support subdividing work and enable multiple threads, but there is nothing there right +now and it might not be necessary. ## validation -### ContainerNode vs child nodes discrepancies -TODO: figure out how to validate ContainerNode sizes vs sum(child sizes) in a live system - ### DataNode vs Artifact discrepancies -These can be validated in parallel by multiple threads, subdivide work by bucket. +These can be validated in parallel by multiple threads, subdivide work by bucket if we add +DataNode.storageBucket (== Artifact.uriBucket). ``` -discrepancy 1: Artifact exists but DataNode does not +discrepancy: Artifact exists but DataNode does not explanation: DataNode created, transfer negotiated, DataNode removed, transfer executed -evidence: check for DeletedNodeEvent +evidence: DeletedNodeEvent exists action: remove artifact, create DeletedArtifactEvent -else: ?? -discrepancy 2: DataNode exists but Artifact does not -explanation: DataNode created, Artifact never (successfully) put -evidence: -action: set nodeSize = 0 +discrepancy: Artifact exists but DataNode does not +explanation: DataNode created, Artifact put, DataNode deleted, Artifact delete failed +evidence: only possible with singlePool==false +action: remove artifact, create DeletedArtifactEvent -discrepancy 3: DataNode exists but Artifact does not +discrepancy: DataNode exists but Artifact does not +explanation: DataNode created, Artifact never (successfully) put (normal) +evidence: DataNode.nodeSize == 0 or null +action: none + +discrepancy: DataNode exists but Artifact does not explanation: deleted or lost Artifact -evidence: DataNode.size != 0 (deleted vs lost: DeletedArtifactEvent exists) -action: fix nodeSize +evidence: DataNode.nodeSize != 0 (deleted vs lost: DeletedArtifactEvent exists) +action: lock nodes, fix dataNode and propagate delta to parent -discrepancy 4: nodeSize != Artifact.contentLength -explanation: pending/missed Artifact event -action: fix DataNode and propagate delta to parent ContainerNode (same as incremental) +discrepancy: DataNode.nodeSize != Artifact.contentLength +explanation: artifact written (if DataNode.size > 0: replaced) +action: lock nodes, fix DataNode and propagate delta to parent ``` +Required lock order: child-parent or parent-child OK. The most generic implementation is a merge join of two iterators (see ratik, tantar): ``` -Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); // artifact.uri order -Iterator niter = nodeDAO.iterator(vaultNamespace, bucket); // storageID order +Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); // uriBucket,uri order +Iterator niter = nodeDAO.iterator(bucket); // storageBucket,storageID order +``` + +### ContainerNode vs child nodes discrepancies +These can be validated in +``` +discrepancy 1: container size > sum(child sizes) +explanation: un-propagated delete +evidence: sum(child delta) < 0 +action: none + +discrepancy 1: container size > sum(child sizes) +explanation: bug +evidence: sum(child delta) == 0 +action: fix container size, set container.delta + +discrepancy 1: container size < sum(child sizes) +explanation: un-propagated delta +evidence: sum(child delta) > 0 +action: none + +discrepancy 1: container size < sum(child sizes) +explanation: bug +evidence: sum(child delta) == 0 +action: fix container size, set container.delta +``` +Required lock order: locks the parent of a parent-children relationship. + +The most generic implementation is to iterate over container nodes: +``` +Iterator iter = nodeDAO.containerIterator(false); // order not relevant ``` ## database changes required note: all field and column names TBD note: fields in Node classes probably not transient but TBD -* add `nodeSize` and `delta` fields to ContainerNode -* add `nodeSize` field to DataNode (no size props in LinkNode!) -* add `nodeSize` to the `vospace.Node` table +* add `nbytes` and `delta` fields to ContainerNode +* add `nbytes` field to DataNode (no size props in LinkNode!) +* add `nbytes` to the `vospace.Node` table * add `delta` to the `vospace.Node` table * add `storageBucket` to DataNode -* add `storageBucket` to `vospace.Node` table (validation) +* add `storageBucket` to `vospace.Node` table + ## cadc-inventory-db API required * incremental sync query/iterator: ArtifactDAO.iterator(Namespace ns, String uriBucketPrefix, Date minLastModified)? * lookup DataNode by storageID: NodeDAO.getDataNode(URI storageID)? * validate-by-bucket: use ArtifactDAO.iterator(String uriBucketPrefix, boolean ordered, Namespace ns) * validate-by-bucket: NodeDAO.dataNodeIterator(String storageBucketPrefix, boolean ordered) +* incremental and validate containers: NodeDAO.containerIterator(boolean nonZeroDelta) * indices to support new queries From 76d975d56b9e61e84a66fe6e13e29a8a896171e0 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 22 Feb 2024 11:10:12 -0800 Subject: [PATCH 136/186] add missing dao config for minoc int test --- .../src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java b/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java index 0216cb74b..938898093 100644 --- a/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java +++ b/minoc/src/intTest/java/org/opencadc/minoc/ReplaceArtifactTest.java @@ -121,6 +121,7 @@ public ReplaceArtifactTest() throws Exception { config.put("jndiDataSourceName", "jdbc/minoc"); config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("invSchema", "inventory"); + config.put("genSchema", "inventory"); this.dao = new ArtifactDAO(); dao.setConfig(config); From ddd87e0537fa44900f908ebdb071e8c5d4aef6f6 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 22 Feb 2024 12:15:27 -0800 Subject: [PATCH 137/186] raven: restore test db config --- raven/src/intTest/java/org/opencadc/raven/RavenTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/src/intTest/java/org/opencadc/raven/RavenTest.java b/raven/src/intTest/java/org/opencadc/raven/RavenTest.java index cf48f07ab..58d4bb27f 100644 --- a/raven/src/intTest/java/org/opencadc/raven/RavenTest.java +++ b/raven/src/intTest/java/org/opencadc/raven/RavenTest.java @@ -144,7 +144,7 @@ public RavenTest() throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/inventory"); config.put("invSchema", SCHEMA); - //config.put("genSchema", SCHEMA); + config.put("genSchema", SCHEMA); } catch (Exception ex) { log.error("setup failed", ex); From 86c5079ed94ffcc8377045d273ecc573e4ff6a78 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 23 Feb 2024 13:00:47 -0800 Subject: [PATCH 138/186] vault: fix for test dao setup --- vault/src/intTest/java/org/opencadc/vault/TransferTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java index 070b7aad3..f1392e1ee 100644 --- a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java @@ -123,6 +123,7 @@ public void checkGlobal() throws Exception { config.put(SQLGenerator.class.getName(), SQLGenerator.class); config.put("jndiDataSourceName", "jdbc/inventory"); config.put("invSchema", SCHEMA); + config.put("genSchema", SCHEMA); StorageSiteDAO dao = new StorageSiteDAO(false); dao.setConfig(config); From ba76c2ff39f390d03c861ee9f5c2fe02968aadf8 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 23 Feb 2024 13:43:14 -0800 Subject: [PATCH 139/186] update vault-quota/Design details --- vault-quota/Design.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index e5ef4a6ea..d6b167a9d 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -134,17 +134,20 @@ Iterator iter = nodeDAO.containerIterator(false); // order not re ## database changes required note: all field and column names TBD -note: fields in Node classes probably not transient but TBD -* add `nbytes` and `delta` fields to ContainerNode -* add `nbytes` field to DataNode (no size props in LinkNode!) -* add `nbytes` to the `vospace.Node` table +* add `transient Long bytesUsed` to ContainerNode and DataNode +* add `transient long delta` field to ContainerNode +* add `bytesUsed` to the `vospace.Node` table * add `delta` to the `vospace.Node` table -* add `storageBucket` to DataNode +* add `storageBucket` to DataNode?? TBD * add `storageBucket` to `vospace.Node` table -## cadc-inventory-db API required -* incremental sync query/iterator: ArtifactDAO.iterator(Namespace ns, String uriBucketPrefix, Date minLastModified)? -* lookup DataNode by storageID: NodeDAO.getDataNode(URI storageID)? +## cadc-inventory-db API required immediately +* incremental sync query/iterator: ArtifactDAO.iterator(Namespace ns, String uriBucketPrefix, Date minLastModified, boolean ordered) + order by lastModified if set +* lookup DataNode by storageID: NodeDAO.getDataNode(URI storageID) +* indices to support new queries + +## cadc-inventory-db API required later * validate-by-bucket: use ArtifactDAO.iterator(String uriBucketPrefix, boolean ordered, Namespace ns) * validate-by-bucket: NodeDAO.dataNodeIterator(String storageBucketPrefix, boolean ordered) * incremental and validate containers: NodeDAO.containerIterator(boolean nonZeroDelta) From 93b0860c35304e781e2437f8896655322623d8ff Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 23 Feb 2024 13:46:00 -0800 Subject: [PATCH 140/186] cadc-inventory-db: updates to support node size metadata and workflow --- .../inventory/db/ArtifactDAOTest.java | 67 ++++++- .../org/opencadc/vospace/db/NodeDAOTest.java | 136 ++++++++++++-- .../opencadc/inventory/db/ArtifactDAO.java | 38 +++- .../opencadc/inventory/db/SQLGenerator.java | 173 +++++++++++++----- .../opencadc/vospace/db/InitDatabaseVOS.java | 6 +- .../java/org/opencadc/vospace/db/NodeDAO.java | 51 +++++- .../src/main/resources/vospace.Node.sql | 7 +- .../opencadc/vault/NodePersistenceImpl.java | 39 +++- 8 files changed, 420 insertions(+), 97 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java index bad7928cc..f3a225976 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/inventory/db/ArtifactDAOTest.java @@ -94,6 +94,7 @@ import org.junit.Test; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.StoredArtifactComparator; @@ -955,6 +956,9 @@ public void testIteratorClose() { public void testArtifactIterator() { int num = 10; try { + final Date startDate = new Date(); + Thread.sleep(10L); + int numArtifacts = 0; int numStuffExpected = 0; // artifacts with storageLocation @@ -977,6 +981,7 @@ public void testArtifactIterator() { numStuffExpected++; } } + // some artifacts with no storageLocation collection = "STUFF"; for (int i = num; i < 2 * num; i++) { @@ -996,6 +1001,12 @@ public void testArtifactIterator() { numStuffExpected++; } } + + Thread.sleep(10L); + final Date midDate = new Date(); + Thread.sleep(10L); + final int midNumStuff = numStuffExpected; + // some artifacts with siteLocations UUID siteID = UUID.randomUUID(); int numSiteExpected = 0; @@ -1018,6 +1029,8 @@ public void testArtifactIterator() { numStuffExpected++; } } + Thread.sleep(10L); + final Date endDate = new Date(); log.info("added: " + numArtifacts); log.info("count all..."); @@ -1032,13 +1045,14 @@ public void testArtifactIterator() { Assert.assertEquals("count", numArtifacts, count); log.info("count with criteria..."); + final Namespace ns = new Namespace("cadc:STUFF/"); count = 0; - try (ResourceIterator iter = originDAO.iterator("uri like 'cadc:STUFF/%'", null, false)) { + try (ResourceIterator iter = originDAO.iterator(ns, null, false)) { while (iter.hasNext()) { Artifact actual = iter.next(); count++; log.info("found: " + actual.getURI()); - Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith("cadc:STUFF/")); + Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace())); } } Assert.assertEquals("count", numStuffExpected, count); @@ -1065,7 +1079,8 @@ public void testArtifactIterator() { while (iter.hasNext()) { Artifact actual = iter.next(); count++; - log.info("found: " + actual.getURI()); + log.info("found: " + actual.getBucket() + " " + actual.getURI()); + Assert.assertTrue(actual.getBucket().startsWith(bpre)); } } } @@ -1075,18 +1090,56 @@ public void testArtifactIterator() { count = 0; for (byte b = 0; b < 16; b++) { String bpre = HexUtil.toHex(b).substring(1); - log.debug("bucket prefix: " + bpre); - try (ResourceIterator iter = originDAO.iterator("uri like 'cadc:STUFF/%'", bpre, false)) { + log.info("bucket prefix: " + bpre); + try (ResourceIterator iter = originDAO.iterator(ns, bpre, false)) { while (iter.hasNext()) { Artifact actual = iter.next(); count++; - log.info("found: " + actual.getURI()); - Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith("cadc:STUFF/")); + log.info("found: " + actual.getBucket() + " " + actual.getURI()); + Assert.assertTrue(actual.getBucket().startsWith(bpre)); + Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace())); } } } Assert.assertEquals("count", numStuffExpected, count); + log.info("count vs Namespace incremental from start..."); + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + count = 0; + try (ResourceIterator iter = originDAO.iterator(ns, null, startDate, true)) { + while (iter.hasNext()) { + Artifact actual = iter.next(); + count++; + log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified())); + Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace())); + } + } + Assert.assertEquals("count", numStuffExpected, count); + + log.info("count vs Namespace incremental from mid..."); + count = 0; + try (ResourceIterator iter = originDAO.iterator(ns, null, midDate, true)) { + while (iter.hasNext()) { + Artifact actual = iter.next(); + count++; + log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified())); + Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace())); + } + } + Assert.assertEquals("count", midNumStuff, count); + + log.info("count vs Namespace incremental from end..."); + count = 0; + try (ResourceIterator iter = originDAO.iterator(ns, null, endDate, true)) { + while (iter.hasNext()) { + Artifact actual = iter.next(); + count++; + log.info("found: " + actual.getBucket() + " " + actual.getURI() + " " + df.format(actual.getContentLastModified())); + Assert.assertTrue("STUFF", actual.getURI().toASCIIString().startsWith(ns.getNamespace())); + } + } + Assert.assertEquals("count", 0, count); + } catch (Exception unexpected) { log.error("unexpected exception", unexpected); Assert.fail("unexpected exception: " + unexpected); diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index 4a776c116..dbd217a20 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -182,7 +182,7 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, // put ContainerNode orig = new ContainerNode("container-test"); - orig.parent = root; + orig.parentID = root.getID(); orig.ownerID = "the-owner"; nodeDAO.put(orig); @@ -216,6 +216,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + Assert.assertEquals(orig.bytesUsed, c.bytesUsed); + Assert.assertEquals(orig.delta, c.delta); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); @@ -228,7 +230,6 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Thread.sleep(10L); orig.getReadOnlyGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g1"))); orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); - orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.isPublic = true; orig.inheritPermissions = true; nodeDAO.put(orig); @@ -251,6 +252,8 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, Assert.assertTrue(updated instanceof ContainerNode); ContainerNode uc = (ContainerNode) updated; Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + Assert.assertEquals(orig.bytesUsed, uc.bytesUsed); + Assert.assertEquals(orig.delta, uc.delta); nodeDAO.delete(orig.getID()); @@ -267,7 +270,7 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException // TODO: use get-by-path to find and remove the test node ContainerNode orig = new ContainerNode("container-test"); - orig.parent = root; + orig.parentID = root.getID(); orig.ownerID = "the-owner"; orig.isPublic = true; orig.isLocked = false; @@ -279,8 +282,6 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6.g7"))); orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6_g7"))); orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g6~g7"))); - - orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.getProperties().add(new NodeProperty(URI.create("custom:prop"), "spaces in value")); orig.getProperties().add(new NodeProperty(URI.create("sketchy:a,b"), "comma in uri")); orig.getProperties().add(new NodeProperty(URI.create("sketchy:funny"), "value-with-{delims}")); @@ -316,6 +317,8 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertTrue(a instanceof ContainerNode); ContainerNode c = (ContainerNode) a; Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); + Assert.assertEquals(orig.bytesUsed, c.bytesUsed); + Assert.assertEquals(orig.delta, c.delta); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); @@ -333,7 +336,6 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException orig.getReadWriteGroup().clear(); orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); - orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); orig.inheritPermissions = true; nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -355,6 +357,8 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException Assert.assertTrue(updated instanceof ContainerNode); ContainerNode uc = (ContainerNode) updated; Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); + Assert.assertEquals(orig.bytesUsed, uc.bytesUsed); + Assert.assertEquals(orig.delta, uc.delta); nodeDAO.delete(orig.getID()); Node gone = nodeDAO.get(orig.getID()); @@ -367,9 +371,10 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root"); - DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); - orig.parent = root; + DataNode orig = new DataNode("data-test"); + orig.parentID = root.getID(); orig.ownerID = "the-owner"; + orig.storageID = URI.create("vault:" + UUID.randomUUID().toString()); orig.isPublic = true; orig.isLocked = false; orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, "text/plain")); @@ -423,7 +428,6 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, orig.getReadWriteGroup().clear(); orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); - orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change storageID nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -445,6 +449,7 @@ public void testPutGetUpdateDeleteDataNode() throws InterruptedException, Assert.assertTrue(a instanceof DataNode); DataNode udn = (DataNode) updated; + Assert.assertEquals(orig.bytesUsed, udn.bytesUsed); Assert.assertEquals(orig.storageID, udn.storageID); nodeDAO.delete(orig.getID()); @@ -461,7 +466,7 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, // TODO: use get-by-path to find and remove the test node LinkNode orig = new LinkNode("data-test", URI.create("vos://opencadc.org~srv/path/to/something")); - orig.parent = root; + orig.parentID = root.getID(); orig.ownerID = "the-owner"; orig.isPublic = true; orig.isLocked = false; @@ -514,7 +519,6 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, orig.getReadWriteGroup().clear(); orig.getReadWriteGroup().add(new GroupURI(URI.create("ivo://opencadc.org/gms?g3"))); orig.getProperties().clear(); - orig.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "123")); // don't change target nodeDAO.put(orig); Node updated = nodeDAO.get(orig.getID()); @@ -542,6 +546,29 @@ public void testPutGetUpdateDeleteLinkNode() throws InterruptedException, Assert.assertNull(gone); } + @Test + public void testGetByStorageID() { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + DataNode notFound = nodeDAO.getDataNode(URI.create("vault:not-found")); + Assert.assertNull(notFound); + + DataNode orig = new DataNode("testGetByStorageID"); + orig.parentID = root.getID(); + orig.ownerID = "the-owner"; + orig.storageID = URI.create("vault:" + UUID.randomUUID().toString()); + nodeDAO.put(orig); + + // get-by-storageID + DataNode gbs = nodeDAO.getDataNode(orig.storageID); + Assert.assertNotNull(gbs); + log.info("found: " + gbs.getID() + " aka " + gbs); + Assert.assertEquals(orig.getID(), gbs.getID()); + + nodeDAO.delete(orig.getID()); + } + @Test public void testGetWithLock() { UUID rootID = new UUID(0L, 0L); @@ -549,7 +576,7 @@ public void testGetWithLock() { // put ContainerNode orig = new ContainerNode("container-test"); - orig.parent = root; + orig.parentID = root.getID(); orig.ownerID = "the-owner"; nodeDAO.put(orig); @@ -571,13 +598,88 @@ public void testGetWithLock() { Assert.assertNull(gone); } + @Test + public void testUpdateNodeSize() throws InterruptedException, + NoSuchAlgorithmException { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + final ContainerNode cnode = new ContainerNode("testUpdateNodeSize-container"); + cnode.parentID = root.getID(); + cnode.ownerID = "the-owner"; + nodeDAO.put(cnode); + + final DataNode dnode = new DataNode("testUpdateNodeSize-data"); + dnode.ownerID = "the-owner"; + dnode.storageID = URI.create("cadc:vault/" + UUID.randomUUID()); + dnode.parentID = cnode.getID(); + nodeDAO.put(dnode); + + final ContainerNode c1 = (ContainerNode) nodeDAO.get(cnode.getID()); + Assert.assertNotNull(c1); + log.info("found: " + c1.getID() + " aka " + c1); + Assert.assertEquals(cnode.getID(), c1.getID()); + Assert.assertEquals(cnode.getName(), c1.getName()); + Assert.assertEquals(root.getID(), c1.parentID); + Assert.assertNull(c1.bytesUsed); + Assert.assertEquals(0L, c1.delta); + + final DataNode d1 = (DataNode) nodeDAO.get(dnode.getID()); + Assert.assertNotNull(d1); + log.info("found: " + d1.getID() + " aka " + d1); + Assert.assertEquals(dnode.getID(), d1.getID()); + Assert.assertEquals(dnode.getName(), d1.getName()); + Assert.assertEquals(cnode.getID(), d1.parentID); + Assert.assertNull(d1.bytesUsed); + + final URI ccs = c1.getMetaChecksum(); + final URI dcs = d1.getMetaChecksum(); + + log.info("update DataNode"); + d1.bytesUsed = 123L; + nodeDAO.put(d1, true, false); + final DataNode d2 = (DataNode) nodeDAO.get(dnode.getID()); + Assert.assertNotNull(d2); + Assert.assertNotNull(d2.bytesUsed); + Assert.assertEquals(d1.bytesUsed, d2.bytesUsed); + URI dcs2 = d2.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals(dnode.getMetaChecksum(), dcs2); // no update + Assert.assertEquals(dnode.getLastModified(), d2.getLastModified()); // no update + + log.info("update ContainerNode.delta"); + c1.delta = 123L; + nodeDAO.put(c1, true, false); + final ContainerNode c2 = (ContainerNode) nodeDAO.get(cnode.getID()); + Assert.assertNotNull(c2); + Assert.assertEquals(123L, c2.delta); + URI ccs2 = c2.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals(cnode.getMetaChecksum(), ccs2); // no update + Assert.assertEquals(cnode.getLastModified(), c2.getLastModified()); // no update + + log.info("update ContainerNode.bytesUsed"); + c2.bytesUsed = c2.delta; + c2.delta = 0L; + nodeDAO.put(c2, true, false); + final ContainerNode c3 = (ContainerNode) nodeDAO.get(cnode.getID()); + Assert.assertNotNull(c3); + Assert.assertEquals(0L, c3.delta); + Assert.assertNotNull(c3.bytesUsed); + Assert.assertEquals(123L, c3.bytesUsed.longValue()); + URI ccs3 = c3.computeMetaChecksum(MessageDigest.getInstance("MD5")); + Assert.assertEquals(cnode.getMetaChecksum(), ccs3); // no update + Assert.assertEquals(cnode.getLastModified(), c3.getLastModified()); // no update + + nodeDAO.delete(dnode.getID()); + nodeDAO.delete(cnode.getID()); + } + @Test public void testContainerNodeIterator() throws IOException { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root"); ContainerNode orig = new ContainerNode("container-test"); - orig.parent = root; + orig.parentID = root.getID(); orig.ownerID = "the-owner"; nodeDAO.put(orig); @@ -610,13 +712,13 @@ public void testContainerNodeIterator() throws IOException { // add children ContainerNode cont = new ContainerNode("container1"); - cont.parent = orig; + cont.parentID = orig.getID(); cont.ownerID = orig.ownerID; DataNode data = new DataNode(UUID.randomUUID(), "data1", URI.create("cadc:vault/" + UUID.randomUUID())); - data.parent = orig; + data.parentID = orig.getID(); data.ownerID = orig.ownerID; LinkNode link = new LinkNode("link1", URI.create("cadc:ARCHIVE/data")); - link.parent = orig; + link.parentID = orig.getID(); link.ownerID = orig.ownerID; log.info("put child: " + cont + " of " + cont.parent); nodeDAO.put(cont); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/ArtifactDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/ArtifactDAO.java index 200b4e63c..cb7964ce8 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/ArtifactDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/ArtifactDAO.java @@ -69,9 +69,11 @@ import ca.nrc.cadc.io.ResourceIterator; import java.net.URI; +import java.util.Date; import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.SiteLocation; import org.opencadc.inventory.StorageLocation; import org.springframework.jdbc.BadSqlGrammarException; @@ -190,8 +192,8 @@ public ResourceIterator storedIterator(String storageBucketPrefix) { try { SQLGenerator.ArtifactIteratorQuery iter = (SQLGenerator.ArtifactIteratorQuery) gen.getEntityIteratorQuery(Artifact.class); iter.setStorageLocationRequired(true); + iter.setStorageBucket(storageBucketPrefix); iter.setOrderedOutput(true); - iter.setPrefix(storageBucketPrefix); return iter.query(dataSource); } catch (BadSqlGrammarException ex) { handleInternalFail(ex); @@ -218,8 +220,8 @@ public ResourceIterator unstoredIterator(String uriBucketPrefix) { try { SQLGenerator.ArtifactIteratorQuery iter = (SQLGenerator.ArtifactIteratorQuery) gen.getEntityIteratorQuery(Artifact.class); iter.setStorageLocationRequired(false); + iter.setUriBucket(uriBucketPrefix); iter.setOrderedOutput(true); - iter.setPrefix(uriBucketPrefix); return iter.query(dataSource); } catch (BadSqlGrammarException ex) { handleInternalFail(ex); @@ -240,7 +242,7 @@ public ResourceIterator unstoredIterator(String uriBucketPrefix) { * @return iterator over artifacts */ public ResourceIterator iterator(String uriBucketPrefix, boolean ordered) { - return iterator((String) null, uriBucketPrefix, ordered); + return iterator((Namespace) null, uriBucketPrefix, ordered); } /** @@ -250,19 +252,37 @@ public ResourceIterator iterator(String uriBucketPrefix, boolean order * *

Use case: local cleanup by arbitrary criteria * - * @param criteria conditions for selecting artifacts + * @param ns namespace for selecting artifacts * @param uriBucketPrefix null, prefix, or complete Artifact.uriBucket string * @param ordered order by Artifact.uri (true) or not ordered (false) * @return iterator over artifacts matching criteria */ - public ResourceIterator iterator(String criteria, String uriBucketPrefix, boolean ordered) { + public ResourceIterator iterator(Namespace ns, String uriBucketPrefix, boolean ordered) { + return iterator(ns, uriBucketPrefix, null, ordered); + } + + /** + * Iterate over artifacts that match criteria. This method adds an optional Date argument to + * support incremental processing. In this case, ordered will be in timestamp order rather than + * uri order. + * + *

Use case: process artifact events directly in the database + * + * @param ns namespace for selecting artifacts + * @param uriBucketPrefix null, prefix, or complete Artifact.uriBucket string + * @param minLastModified minimum Artifact.lastModified to consider (incremental mode) + * @param ordered order by Artifact.uri (true) or not ordered (false) + * @return iterator over artifacts matching criteria + */ + public ResourceIterator iterator(Namespace ns, String uriBucketPrefix, Date minLastModified, boolean ordered) { checkInit(); long t = System.currentTimeMillis(); try { SQLGenerator.ArtifactIteratorQuery iter = (SQLGenerator.ArtifactIteratorQuery) gen.getEntityIteratorQuery(Artifact.class); - iter.setPrefix(uriBucketPrefix); - iter.setCriteria(criteria); + iter.setUriBucket(uriBucketPrefix); + iter.setNamespace(ns); + iter.setMinLastModified(minLastModified); iter.setOrderedOutput(ordered); return iter.query(dataSource); } catch (BadSqlGrammarException ex) { @@ -273,7 +293,7 @@ public ResourceIterator iterator(String criteria, String uriBucketPref } throw new RuntimeException("BUG: should be unreachable"); } - + /** * Iterate over Artifacts from a specific site. If a siteID is specified, only artifacts where * artifact.siteLocations includes that siteID are returned; this is only applicable in a global @@ -292,7 +312,7 @@ public ResourceIterator iterator(UUID siteID, String uriBucketPrefix, try { SQLGenerator.ArtifactIteratorQuery iter = (SQLGenerator.ArtifactIteratorQuery) gen.getEntityIteratorQuery(Artifact.class); - iter.setPrefix(uriBucketPrefix); + iter.setUriBucket(uriBucketPrefix); iter.setSiteID(siteID); iter.setOrderedOutput(ordered); return iter.query(dataSource); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index 16d07a4c7..ffe8f8ef1 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -94,6 +94,7 @@ import org.opencadc.inventory.DeletedArtifactEvent; import org.opencadc.inventory.DeletedStorageLocationEvent; import org.opencadc.inventory.InventoryUtil; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.ObsoleteStorageLocation; import org.opencadc.inventory.PreauthKeyPair; import org.opencadc.inventory.SiteLocation; @@ -260,8 +261,11 @@ protected void init() { "readWriteGroups", "properties", "inheritPermissions", + "delta", + "bytesUsed", "busy", "storageID", + "storageBucket", "target", "lastModified", "metaChecksum", @@ -675,10 +679,15 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce class ArtifactIteratorQuery implements EntityIteratorQuery { private Boolean storageLocationRequired; - private String prefix; + private String storageBucket; + private UUID siteID; - private String whereClause; + private String uriBucket; + private Namespace namespace; + private Date minLastModified; private boolean ordered; + + private final Calendar utc = Calendar.getInstance(DateUtil.UTC); public ArtifactIteratorQuery() { } @@ -694,22 +703,30 @@ public void setStorageLocationRequired(Boolean slr) { this.storageLocationRequired = slr; } - public void setPrefix(String prefix) { + public void setStorageBucket(String prefix) { if (StringUtil.hasText(prefix)) { - this.prefix = prefix.trim(); + this.storageBucket = prefix.trim(); } else { - this.prefix = null; + this.storageBucket = null; } } - public void setCriteria(String whereClause) { - if (StringUtil.hasText(whereClause)) { - this.whereClause = whereClause.trim(); + public void setUriBucket(String uriBucket) { + if (StringUtil.hasText(uriBucket)) { + this.uriBucket = uriBucket.trim(); } else { - this.whereClause = null; + uriBucket = null; } } + public void setNamespace(Namespace namespace) { + this.namespace = namespace; + } + + public void setMinLastModified(Date minLastModified) { + this.minLastModified = minLastModified; + } + public void setOrderedOutput(boolean ordered) { this.ordered = ordered; } @@ -720,13 +737,13 @@ public void setSiteID(UUID siteID) { @Override public ResourceIterator query(DataSource ds) { - - StringBuilder sb = getSelectFromSQL(Artifact.class, false); - sb.append(" WHERE"); + StringBuilder select = getSelectFromSQL(Artifact.class, false); + StringBuilder sb = new StringBuilder(" WHERE"); + int whereLen = sb.length(); if (storageLocationRequired != null && storageLocationRequired) { // ArtifactDAO.storedIterator - if (StringUtil.hasText(prefix)) { + if (storageBucket != null) { sb.append(" storageLocation_storageBucket LIKE ? AND"); } sb.append(" storageLocation_storageID IS NOT NULL"); @@ -739,45 +756,54 @@ public ResourceIterator query(DataSource ds) { sb.append(" ORDER BY storageLocation_storageBucket, storageLocation_storageID"); } } else if (storageLocationRequired != null && !storageLocationRequired) { - // ArtifactDAO.unstoredIterator - if (StringUtil.hasText(prefix)) { - sb.append(" uriBucket LIKE ? AND"); - } sb.append(" storageLocation_storageID IS NULL"); - if (ordered) { - sb.append(" ORDER BY uri"); - } - } else if (siteID != null) { - if (prefix != null && siteID != null) { - sb.append(" uriBucket LIKE ? AND ").append("siteLocations @> ARRAY[?]"); - } else { - sb.append(" siteLocations @> ARRAY[?]"); + } + + if (uriBucket != null) { + // ArtifactDAO.iterator(null, ...) + if (sb.length() > whereLen) { + sb.append(" AND"); } - if (ordered) { - sb.append(" ORDER BY uri"); + sb.append(" uriBucket LIKE ?"); + } + if (siteID != null) { + // ArtifactDAO.iterator(UUID, ...) + if (sb.length() > whereLen) { + sb.append(" AND"); } - } else if (whereClause != null) { - if (prefix != null && whereClause != null) { - sb.append(" uriBucket LIKE ? AND ( ").append(whereClause).append(" )"); - } else { - sb.append(" (").append(whereClause).append(" )"); + sb.append(" siteLocations @> ARRAY[?]"); + } + if (namespace != null) { + // ArtifactDAO.iterator(Namespace, ...) + if (sb.length() > whereLen) { + sb.append(" AND"); } - if (ordered) { - sb.append(" ORDER BY uri"); + sb.append(" uri LIKE ?"); + } + if (minLastModified != null) { + if (sb.length() > whereLen) { + sb.append(" AND"); } - } else if (prefix != null) { - sb.append(" uriBucket LIKE ?"); - if (ordered) { + sb.append(" lastModified >= ?"); + } + if (ordered && !(storageLocationRequired != null && storageLocationRequired)) { + if (minLastModified != null) { + sb.append(" ORDER BY lastModified ASC"); + } else { sb.append(" ORDER BY uri"); } - } else { - // trim off " WHERE" - sb.delete(sb.length() - 6, sb.length()); } - String sql = sb.toString(); - log.debug("sql: " + sql); + if (sb.length() > whereLen) { + select.append(sb.toString()); + } + String sql = select.toString(); + log.warn("sql: " + sql); + // params: + // storageBucket OR uriBucket + // siteID + // namespace try { Connection con = ds.getConnection(); log.debug("ArtifactIterator: setAutoCommit(false)"); @@ -787,15 +813,29 @@ public ResourceIterator query(DataSource ds) { ps.setFetchSize(1000); ps.setFetchDirection(ResultSet.FETCH_FORWARD); int col = 1; - if (prefix != null) { - String val = prefix + "%"; + if (storageBucket != null) { + String val = storageBucket + "%"; + log.debug("bucket prefix: " + val); + ps.setString(col++, val); + } else if (uriBucket != null) { + String val = uriBucket + "%"; log.debug("bucket prefix: " + val); ps.setString(col++, val); } if (siteID != null) { log.debug("siteID: " + siteID); ps.setObject(col++, siteID); + } + if (namespace != null) { + String val = namespace.getNamespace() + "%"; + log.debug("namespace prefix: " + val); + ps.setString(col++, val); + } + if (minLastModified != null) { + log.debug("min lastModified: " + minLastModified); + ps.setTimestamp(col++, new Timestamp(minLastModified.getTime()), utc); } + ResultSet rs = ps.executeQuery(); return new ArtifactResultSetIterator(con, rs); @@ -939,6 +979,7 @@ public class NodeGet implements EntityGet { private UUID id; private ContainerNode parent; private String name; + private URI storageID; private final boolean forUpdate; public NodeGet(boolean forUpdate) { @@ -954,6 +995,10 @@ public void setPath(ContainerNode parent, String name) { this.parent = parent; this.name = name; } + + public void setStorageID(URI storageID) { + this.storageID = storageID; + } @Override public Node execute(JdbcTemplate jdbc) { @@ -967,10 +1012,12 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce if (id != null) { String col = getKeyColumn(Node.class, true); sb.append(col).append(" = ?"); + } else if (storageID != null) { + String col = "storageID"; + sb.append(col).append(" = ?"); } else if (parent != null && name != null) { String pidCol = "parentID"; String nameCol = "name"; - // TODO: better way to get column names? sb.append(pidCol).append(" = ? and ").append(nameCol).append(" = ?"); } else { throw new IllegalStateException("primary key is null"); @@ -983,6 +1030,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce PreparedStatement prep = conn.prepareStatement(sql); if (id != null) { prep.setObject(1, id); + } else if (storageID != null) { + prep.setObject(1, storageID.toASCIIString()); } else { prep.setObject(1, parent.getID()); prep.setObject(2, name); @@ -1466,12 +1515,29 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce safeSetArray(prep, col++, value.getReadOnlyGroup()); safeSetArray(prep, col++, value.getReadWriteGroup()); safeSetProps(prep, col++, value.getProperties()); + + // ContainerNode-specific fields if (value instanceof ContainerNode) { ContainerNode cn = (ContainerNode) value; safeSetBoolean(prep, col++, cn.inheritPermissions); + safeSetLong(prep, col++, cn.delta); } else { safeSetBoolean(prep, col++, null); + safeSetLong(prep, col++, null); + } + + // bytesUsed is in between CN and DN specific columns + if (value instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) value; + safeSetLong(prep, col++, cn.bytesUsed); + } else if (value instanceof DataNode) { + DataNode dn = (DataNode) value; + safeSetLong(prep, col++, dn.bytesUsed); + } else { + safeSetLong(prep, col++, null); } + + // DataNode specific fields if (value instanceof DataNode) { DataNode dn = (DataNode) value; if (dn.storageID == null) { @@ -1479,10 +1545,14 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce } safeSetBoolean(prep, col++, dn.busy); safeSetString(prep, col++, dn.storageID); + safeSetString(prep, col++, InventoryUtil.computeBucket(dn.storageID, 5)); // same as Artifact } else { safeSetBoolean(prep, col++, null); safeSetString(prep, col++, (URI) null); + safeSetString(prep, col++, (URI) null); } + + // LinkNode-specific fields if (value instanceof LinkNode) { LinkNode ln = (LinkNode) value; prep.setString(col++, ln.getTarget().toASCIIString()); @@ -1490,6 +1560,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce safeSetString(prep, col++, (URI) null); } + // Entity fields prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc); prep.setString(col++, value.getMetaChecksum().toASCIIString()); prep.setObject(col++, value.getID()); @@ -1904,8 +1975,12 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro final String rawRWG = rs.getString(col++); final String rawProps = rs.getString(col++); final Boolean inheritPermissions = Util.getBoolean(rs, col++); + final Long delta = Util.getLong(rs, col++); + final Long bytesUsed = Util.getLong(rs, col++); final Boolean busy = Util.getBoolean(rs, col++); final URI storageID = Util.getURI(rs, col++); + final String storageBucket = rs.getString(col++); + // TODO: return this somehow or just use in DataNode iterator? final URI linkTarget = Util.getURI(rs, col++); final Date lastModified = Util.getDate(rs, col++, utc); final URI metaChecksum = Util.getURI(rs, col++); @@ -1915,9 +1990,13 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro if (nodeType.equals("C")) { ContainerNode cn = new ContainerNode(id, name); cn.inheritPermissions = inheritPermissions; + cn.bytesUsed = bytesUsed; + cn.delta = (delta == null ? 0 : delta); ret = cn; } else if (nodeType.equals("D")) { - ret = new DataNode(id, name, storageID); + DataNode dn = new DataNode(id, name, storageID); + dn.bytesUsed = bytesUsed; + ret = dn; } else if (nodeType.equals("L")) { ret = new LinkNode(id, name, linkTarget); } else { @@ -2148,7 +2227,5 @@ public Node extractData(ResultSet rs) throws SQLException, DataAccessException { return mapRowToNode(rs, utc, parent); } - - } } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java index 5ac9910e7..05d8c45b4 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -80,8 +80,8 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); public static final String MODEL_NAME = "vospace-inventory"; - public static final String MODEL_VERSION = "0.3"; - public static final String PREV_MODEL_VERSION = "0.2"; + public static final String MODEL_VERSION = "0.15"; + public static final String PREV_MODEL_VERSION = "0.3"; static String[] CREATE_SQL = new String[] { "generic.ModelVersion.sql", @@ -93,7 +93,7 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { }; static String[] UPGRADE_SQL = new String[] { - "generic.HarvestState.sql", + "vospace.upgrade-0.15.sql", "generic.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index 58033b796..d2e01e854 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -68,11 +68,13 @@ package org.opencadc.vospace.db; import ca.nrc.cadc.io.ResourceIterator; +import java.net.URI; import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.db.AbstractDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; import org.opencadc.vospace.Node; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; @@ -88,20 +90,29 @@ public NodeDAO() { super(true); } - // needed by vault migration tool: untested public NodeDAO(boolean origin) { super(origin); } @Override public void put(Node val) { - // TBD: caller can assign parent or parentID before put, but here we need - // parentID and it must be assigned before metaChecksum compute in super.put() - if (val.parentID == null && val.parent != null) { - val.parentID = val.parent.getID(); - } super.put(val); } + + /** + * Update that optionally includes extended content. Extended content is computed + * or transient fields that do not trigger a metaChecksum change so would normally + * be skipped. + * + * @param val the Node to update + * @param extendedUpdate true to force db update + * @param timestampUpdate true to force lastModified update + */ + @Override + protected void put(Node val, boolean extendedUpdate, boolean timestampUpdate) { + super.put(val, extendedUpdate, timestampUpdate); + } + @Override public Node lock(Node n) { @@ -113,6 +124,7 @@ public Node lock(Node n) { } public Node get(UUID id) { + checkInit(); return super.get(Node.class, id); } @@ -135,6 +147,25 @@ public Node get(ContainerNode parent, String name) { throw new RuntimeException("BUG: handleInternalFail did not throw"); } + public DataNode getDataNode(URI storageID) { + checkInit(); + log.debug("GET: " + storageID); + long t = System.currentTimeMillis(); + + try { + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + SQLGenerator.NodeGet get = (SQLGenerator.NodeGet) gen.getEntityGet(Node.class); + get.setStorageID(storageID); + return (DataNode) get.execute(jdbc); + } catch (BadSqlGrammarException ex) { + handleInternalFail(ex); + } finally { + long dt = System.currentTimeMillis() - t; + log.debug("GET: " + storageID + " " + dt + "ms"); + } + throw new RuntimeException("BUG: handleInternalFail did not throw"); + } + public boolean isEmpty(ContainerNode parent) { checkInit(); log.debug("isEmpty: " + parent.getID()); @@ -159,6 +190,14 @@ public void delete(UUID id) { super.delete(Node.class, id); } + /** + * Get iterator of child nodes. + * + * @param parent the container node to list + * @param limit max number of nodes to return, or null + * @param start list starting point, or null + * @return iterator of child nodes matching the arguments + */ public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) { if (parent == null) { throw new IllegalArgumentException("childIterator: parent cannot be null"); diff --git a/cadc-inventory-db/src/main/resources/vospace.Node.sql b/cadc-inventory-db/src/main/resources/vospace.Node.sql index e52a5ed8c..8e96a67f0 100644 --- a/cadc-inventory-db/src/main/resources/vospace.Node.sql +++ b/cadc-inventory-db/src/main/resources/vospace.Node.sql @@ -6,20 +6,23 @@ create table .Node ( nodeType char(1) not null, ownerID varchar(256) not null, + bytesUsed bigint, isPublic boolean, isLocked boolean, readOnlyGroups text, readWriteGroups text, - + -- store all props in a 2D array properties text[][], -- ContainerNode inheritPermissions boolean, + delta bigint; -- DataNode busy boolean, storageID varchar(512), + storageBucket varchar(5), -- LinkNode target text, @@ -32,3 +35,5 @@ create table .Node ( create unique index node_parent_child on .Node(parentID,name); create index node_lastmodified on .Node(lastModified); + +create unique index node_storageID on .Node(storageID); \ No newline at end of file diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 0afa377a0..f41112cea 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -323,7 +323,8 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (cd != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTDATE, df.format(cd))); } - + // TODO: a.getContentLength() is correct, but might differ from dn.bytesUsed?? eg eventual consistency + // child listing iterator can only report dn.bytesUsed ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, a.getContentLength().toString())); // assume MD5 ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTMD5, a.getContentChecksum().getSchemeSpecificPart())); @@ -334,10 +335,18 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (a.contentType != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); } + } else if (dn.bytesUsed != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, dn.bytesUsed.toString())); } else { // default size to 0 ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "0")); } + } else if (ret instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) ret; + if (cn.bytesUsed != null) { + // TBD: long num = bytesUsed + cn.delta; // include unpropagated delta in output?? + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, cn.bytesUsed.toString())); + } } return ret; } @@ -360,18 +369,19 @@ public ResourceIterator iterator(ContainerNode parent, Integer limit, Stri } NodeDAO dao = getDAO(); ResourceIterator ret = dao.iterator(parent, limit, start); - return new IdentWrapper(parent, ret); + return new ChildNodeWrapper(parent, ret); } - private class IdentWrapper implements ResourceIterator { + // wrapper to add parent, owner, and props to child nodes + private class ChildNodeWrapper implements ResourceIterator { private final ContainerNode parent; private final ResourceIterator childIter; - private IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); - private Map identCache = new TreeMap<>(); + private final IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + private final Map identCache = new TreeMap<>(); - IdentWrapper(ContainerNode parent, ResourceIterator childIter) { + ChildNodeWrapper(ContainerNode parent, ResourceIterator childIter) { this.parent = parent; this.childIter = childIter; // prime cache with caller @@ -395,6 +405,8 @@ public boolean hasNext() { public Node next() { Node ret = childIter.next(); ret.parent = parent; + + // owner Subject s = identCache.get(ret.ownerID); if (s == null) { s = identityManager.toSubject(ret.ownerID); @@ -402,6 +414,21 @@ public Node next() { } ret.owner = s; ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + + // props + if (ret instanceof DataNode) { + DataNode dn = (DataNode) ret; + if (dn.bytesUsed != null) { + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, dn.bytesUsed.toString())); + } // TBD: default to 0? + } else if (ret instanceof ContainerNode) { + ContainerNode cn = (ContainerNode) ret; + if (cn.bytesUsed != null) { + // TBD: long num = bytesUsed + cn.delta; // include unpropagated delta in output?? + ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, cn.bytesUsed.toString())); + } // TBD: default to 0? + } // no size for LinkNode + return ret; } From 4bad11d22e7c1270c9819edafb3d5813c6bbd2c2 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 23 Feb 2024 13:47:45 -0800 Subject: [PATCH 141/186] add vospace upgrade file --- .../src/main/resources/vospace.upgrade-0.15.sql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql diff --git a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql new file mode 100644 index 000000000..d418fc146 --- /dev/null +++ b/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql @@ -0,0 +1,8 @@ + +alter table .Node + add column bytesUsed bigint, + add column storageBucket varchar(5), + add column delta bigint +; + +create unique index node_storageID on .Node(storageID); \ No newline at end of file From 94d395e79b5cdb34eab3a22f1028bc4200645755 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 26 Feb 2024 12:53:11 -0800 Subject: [PATCH 142/186] add comment to inventory upgrade script --- .../src/main/resources/inventory.upgrade-0.15.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql index 2505aaac1..e0ce85860 100644 --- a/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql +++ b/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql @@ -1,2 +1,5 @@ +-- changes for this version: incomplete + alter table .HarvestState add column instanceID uuid; + From 7ab48e38a21e01fa68c3df96ca1c703c5129def3 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 26 Feb 2024 14:44:12 -0800 Subject: [PATCH 143/186] remove unused changes in StorageSite --- .../main/java/org/opencadc/inventory/StorageSite.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java b/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java index a112e0c63..13dad0efa 100644 --- a/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java +++ b/cadc-inventory/src/main/java/org/opencadc/inventory/StorageSite.java @@ -69,8 +69,6 @@ import java.net.URI; import java.util.Objects; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.UUID; import org.apache.log4j.Logger; @@ -86,15 +84,10 @@ public class StorageSite extends Entity implements Comparable { private static final Logger log = Logger.getLogger(StorageSite.class); private URI resourceID; - private String name; // deprecate? + private String name; private boolean allowRead; private boolean allowWrite; - // TODO: resourceID of trusted services - intended from config - private SortedSet trustedSigners = new TreeSet<>(); - // TODO: checksum of the pubkey of trusted services - actual from pub key retrieval - private SortedSet trustedPubKeyChecksums = new TreeSet<>(); - /** * Create a new StorageSite. * From b68b237772f0e24dad565cacb74c6f60086b30fb Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 27 Feb 2024 10:54:45 -0800 Subject: [PATCH 144/186] update dependencies --- cadc-inventory-db/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index f6f235f80..1b5a7ca8f 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -25,10 +25,10 @@ def git_url = 'https://github.com/opencadc/storage-inventory' mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { - compile 'org.opencadc:cadc-util:[1.10.3,2.0)' + compile 'org.opencadc:cadc-util:[1.11.0,2.0)' compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' - compile 'org.opencadc:cadc-vos:[2.0,3.0)' + compile 'org.opencadc:cadc-vos:[2.0.5,3.0)' testCompile 'junit:junit:[4.0,)' From 44a1c29623a250d36bf3b0f96c37e7aead5a3295 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 27 Feb 2024 15:00:24 -0800 Subject: [PATCH 145/186] vault: transfer generator bug, API doc fix --- .../main/java/org/opencadc/vault/VaultTransferGenerator.java | 4 ++-- vault/src/main/webapp/service.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 97064afb8..0e6a9da79 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -193,8 +193,8 @@ private List handleDataNode(DataNode node, String filename, Transfer t log.debug("requested protocol: " + p); if (!protoURIs.contains(p.getUri())) { Protocol anonProto = new Protocol(p.getUri()); - anonProto.setSecurityMethod(Standards.SECURITY_METHOD_ANON); - artifactTrans.getProtocols().add(p); + //anonProto.setSecurityMethod(Standards.SECURITY_METHOD_ANON); + artifactTrans.getProtocols().add(anonProto); protoURIs.add(p.getUri()); log.debug("Added anon protocol for " + p.getUri()); } diff --git a/vault/src/main/webapp/service.yaml b/vault/src/main/webapp/service.yaml index 4b0268c45..0ed9ed7a9 100644 --- a/vault/src/main/webapp/service.yaml +++ b/vault/src/main/webapp/service.yaml @@ -96,7 +96,7 @@ paths: $ref: '#/definitions/Node' delete: description: | - Delete a node. When the target is a ContainerNode, all its children (the contents of the container) SHALL also be deleted. + Delete a node. To delete a non-empty ContainerNode, see async-delete below. tags: - Nodes responses: @@ -315,7 +315,7 @@ paths: schema: $ref: '#/definitions/Error' parameters: - - name: nodeURI + - name: target in: query description: The base node (typically a container) to recursively delete required: true From 3cd0dfd5e3a5b1ba2d94dee82b6b56a0000a1dd9 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 28 Feb 2024 11:05:22 -0800 Subject: [PATCH 146/186] updates to vault-quota design --- vault-quota/Design.md | 49 +++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/vault-quota/Design.md b/vault-quota/Design.md index d6b167a9d..08f81f0b9 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -1,7 +1,7 @@ # vault quota design/algorithms The definitive source of content-length (file size) of a DataNode comes from the -`inventory.Artifact` table and it not known until a PUT to storage is completed. +`inventory.Artifact` table and is not known until a PUT to storage is completed. In the case of a `vault` service co-located with a single storage site (`minoc`), the new Artifact is visible in the database as soon as the PUT to `minoc` is completed. In the case of a `vault` service co-located with a global SI, the new @@ -13,6 +13,22 @@ The design below only takes into account incremental propagation of space used by stored files. It is not complete/verified until we also come up with a validation algorithm that can detect and fix discrepancies in a live `vault`. +## operations that effect node size +The following operations effect node size: +* delete node removed the node and applies a negative delta to parent +* recursive delete will need to update twice as many nodes in small transactions +* move node will applies a negative delta to previous parent and positive delta to new parent +* copy applies a positive delta to new parent +* transfer negotiation needs to check allocationNode quota vs size to allow a put to proceed + +This has to be done entirely inside the NodePersistence implementation; that should be feasible +since the argument of NodePersistence methods is the previously retrieved node with the full parent +tree intact. It's not clear which of these will require changes in cadc-vos-server, but if they do +it will need to be possible for them to be optional and/or gracefully not do anything. + +In any case, any solution with container node delta(s) is inherently multi-threaded because +user requests can modify them. + ## DataNode size algorithm: This is an event watcher that gets Artifact events (after a PUT) and intiates the propagation of sizes (space used). @@ -51,7 +67,7 @@ for each ContainerNode: The above sequence finds candidate propagations, locks (order: parent-child), and applies the propagation. This moves the outstanding delta up the tree one level. If the sequence acts on multiple child containers before the parent, the delta(s) naturally -_merge_ and there are fewer larger delta propagations in the upper part of the tree. +_merge_ and fewer delta propagations occur in the upper part of the tree. The most generic implementation is to iterate over container nodes: ``` @@ -105,27 +121,24 @@ Iterator niter = nodeDAO.iterator(bucket); // storageBucket,stora ### ContainerNode vs child nodes discrepancies These can be validated in ``` -discrepancy 1: container size > sum(child sizes) -explanation: un-propagated delete -evidence: sum(child delta) < 0 -action: none - -discrepancy 1: container size > sum(child sizes) -explanation: bug -evidence: sum(child delta) == 0 -action: fix container size, set container.delta - -discrepancy 1: container size < sum(child sizes) -explanation: un-propagated delta -evidence: sum(child delta) > 0 +discrepancy 1: container size != sum(child size) +explanation: un-propagated delta from put or delete +evidence: sum(child delta) != 0 action: none -discrepancy 1: container size < sum(child sizes) +discrepancy 1: container size != sum(child size) explanation: bug evidence: sum(child delta) == 0 -action: fix container size, set container.delta +action: fix?? container size, set container.delta ``` -Required lock order: locks the parent of a parent-children relationship. +Required lock order: locks the parent of a parent-children relationship so propagations are blocked, +then do the aggregate query (select sum(child size), sum(child delta) where parentID=?) +but child state is still not stable (delete child node, move child out, copy/move node in, +sync child nodes from remote) so all of these would have to lock in the same order to avoid deadlock. +I don't see any way to avoid deadlocks when user requests can lock multiple nodes. + +Recursive delete, container size propagation, datanode validation, and container validation can will +all potentially modify child delta(s). The most generic implementation is to iterate over container nodes: ``` From 5c2dcc0a751d47a7a03baa1cec759e864f0ae7e7 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 4 Mar 2024 12:02:52 -0800 Subject: [PATCH 147/186] vault bytesUsed rework --- .../org/opencadc/vospace/db/NodeDAOTest.java | 36 +++---------- .../opencadc/inventory/db/SQLGenerator.java | 5 -- .../java/org/opencadc/vospace/db/NodeDAO.java | 15 ------ .../src/main/resources/vospace.Node.sql | 10 ++-- .../main/resources/vospace.upgrade-0.15.sql | 3 +- .../opencadc/vault/NodePersistenceImpl.java | 50 +++++++++---------- .../org/opencadc/vault/files/GetAction.java | 10 ++-- .../org/opencadc/vault/files/HeadAction.java | 11 ++-- 8 files changed, 48 insertions(+), 92 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java index dbd217a20..4f4e52e9a 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/NodeDAOTest.java @@ -217,7 +217,6 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, ContainerNode c = (ContainerNode) a; Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); Assert.assertEquals(orig.bytesUsed, c.bytesUsed); - Assert.assertEquals(orig.delta, c.delta); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); @@ -253,8 +252,6 @@ public void testPutGetUpdateDeleteContainerNode() throws InterruptedException, ContainerNode uc = (ContainerNode) updated; Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); Assert.assertEquals(orig.bytesUsed, uc.bytesUsed); - Assert.assertEquals(orig.delta, uc.delta); - nodeDAO.delete(orig.getID()); Node gone = nodeDAO.get(orig.getID()); @@ -318,7 +315,6 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException ContainerNode c = (ContainerNode) a; Assert.assertEquals(orig.inheritPermissions, c.inheritPermissions); Assert.assertEquals(orig.bytesUsed, c.bytesUsed); - Assert.assertEquals(orig.delta, c.delta); // these are set in put Assert.assertEquals(orig.getMetaChecksum(), a.getMetaChecksum()); @@ -358,7 +354,6 @@ public void testPutGetUpdateDeleteContainerNodeMax() throws InterruptedException ContainerNode uc = (ContainerNode) updated; Assert.assertEquals(orig.inheritPermissions, uc.inheritPermissions); Assert.assertEquals(orig.bytesUsed, uc.bytesUsed); - Assert.assertEquals(orig.delta, uc.delta); nodeDAO.delete(orig.getID()); Node gone = nodeDAO.get(orig.getID()); @@ -622,7 +617,6 @@ public void testUpdateNodeSize() throws InterruptedException, Assert.assertEquals(cnode.getName(), c1.getName()); Assert.assertEquals(root.getID(), c1.parentID); Assert.assertNull(c1.bytesUsed); - Assert.assertEquals(0L, c1.delta); final DataNode d1 = (DataNode) nodeDAO.get(dnode.getID()); Assert.assertNotNull(d1); @@ -637,37 +631,19 @@ public void testUpdateNodeSize() throws InterruptedException, log.info("update DataNode"); d1.bytesUsed = 123L; - nodeDAO.put(d1, true, false); + nodeDAO.put(d1); final DataNode d2 = (DataNode) nodeDAO.get(dnode.getID()); Assert.assertNotNull(d2); Assert.assertNotNull(d2.bytesUsed); Assert.assertEquals(d1.bytesUsed, d2.bytesUsed); - URI dcs2 = d2.computeMetaChecksum(MessageDigest.getInstance("MD5")); - Assert.assertEquals(dnode.getMetaChecksum(), dcs2); // no update - Assert.assertEquals(dnode.getLastModified(), d2.getLastModified()); // no update - log.info("update ContainerNode.delta"); - c1.delta = 123L; - nodeDAO.put(c1, true, false); + log.info("update ContainerNode.bytesUsed"); + c1.bytesUsed = 123L; + nodeDAO.put(c1); final ContainerNode c2 = (ContainerNode) nodeDAO.get(cnode.getID()); Assert.assertNotNull(c2); - Assert.assertEquals(123L, c2.delta); - URI ccs2 = c2.computeMetaChecksum(MessageDigest.getInstance("MD5")); - Assert.assertEquals(cnode.getMetaChecksum(), ccs2); // no update - Assert.assertEquals(cnode.getLastModified(), c2.getLastModified()); // no update - - log.info("update ContainerNode.bytesUsed"); - c2.bytesUsed = c2.delta; - c2.delta = 0L; - nodeDAO.put(c2, true, false); - final ContainerNode c3 = (ContainerNode) nodeDAO.get(cnode.getID()); - Assert.assertNotNull(c3); - Assert.assertEquals(0L, c3.delta); - Assert.assertNotNull(c3.bytesUsed); - Assert.assertEquals(123L, c3.bytesUsed.longValue()); - URI ccs3 = c3.computeMetaChecksum(MessageDigest.getInstance("MD5")); - Assert.assertEquals(cnode.getMetaChecksum(), ccs3); // no update - Assert.assertEquals(cnode.getLastModified(), c3.getLastModified()); // no update + Assert.assertNotNull(c2.bytesUsed); + Assert.assertEquals(123L, c2.bytesUsed.longValue()); nodeDAO.delete(dnode.getID()); nodeDAO.delete(cnode.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index ffe8f8ef1..d166f869f 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -261,7 +261,6 @@ protected void init() { "readWriteGroups", "properties", "inheritPermissions", - "delta", "bytesUsed", "busy", "storageID", @@ -1520,10 +1519,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce if (value instanceof ContainerNode) { ContainerNode cn = (ContainerNode) value; safeSetBoolean(prep, col++, cn.inheritPermissions); - safeSetLong(prep, col++, cn.delta); } else { safeSetBoolean(prep, col++, null); - safeSetLong(prep, col++, null); } // bytesUsed is in between CN and DN specific columns @@ -1975,7 +1972,6 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro final String rawRWG = rs.getString(col++); final String rawProps = rs.getString(col++); final Boolean inheritPermissions = Util.getBoolean(rs, col++); - final Long delta = Util.getLong(rs, col++); final Long bytesUsed = Util.getLong(rs, col++); final Boolean busy = Util.getBoolean(rs, col++); final URI storageID = Util.getURI(rs, col++); @@ -1991,7 +1987,6 @@ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) thro ContainerNode cn = new ContainerNode(id, name); cn.inheritPermissions = inheritPermissions; cn.bytesUsed = bytesUsed; - cn.delta = (delta == null ? 0 : delta); ret = cn; } else if (nodeType.equals("D")) { DataNode dn = new DataNode(id, name, storageID); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index d2e01e854..bee209489 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -99,21 +99,6 @@ public void put(Node val) { super.put(val); } - /** - * Update that optionally includes extended content. Extended content is computed - * or transient fields that do not trigger a metaChecksum change so would normally - * be skipped. - * - * @param val the Node to update - * @param extendedUpdate true to force db update - * @param timestampUpdate true to force lastModified update - */ - @Override - protected void put(Node val, boolean extendedUpdate, boolean timestampUpdate) { - super.put(val, extendedUpdate, timestampUpdate); - } - - @Override public Node lock(Node n) { if (n == null) { diff --git a/cadc-inventory-db/src/main/resources/vospace.Node.sql b/cadc-inventory-db/src/main/resources/vospace.Node.sql index 8e96a67f0..c488e2499 100644 --- a/cadc-inventory-db/src/main/resources/vospace.Node.sql +++ b/cadc-inventory-db/src/main/resources/vospace.Node.sql @@ -17,12 +17,12 @@ create table .Node ( -- ContainerNode inheritPermissions boolean, - delta bigint; -- DataNode busy boolean, - storageID varchar(512), - storageBucket varchar(5), + bytesUsed bigint, + storageID varchar(512), -- Artifact.uri + storageBucket varchar(5), -- Artifact.storageBucket -- LinkNode target text, @@ -32,8 +32,12 @@ create table .Node ( id uuid not null primary key ); +-- usage: vault path navigation create unique index node_parent_child on .Node(parentID,name); +-- usage: Node metadata-sync create index node_lastmodified on .Node(lastModified); +-- usage: vault incremental Artifact to Node for bytesUsed +-- usage: vault Node vs Artifact validation create unique index node_storageID on .Node(storageID); \ No newline at end of file diff --git a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql index d418fc146..bfb33166e 100644 --- a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql +++ b/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql @@ -1,8 +1,7 @@ alter table .Node add column bytesUsed bigint, - add column storageBucket varchar(5), - add column delta bigint + add column storageBucket varchar(5) ; create unique index node_storageID on .Node(storageID); \ No newline at end of file diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index f41112cea..4553a829c 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -265,6 +265,11 @@ public ContainerNode getRootNode() { return root; } + @Override + public Set getAllocationHolders() { + throw new UnsupportedOperationException(); + } + @Override public Set getAdminProps() { return Collections.unmodifiableSet(ADMIN_PROPS); @@ -323,9 +328,7 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (cd != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTDATE, df.format(cd))); } - // TODO: a.getContentLength() is correct, but might differ from dn.bytesUsed?? eg eventual consistency - // child listing iterator can only report dn.bytesUsed - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, a.getContentLength().toString())); + // assume MD5 ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTMD5, a.getContentChecksum().getSchemeSpecificPart())); @@ -335,17 +338,17 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (a.contentType != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); } - } else if (dn.bytesUsed != null) { - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, dn.bytesUsed.toString())); - } else { - // default size to 0 - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, "0")); + // TODO: a.getContentLength() is correct, but might differ from dn.bytesUsed due to eventual consistency + // child listing iterator can only report dn.bytesUsed + // correct or consistency with listing?? + + // currently needed for consistency-requiring FilesTest + log.warn("DataNode.bytesUsed: " + dn.bytesUsed + " -> " + a.getContentLength()); + dn.bytesUsed = a.getContentLength(); } - } else if (ret instanceof ContainerNode) { - ContainerNode cn = (ContainerNode) ret; - if (cn.bytesUsed != null) { - // TBD: long num = bytesUsed + cn.delta; // include unpropagated delta in output?? - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, cn.bytesUsed.toString())); + if (dn.bytesUsed == null) { + log.warn("DataNode.bytesUsed: 0 (no artifact)"); + dn.bytesUsed = 0L; // no data stored } } return ret; @@ -415,20 +418,12 @@ public Node next() { ret.owner = s; ret.ownerDisplay = identityManager.toDisplayString(ret.owner); - // props if (ret instanceof DataNode) { DataNode dn = (DataNode) ret; - if (dn.bytesUsed != null) { - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, dn.bytesUsed.toString())); - } // TBD: default to 0? - } else if (ret instanceof ContainerNode) { - ContainerNode cn = (ContainerNode) ret; - if (cn.bytesUsed != null) { - // TBD: long num = bytesUsed + cn.delta; // include unpropagated delta in output?? - ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_CONTENTLENGTH, cn.bytesUsed.toString())); - } // TBD: default to 0? - } // no size for LinkNode - + if (dn.bytesUsed == null) { + dn.bytesUsed = 0L; + } + } return ret; } @@ -735,4 +730,9 @@ public void delete(Node node) throws TransientException { } } } + + // needed by vault-migrate to configure a HarvestStateDAO for delete processing + public Map getNodeDaoConfig() { + return nodeDaoConfig; + } } diff --git a/vault/src/main/java/org/opencadc/vault/files/GetAction.java b/vault/src/main/java/org/opencadc/vault/files/GetAction.java index b8363eea4..6337dbccb 100644 --- a/vault/src/main/java/org/opencadc/vault/files/GetAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/GetAction.java @@ -96,12 +96,10 @@ public GetAction() { public void doAction() throws Exception { DataNode node = resolveAndSetMetadata(); - for (NodeProperty prop : node.getProperties()) { - if (prop.getKey().equals(VOS.PROPERTY_URI_CONTENTLENGTH) && prop.getValue().equals("0")) { - // empty file - syncOutput.setCode(HttpURLConnection.HTTP_NO_CONTENT); - return; - } + if (node.bytesUsed == null || node.bytesUsed == 0L) { + // empty file + syncOutput.setCode(HttpURLConnection.HTTP_NO_CONTENT); + return; } VOSURI targetURI = localServiceURI.getURI(node); diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java index 6344ef72c..8e4505c3b 100644 --- a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -139,15 +139,14 @@ DataNode resolveAndSetMetadata() throws Exception { if (!(node instanceof DataNode)) { throw new IllegalArgumentException("Resolved target is not a data node: " + Utils.getPath(node)); } + + log.debug("node path resolved: " + node.getName() + " type: " + node.getClass().getName()); - VOSURI nodeURI = localServiceURI.getURI(node); - - log.debug("node path resolved: " + node.getName()); - log.debug("node type: " + node.getClass().getCanonicalName()); - syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + nodeURI.getName() + "\""); + DataNode dn = (DataNode) node; + syncOutput.setHeader("Content-Length", dn.bytesUsed); + syncOutput.setHeader("Content-Disposition", "inline; filename=\"" + node.getName() + "\""); syncOutput.setHeader("Content-Type", node.getPropertyValue(VOS.PROPERTY_URI_TYPE)); syncOutput.setHeader("Content-Encoding", node.getPropertyValue(VOS.PROPERTY_URI_CONTENTENCODING)); - syncOutput.setHeader("Content-Length", node.getPropertyValue(VOS.PROPERTY_URI_CONTENTLENGTH)); if (node.getPropertyValue(VOS.PROPERTY_URI_DATE) != null) { Date lastMod = NodeWriter.getDateFormat().parse(node.getPropertyValue(VOS.PROPERTY_URI_DATE)); syncOutput.setLastModified(lastMod); From 2e4b8d68f7ea175ecc2b236ca26249fedc313f29 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 5 Mar 2024 10:47:26 -0800 Subject: [PATCH 148/186] vault: add allocationParent support --- vault/README.md | 12 ++++ .../opencadc/vault/NodePersistenceImpl.java | 62 ++++++++++++++++++- .../org/opencadc/vault/VaultInitAction.java | 29 ++++++++- 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/vault/README.md b/vault/README.md index 02e701eab..4e3c5fbc9 100644 --- a/vault/README.md +++ b/vault/README.md @@ -83,6 +83,9 @@ org.opencadc.vault.resourceID = ivo://{authority}/{name} # consistency settings org.opencadc.vault.consistency.preventNotFound=true|false +# (optional) identify which container nodes are allocations +org.opencadc.vault.allocationParent = {top level node} + # vault database settings org.opencadc.vault.inventory.schema = {inventory schema name} org.opencadc.vault.vospace.schema = {vospace schema name} @@ -102,6 +105,15 @@ _all known_ sites. It only makes sense to enable this when `vault` is running in `raven` and/or `fenwick` instances syncing artifact metadata. This feature introduces an overhead for the genuine not-found cases: transfer negotiation to GET the file that was never PUT. +The _allocationParent_ is a path to a container node (directory) which contains space allocations. An allocation +is owned by a user (uisually different from the _rootOwner_ admin user) who is responsible for the allocation +and all conntent therein. The owner of an allocation is granted additional permissions within their +allocation (they can read/write/delete anything) so the owner cannot be blocked from access to any content +within their allocation. This probably only matters for multi-user projects. Multiple _allocationParent_(s) may +be configured to organise the top level of the content (e.g. /home and /projects). Paths configured to be +_allocationParent_(s) will be automatically created (if necessary), owned by the _rootOwner_, and will be +anonymously readable (public). Limitation: only a single level of top-level _allocationParent_(s) are supported. + The _inventory.schema_ name is the name of the database schema used for all inventory database objects. This currently must be "inventory" due to configuration limitations in luskan. diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 4553a829c..611ebd8a4 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -80,10 +80,12 @@ import java.io.IOException; import java.net.URI; import java.text.DateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; @@ -160,6 +162,7 @@ public class NodePersistenceImpl implements NodePersistence { private final boolean singlePool; private final ContainerNode root; + private final List allocationParents = new ArrayList<>(); private final Namespace storageNamespace; private final boolean localGroupsOnly; @@ -191,6 +194,35 @@ public NodePersistenceImpl(URI resourceID) { root.ownerID = identityManager.toOwner(root.owner); root.isPublic = true; root.inheritPermissions = false; + + // allocations + for (String ap : VaultInitAction.getAllocationParents(config)) { + if (ap.isEmpty()) { + // allocations are in root + allocationParents.add(root); + log.info("allocationParent: /"); + } else { + try { + + // simple top-level names only + ContainerNode cn = (ContainerNode) get(root, ap); + String str = ""; + if (cn == null) { + cn = new ContainerNode(ap); + cn.parent = root; + str = "created/"; + } + cn.isPublic = true; + cn.owner = root.owner; + cn.inheritPermissions = false; + put(cn); + allocationParents.add(cn); + log.info(str + "loaded allocationParent: /" + cn.getName()); + } catch (NodeNotSupportedException bug) { + throw new RuntimeException("BUG: failed to update isPublic=true on allocationParent " + ap, bug); + } + } + } String ns = config.getFirstPropertyValue(VaultInitAction.STORAGE_NAMESPACE_KEY); this.storageNamespace = new Namespace(ns); @@ -266,8 +298,34 @@ public ContainerNode getRootNode() { } @Override - public Set getAllocationHolders() { - throw new UnsupportedOperationException(); + public boolean isAllocation(ContainerNode cn) { + if (cn.parent == null) { + return false; // root is never an allocation + } + ContainerNode p = cn.parent; + for (ContainerNode ap : allocationParents) { + if (absoluteEquals(p.parent, ap)) { + return true; + } + } + return false; + } + + private boolean absoluteEquals(ContainerNode c1, ContainerNode c2) { + // note: cavern does not use/preserve Node.id except for root + if (!c1.getName().equals(c2.getName())) { + return false; + } + // same name, check parents + if (c1.parent == null && c2.parent == null) { + // both root + return true; + } + if (c1.parent == null || c2.parent == null) { + // one is root + return false; + } + return absoluteEquals(c1.parent, c2.parent); } @Override diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 6e62a021d..6b5faf21b 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -69,6 +69,7 @@ import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.InitAction; +import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; import ca.nrc.cadc.util.RsaSignatureGenerator; @@ -76,6 +77,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.KeyPair; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.TreeMap; import javax.naming.Context; @@ -115,9 +118,8 @@ public class VaultInitAction extends InitAction { static final String INVENTORY_SCHEMA_KEY = VAULT_KEY + ".inventory.schema"; static final String VOSPACE_SCHEMA_KEY = VAULT_KEY + ".vospace.schema"; static final String SINGLE_POOL_KEY = VAULT_KEY + ".singlePool"; - - static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; // numeric? - + static final String ALLOCATION_PARENT = VAULT_KEY + ".allocationParent"; + static final String ROOT_OWNER = VAULT_KEY + ".root.owner"; static final String STORAGE_NAMESPACE_KEY = VAULT_KEY + ".storage.namespace"; MultiValuedProperties props; @@ -125,6 +127,7 @@ public class VaultInitAction extends InitAction { private Namespace storageNamespace; private Map vosDaoConfig; private Map invDaoConfig; + private List allocationParents = new ArrayList<>(); private String jndiNodePersistence; private String jndiPreauthKeys; @@ -239,6 +242,26 @@ static MultiValuedProperties getConfig() { return mvp; } + static List getAllocationParents(MultiValuedProperties props) { + List ret = new ArrayList<>(); + for (String sap : props.getProperty(ALLOCATION_PARENT)) { + String ap = sap; + if (ap.charAt(0) == '/') { + ap = ap.substring(1); + } + if (ap.length() > 0 && ap.charAt(ap.length() - 1) == '/') { + ap = ap.substring(0, ap.length() - 1); + } + if (ap.indexOf('/') >= 0) { + throw new InvalidConfigException("invalid " + ALLOCATION_PARENT + ": " + sap + + " reason: must be a top-level container node name"); + } + // empty string means root, otherwise child of root + ret.add(ap); + } + return ret; + } + static Map getDaoConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now From 2cec055b06c60bdecb2ae47435fa7d11189137ab Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Fri, 8 Mar 2024 13:19:18 -0800 Subject: [PATCH 149/186] remove duplicate column from Node table sql --- cadc-inventory-db/src/main/resources/vospace.Node.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cadc-inventory-db/src/main/resources/vospace.Node.sql b/cadc-inventory-db/src/main/resources/vospace.Node.sql index c488e2499..3e23b5b2b 100644 --- a/cadc-inventory-db/src/main/resources/vospace.Node.sql +++ b/cadc-inventory-db/src/main/resources/vospace.Node.sql @@ -6,7 +6,6 @@ create table .Node ( nodeType char(1) not null, ownerID varchar(256) not null, - bytesUsed bigint, isPublic boolean, isLocked boolean, readOnlyGroups text, @@ -21,8 +20,9 @@ create table .Node ( -- DataNode busy boolean, bytesUsed bigint, - storageID varchar(512), -- Artifact.uri - storageBucket varchar(5), -- Artifact.storageBucket + -- Artifact.uri and Artifact.uriBucket + storageID varchar(512), + storageBucket varchar(5), -- LinkNode target text, From 91868b35be156bb55d3f6d9ea741274c1060925b Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 8 Mar 2024 16:10:53 -0800 Subject: [PATCH 150/186] Initial version --- cadc-inventory-db/build.gradle | 2 +- .../org/opencadc/ArtifactSyncWorkerTest.java | 235 ++++++++++++++++++ .../inventory/ArtifactSyncWorker.java | 130 ++++++++++ 3 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java create mode 100644 cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index f6f235f80..e556bf78f 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -28,7 +28,7 @@ dependencies { compile 'org.opencadc:cadc-util:[1.10.3,2.0)' compile 'org.opencadc:cadc-gms:[1.0.0,)' compile 'org.opencadc:cadc-inventory:[0.9.4,)' - compile 'org.opencadc:cadc-vos:[2.0,3.0)' + compile 'org.opencadc:cadc-vos:[2.0.5,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java new file mode 100644 index 000000000..e98c86e8b --- /dev/null +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java @@ -0,0 +1,235 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.net.URI; +import java.sql.Connection; +import java.util.Date; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.ArtifactSyncWorker; +import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.HarvestState; +import org.opencadc.inventory.db.HarvestStateDAO; +import org.opencadc.inventory.db.SQLGenerator; +import org.opencadc.inventory.db.TestUtil; +import org.opencadc.inventory.db.version.InitDatabaseSI; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.db.InitDatabaseVOS; +import org.opencadc.vospace.db.NodeDAO; + +/** + * + * @author adriand + */ +public class ArtifactSyncWorkerTest { + private static final Logger log = Logger.getLogger(ArtifactSyncWorkerTest.class); + + static { + Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); + Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace", Level.INFO); + Log4jInit.setLevel("org.opencadc.vospace.db", Level.INFO); + } + + HarvestStateDAO harvestStateDAO; + NodeDAO nodeDAO; + ArtifactDAO artifactDAO; + + + public ArtifactSyncWorkerTest() throws Exception { + DBConfig dbrc = new DBConfig(); + ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); + DBUtil.PoolConfig pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); + DBUtil.createJNDIDataSource("jdbc/ArtifactSyncWorkerTest-node", pool); + + Map config = new TreeMap<>(); + config.put(SQLGenerator.class.getName(), SQLGenerator.class); + config.put("jndiDataSourceName", "jdbc/ArtifactSyncWorkerTest-node"); + config.put("database", TestUtil.DATABASE); + config.put("invSchema", TestUtil.SCHEMA); + config.put("genSchema", TestUtil.SCHEMA); + config.put("vosSchema", TestUtil.VOS_SCHEMA); + + this.harvestStateDAO = new HarvestStateDAO(); + harvestStateDAO.setConfig(config); + this.nodeDAO = new NodeDAO(); + nodeDAO.setConfig(config); + + pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); + DBUtil.createJNDIDataSource("jdbc/ArtifactSyncWorkerTest-artifact", pool); + + config.put("jndiDataSourceName", "jdbc/ArtifactSyncWorkerTest-artifact"); + + this.artifactDAO = new ArtifactDAO(); + artifactDAO.setConfig(config); + } + + @Before + public void init_cleanup() throws Exception { + log.info("init database..."); + InitDatabaseSI initSI = new InitDatabaseSI(artifactDAO.getDataSource(), TestUtil.DATABASE, TestUtil.SCHEMA); + initSI.doInit(); + log.info("init SI database... OK"); + InitDatabaseVOS initVOS = new InitDatabaseVOS(nodeDAO.getDataSource(), TestUtil.DATABASE, TestUtil.VOS_SCHEMA); + initVOS.doInit(); + log.info("init VOS database... OK"); + + log.info("clearing old content..."); + // src DB + SQLGenerator gen = artifactDAO.getSQLGenerator(); + DataSource ds = artifactDAO.getDataSource(); + String sql = "delete from " + gen.getTable(Artifact.class); + Connection con = ds.getConnection(); + con.createStatement().execute(sql); + con.close(); + + gen = harvestStateDAO.getSQLGenerator(); + ds = harvestStateDAO.getDataSource(); + sql = "delete from " + gen.getTable(HarvestState.class); + con = ds.getConnection(); + con.createStatement().execute(sql); + + gen = nodeDAO.getSQLGenerator(); + sql = "delete from " + gen.getTable(ContainerNode.class); + log.info("pre-test cleanup: " + sql); + con.createStatement().execute(sql); + con.close(); + + log.info("clearing old content... OK"); + } + + @Test + public void testSyncArtifact() { + UUID rootID = new UUID(0L, 0L); + ContainerNode root = new ContainerNode(rootID, "root"); + + // create the data node + DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); + orig.parent = root; + orig.ownerID = "the-owner"; + orig.isPublic = true; + orig.isLocked = false; + nodeDAO.put(orig); + + // get-by-id + Node a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertNull(orig.bytesUsed); + + // create the corresponding artifact + Long length = 666L; + Namespace siNamespace = new Namespace("cadc:VOS/"); + String artifactURI = siNamespace.getNamespace() + "root/filename"; + Artifact expected = new Artifact( + URI.create(artifactURI), + URI.create("md5:d41d8cd98f00b204e9800998ecf8427e"), + new Date(), + length); + log.info("expected: " + expected); + + artifactDAO.put(expected); + Artifact actual = artifactDAO.get(expected.getID()); + Assert.assertNotNull(actual); + Assert.assertEquals(length, actual.getContentLength()); + + String hsName = "ArtifactSize"; + URI resourceID = URI.create("ivo://myorg.org/vospace"); + HarvestState hs = new HarvestState(hsName, resourceID); + harvestStateDAO.put(hs); + hs = harvestStateDAO.get(hsName, resourceID); + + ArtifactSyncWorker asWorker = new ArtifactSyncWorker(harvestStateDAO, hs, artifactDAO, siNamespace); + asWorker.run(); + + a = nodeDAO.get(orig.getID()); + Assert.assertNotNull(a); + log.info("found: " + a.getID() + " aka " + a); + Assert.assertEquals(length, orig.bytesUsed); + + + } + +} diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java new file mode 100644 index 000000000..5915511b7 --- /dev/null +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java @@ -0,0 +1,130 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ + */ + +package org.opencadc.inventory; + +import ca.nrc.cadc.io.ResourceIterator; +import org.apache.log4j.Logger; +import org.opencadc.inventory.db.ArtifactDAO; +import org.opencadc.inventory.db.HarvestState; +import org.opencadc.inventory.db.HarvestStateDAO; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.db.NodeDAO; + +/** + * This class performs the work of synchronizing the size of Data Node between backend storage and Node Persistence + * + * @author adriand + */ +public class ArtifactSyncWorker implements Runnable { + private static final Logger log = Logger.getLogger(ArtifactSyncWorker.class); + + private final HarvestState harvestState; + private final NodeDAO nodeDAO = null; + private final ArtifactDAO artifactDAO; + private final HarvestStateDAO harvestStateDAO; + private final Namespace storageNamespace; + + public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, Namespace namespace) { + this.harvestState = harvestState; + this.harvestStateDAO = harvestStateDAO; + //this.nodeDAO = new NodeDAO(harvestStateDAO); + this.artifactDAO = artifactDAO; + this.storageNamespace = namespace; + } + + @Override + public void run() { + log.debug("Start harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + + //ResourceIterator iter = artifactDAO.iterator(storageNamespace, null, harvestState.curLastModified, true); + ResourceIterator iter = null; + while (iter.hasNext()) { + Artifact artifact = iter.next(); + DataNode node = null; //nodeDAO.getDataNode(artifact.getURI()); + if (!node.bytesUsed.equals(artifact.getContentLength())) { + node.bytesUsed = artifact.getContentLength(); + nodeDAO.getTransactionManager().startTransaction(); + try { + nodeDAO.put(node); + harvestState.curLastModified = artifact.getLastModified(); + harvestState.curID = node.getID(); + harvestStateDAO.put(harvestState, true); + nodeDAO.getTransactionManager().commitTransaction(); + log.debug("Updated size of data node " + node.getName()); + } catch (Exception ex) { + log.debug("Failed to update data node size for " + node.getName(), ex); + nodeDAO.getTransactionManager().rollbackTransaction(); + throw ex; + } + } + } + + log.debug("End harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + + } +} From 3fed5969a4ef86d1af5893ee5579ede16e6e32d0 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Fri, 8 Mar 2024 17:59:28 -0800 Subject: [PATCH 151/186] Updated after pulling upstream updates --- .../org/opencadc/ArtifactSyncWorkerTest.java | 45 ++++++++++------- .../inventory/ArtifactSyncWorker.java | 50 +++++++++++-------- .../java/org/opencadc/vospace/db/NodeDAO.java | 5 ++ 3 files changed, 60 insertions(+), 40 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java index e98c86e8b..f21463291 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java @@ -94,7 +94,6 @@ import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.Node; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.db.NodeDAO; @@ -181,39 +180,38 @@ public void init_cleanup() throws Exception { } @Test - public void testSyncArtifact() { + public void testSyncArtifact() throws Exception { UUID rootID = new UUID(0L, 0L); ContainerNode root = new ContainerNode(rootID, "root"); // create the data node - DataNode orig = new DataNode(UUID.randomUUID(), "data-test", URI.create("cadc:vault/" + UUID.randomUUID())); - orig.parent = root; + Namespace siNamespace = new Namespace("myorg:VOS/"); + URI artifactURI = URI.create(siNamespace.getNamespace() + UUID.randomUUID()); + DataNode orig = new DataNode(UUID.randomUUID(), "data-test", artifactURI); + orig.parentID = root.getID(); orig.ownerID = "the-owner"; orig.isPublic = true; orig.isLocked = false; nodeDAO.put(orig); // get-by-id - Node a = nodeDAO.get(orig.getID()); - Assert.assertNotNull(a); - log.info("found: " + a.getID() + " aka " + a); + DataNode actual = (DataNode)nodeDAO.get(orig.getID()); + Assert.assertNotNull(actual); + log.info("found: " + actual.getID() + " aka " + actual); Assert.assertNull(orig.bytesUsed); // create the corresponding artifact - Long length = 666L; - Namespace siNamespace = new Namespace("cadc:VOS/"); - String artifactURI = siNamespace.getNamespace() + "root/filename"; Artifact expected = new Artifact( - URI.create(artifactURI), + artifactURI, URI.create("md5:d41d8cd98f00b204e9800998ecf8427e"), new Date(), - length); + 666L); log.info("expected: " + expected); artifactDAO.put(expected); - Artifact actual = artifactDAO.get(expected.getID()); + Artifact actualArtifact = artifactDAO.get(expected.getID()); Assert.assertNotNull(actual); - Assert.assertEquals(length, actual.getContentLength()); + Assert.assertEquals(expected.getContentLength(), actualArtifact.getContentLength()); String hsName = "ArtifactSize"; URI resourceID = URI.create("ivo://myorg.org/vospace"); @@ -224,11 +222,22 @@ public void testSyncArtifact() { ArtifactSyncWorker asWorker = new ArtifactSyncWorker(harvestStateDAO, hs, artifactDAO, siNamespace); asWorker.run(); - a = nodeDAO.get(orig.getID()); - Assert.assertNotNull(a); - log.info("found: " + a.getID() + " aka " + a); - Assert.assertEquals(length, orig.bytesUsed); + actual = (DataNode)nodeDAO.get(orig.getID()); + Assert.assertNotNull(actual); + log.info("found: " + actual.getID() + " aka " + actual); + Assert.assertEquals(expected.getContentLength(), actual.bytesUsed); + // update + Thread.sleep(20L); + expected = new Artifact(expected.getURI(), expected.getMetaChecksum(), new Date(), 333L); + artifactDAO.put(expected); + actual = (DataNode)nodeDAO.get(orig.getID()); + Assert.assertNotEquals(expected.getContentLength(), actual.bytesUsed); + + // do the update + asWorker.run(); + actual = (DataNode)nodeDAO.get(orig.getID()); + Assert.assertEquals(expected.getContentLength(), actual.bytesUsed); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java index 5915511b7..121a8fd53 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java @@ -67,7 +67,9 @@ package org.opencadc.inventory; +import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; +import java.io.IOException; import org.apache.log4j.Logger; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; @@ -84,7 +86,7 @@ public class ArtifactSyncWorker implements Runnable { private static final Logger log = Logger.getLogger(ArtifactSyncWorker.class); private final HarvestState harvestState; - private final NodeDAO nodeDAO = null; + private final NodeDAO nodeDAO; private final ArtifactDAO artifactDAO; private final HarvestStateDAO harvestStateDAO; private final Namespace storageNamespace; @@ -92,7 +94,7 @@ public class ArtifactSyncWorker implements Runnable { public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, Namespace namespace) { this.harvestState = harvestState; this.harvestStateDAO = harvestStateDAO; - //this.nodeDAO = new NodeDAO(harvestStateDAO); + this.nodeDAO = new NodeDAO(harvestStateDAO); this.artifactDAO = artifactDAO; this.storageNamespace = namespace; } @@ -101,29 +103,33 @@ public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS public void run() { log.debug("Start harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); - //ResourceIterator iter = artifactDAO.iterator(storageNamespace, null, harvestState.curLastModified, true); - ResourceIterator iter = null; - while (iter.hasNext()) { - Artifact artifact = iter.next(); - DataNode node = null; //nodeDAO.getDataNode(artifact.getURI()); - if (!node.bytesUsed.equals(artifact.getContentLength())) { - node.bytesUsed = artifact.getContentLength(); - nodeDAO.getTransactionManager().startTransaction(); - try { - nodeDAO.put(node); - harvestState.curLastModified = artifact.getLastModified(); - harvestState.curID = node.getID(); - harvestStateDAO.put(harvestState, true); - nodeDAO.getTransactionManager().commitTransaction(); - log.debug("Updated size of data node " + node.getName()); - } catch (Exception ex) { - log.debug("Failed to update data node size for " + node.getName(), ex); - nodeDAO.getTransactionManager().rollbackTransaction(); - throw ex; + try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, null, + harvestState.curLastModified, true)) { + while (iter.hasNext()) { + Artifact artifact = iter.next(); + DataNode node = nodeDAO.getDataNode(artifact.getURI()); + if ((node != null) && !artifact.getContentLength().equals(node.bytesUsed)) { + node.bytesUsed = artifact.getContentLength(); + TransactionManager tm = nodeDAO.getTransactionManager(); + tm.startTransaction(); + try { + nodeDAO.put(node); + harvestState.curLastModified = artifact.getLastModified(); + harvestState.curID = node.getID(); + harvestStateDAO.put(harvestState, true); + tm.commitTransaction(); + log.debug("Updated size of data node " + node.getName()); + } catch (Exception ex) { + log.debug("Failed to update data node size for " + node.getName(), ex); + tm.rollbackTransaction(); + throw ex; + } } } + } catch (IOException ex) { + //log.error("Error closing iterator", ex); + throw new RuntimeException("error while closing ResourceIterator", ex); } - log.debug("End harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); } diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java index bee209489..8088c66b7 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java @@ -72,6 +72,7 @@ import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.db.AbstractDAO; +import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.vospace.ContainerNode; import org.opencadc.vospace.DataNode; @@ -94,6 +95,10 @@ public NodeDAO(boolean origin) { super(origin); } + public NodeDAO(HarvestStateDAO harvestStateDAO) { + super(harvestStateDAO); + } + @Override public void put(Node val) { super.put(val); From 2387d3aad1fda32890c32d3c9dcc48c6ccdf4218 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 11 Mar 2024 19:10:14 -0700 Subject: [PATCH 152/186] vault: ArtifactSync init intTest enable --- .../opencadc/inventory/db/HarvestState.java | 3 + .../java/org/opencadc/vault/Constants.java | 92 ++++++++++ .../java/org/opencadc/vault/FilesTest.java | 6 +- .../java/org/opencadc/vault/NodesTest.java | 7 +- .../java/org/opencadc/vault/TransferTest.java | 5 +- .../opencadc/vault/NodePersistenceImpl.java | 2 +- .../org/opencadc/vault/VaultInitAction.java | 35 +++- .../org/opencadc/vault/files/GetAction.java | 10 ++ .../org/opencadc/vault/files/HeadAction.java | 2 +- .../opencadc/vault/metadata/ArtifactSync.java | 157 ++++++++++++++++++ vault/src/main/webapp/META-INF/context.xml | 13 ++ 11 files changed, 317 insertions(+), 15 deletions(-) create mode 100644 vault/src/intTest/java/org/opencadc/vault/Constants.java create mode 100644 vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java index 04a0b7248..4d012cc24 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java @@ -137,6 +137,9 @@ public void setResourceID(URI resourceID) { @Override public String toString() { + if (instanceID != null) { + return HarvestState.class.getSimpleName() + "[" + instanceID + "," + name + "," + resourceID + "]"; + } return HarvestState.class.getSimpleName() + "[" + name + "," + resourceID + "]"; } } diff --git a/vault/src/intTest/java/org/opencadc/vault/Constants.java b/vault/src/intTest/java/org/opencadc/vault/Constants.java new file mode 100644 index 000000000..c07bdad08 --- /dev/null +++ b/vault/src/intTest/java/org/opencadc/vault/Constants.java @@ -0,0 +1,92 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault; + +import ca.nrc.cadc.util.FileUtil; +import java.io.File; +import java.net.URI; +import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; + +/** + * + * @author pdowler + */ +public class Constants { + private static final Logger log = Logger.getLogger(Constants.class); + + static URI RESOURCE_ID = URI.create("ivo://opencadc.org/vault"); + + static File ADMIN_CERT = FileUtil.getFileFromResource("vault-test.pem", Constants.class); + static File ALT_CERT = FileUtil.getFileFromResource("vault-auth-test.pem", Constants.class); + + static GroupURI ALT_GROUP = new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?opencadc-vospace-test")); + + private Constants() { + } +} diff --git a/vault/src/intTest/java/org/opencadc/vault/FilesTest.java b/vault/src/intTest/java/org/opencadc/vault/FilesTest.java index c0a64bcd5..5002c5fe6 100644 --- a/vault/src/intTest/java/org/opencadc/vault/FilesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/FilesTest.java @@ -87,9 +87,9 @@ public class FilesTest extends org.opencadc.conformance.vos.FilesTest { Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); } - private static File ADMIN_CERT = FileUtil.getFileFromResource("vault-test.pem", FilesTest.class); - public FilesTest() { - super(URI.create("ivo://opencadc.org/vault"), ADMIN_CERT); + super(Constants.RESOURCE_ID, Constants.ADMIN_CERT); + + enableTestDataNodePermission(Constants.ALT_CERT); } } diff --git a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java index 869ca0025..3d8ec5cab 100644 --- a/vault/src/intTest/java/org/opencadc/vault/NodesTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/NodesTest.java @@ -88,13 +88,10 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest { Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); } - private static File ADMIN_CERT = FileUtil.getFileFromResource("vault-test.pem", NodesTest.class); - public NodesTest() { - super(URI.create("ivo://opencadc.org/vault"), ADMIN_CERT); + super(Constants.RESOURCE_ID, Constants.ADMIN_CERT); - File altCert = FileUtil.getFileFromResource("vault-auth-test.pem", NodesTest.class); - enablePermissionTests(new GroupURI(URI.create("ivo://cadc.nrc.ca/gms?opencadc-vospace-test")), altCert); + enablePermissionTests(Constants.ALT_GROUP, Constants.ALT_CERT); // vault does not check the actual groups in the permission props tests, hence they can be made up. enablePermissionPropsTest(new GroupURI(URI.create("ivo://myauth/gms?gr1")), new GroupURI(URI.create("ivo://myauth/gms?gr2"))); diff --git a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java index f1392e1ee..3122a3af8 100644 --- a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java @@ -104,10 +104,9 @@ public class TransferTest extends org.opencadc.conformance.vos.TransferTest { static String DATABASE = "cadctest"; static String SCHEMA = "inventory"; - private static File ADMIN_CERT = FileUtil.getFileFromResource("vault-test.pem", NodesTest.class); - public TransferTest() { - super(URI.create("ivo://opencadc.org/vault"), ADMIN_CERT); + super(Constants.RESOURCE_ID, Constants.ADMIN_CERT); + enableTestDataNodePermission(Constants.ALT_GROUP, Constants.ALT_CERT); } @Before diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index 611ebd8a4..ac4b906b1 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -304,7 +304,7 @@ public boolean isAllocation(ContainerNode cn) { } ContainerNode p = cn.parent; for (ContainerNode ap : allocationParents) { - if (absoluteEquals(p.parent, ap)) { + if (absoluteEquals(p, ap)) { return true; } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 6b5faf21b..6ccd72e38 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -88,11 +88,13 @@ import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.PreauthKeyPair; +import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; +import org.opencadc.vault.metadata.ArtifactSync; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; import org.springframework.dao.DataIntegrityViolationException; @@ -133,6 +135,7 @@ public class VaultInitAction extends InitAction { private String jndiPreauthKeys; private String jndiSiteAvailabilities; private Thread availabilityCheck; + private Thread artifactSync; public VaultInitAction() { super(); @@ -146,6 +149,7 @@ public void doInit() { initNodePersistence(); initKeyPair(); initAvailabilityCheck(); + initBackgroundWorkers(); } @Override @@ -165,6 +169,7 @@ public void doShutdown() { } terminateAvailabilityCheck(); + terminateBackgroundWorkers(); } /** @@ -412,7 +417,7 @@ private void initKeyPair() { } } - void initAvailabilityCheck() { + private void initAvailabilityCheck() { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(); storageSiteDAO.setConfig(getDaoConfig(props)); @@ -423,7 +428,7 @@ void initAvailabilityCheck() { this.availabilityCheck.start(); } - private final void terminateAvailabilityCheck() { + private void terminateAvailabilityCheck() { if (this.availabilityCheck != null) { try { log.info("terminating AvailabilityCheck Thread..."); @@ -436,6 +441,8 @@ private final void terminateAvailabilityCheck() { this.availabilityCheck = null; } } + + // ugh: bind() is inside StorageSiteAvailabilityCheck but unbind() is here try { InitialContext initialContext = new InitialContext(); initialContext.unbind(this.jndiSiteAvailabilities); @@ -444,4 +451,28 @@ private final void terminateAvailabilityCheck() { } } + private void initBackgroundWorkers() { + HarvestStateDAO hsDAO = new HarvestStateDAO(); + hsDAO.setConfig(getDaoConfig(props)); + + terminateBackgroundWorkers(); + this.artifactSync = new Thread(new ArtifactSync(hsDAO)); + artifactSync.setDaemon(true); + artifactSync.start(); + } + + private void terminateBackgroundWorkers() { + if (this.artifactSync != null) { + try { + log.info("terminating ArtifactSync Thread..."); + this.artifactSync.interrupt(); + this.artifactSync.join(); + log.info("terminating ArtifactSync Thread... [OK]"); + } catch (Throwable t) { + log.info("failed to terminate ArtifactSync thread", t); + } finally { + this.artifactSync = null; + } + } + } } diff --git a/vault/src/main/java/org/opencadc/vault/files/GetAction.java b/vault/src/main/java/org/opencadc/vault/files/GetAction.java index 6337dbccb..d3967e5c2 100644 --- a/vault/src/main/java/org/opencadc/vault/files/GetAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/GetAction.java @@ -67,14 +67,18 @@ package org.opencadc.vault.files; +import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.net.TransientException; import java.net.HttpURLConnection; +import java.net.URI; import java.util.List; +import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.opencadc.vospace.DataNode; import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; +import org.opencadc.vospace.server.NodeFault; import org.opencadc.vospace.server.Utils; import org.opencadc.vospace.server.transfers.TransferGenerator; import org.opencadc.vospace.transfer.Direction; @@ -96,6 +100,12 @@ public GetAction() { public void doAction() throws Exception { DataNode node = resolveAndSetMetadata(); + Subject caller = AuthenticationUtil.getCurrentSubject(); + if (!voSpaceAuthorizer.hasSingleNodeReadPermission(node, caller)) { + // TODO: should output requested vos URI here + throw NodeFault.PermissionDenied.getStatus(syncInput.getPath()); + } + if (node.bytesUsed == null || node.bytesUsed == 0L) { // empty file syncOutput.setCode(HttpURLConnection.HTTP_NO_CONTENT); diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java index 8e4505c3b..69b68f00a 100644 --- a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -162,7 +162,7 @@ DataNode resolveAndSetMetadata() throws Exception { } } syncOutput.setCode(200); - return (DataNode)node; + return dn; } } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java new file mode 100644 index 000000000..ffd1879e6 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java @@ -0,0 +1,157 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault.metadata; + +import java.net.URI; +import java.util.Date; +import java.util.UUID; +import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.db.HarvestState; +import org.opencadc.inventory.db.HarvestStateDAO; + +/** + * Main artifact-sync agent that enables incremental sync of Artifact + * metadata to Node. + * + * @author pdowler + */ +public class ArtifactSync implements Runnable { + private static final Logger log = Logger.getLogger(ArtifactSync.class); + + private static final long SHORT_SLEEP = 12000L; + private static final long LONG_SLEEP = 2 * SHORT_SLEEP; + private static final long EVICT_AGE = 3 * LONG_SLEEP; + + private final UUID instanceID = UUID.randomUUID(); + private final HarvestStateDAO dao; + private String name = Artifact.class.getSimpleName(); + private URI resourceID = URI.create("jdbc/inventory"); + + public ArtifactSync(HarvestStateDAO dao) { + this.dao = dao; + + // fenwick setup for production workload: + //dao.setUpdateBufferCount(99); // buffer 99 updates, do every 100 + //dao.setMaintCount(999); // buffer 999 so every 1000 real updates aka every 1e5 events + + // here, we need timestamp updates to retain leader status, so + // dao.setMaintCount(9999); // every 1e4 + } + + @Override + public void run() { + try { + Thread.sleep(SHORT_SLEEP); + + while (true) { + boolean leader = false; + log.debug("check leader " + instanceID); + HarvestState state = dao.get(name, resourceID); + log.debug("check leader " + instanceID + " found: " + state); + if (state.instanceID == null) { + state.instanceID = instanceID; + dao.put(state); + state = dao.get(state.getID()); + log.debug("created: " + state); + } + if (instanceID.equals(state.instanceID)) { + log.debug("still the leader..."); + dao.put(state, true); + leader = true; + } else { + // see if we should perform a coup... + Date now = new Date(); + long age = now.getTime() - state.getLastModified().getTime(); + if (age > EVICT_AGE) { + + state.instanceID = instanceID; + dao.put(state); + state = dao.get(state.getID()); + leader = true; + log.debug("EVICTED " + state.instanceID + " because age " + age + " > " + EVICT_AGE); + } + } + + if (leader) { + log.debug("leader " + state.instanceID + " starting worker..."); + // TODO + dao.flushBufferedState(); + Thread.sleep(SHORT_SLEEP / 2L); // for testing + log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); + Thread.sleep(SHORT_SLEEP); + } else { + log.debug("not leader: sleep=" + LONG_SLEEP); + Thread.sleep(LONG_SLEEP); + } + } + } catch (InterruptedException ex) { + log.debug("interrupted - assuming shutdown", ex); + } + } +} diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index ebd23f61c..94ec7b42f 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -27,6 +27,19 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> + + + Date: Tue, 12 Mar 2024 12:24:22 -0700 Subject: [PATCH 153/186] always close() connections in iterator.close() minor bug fix for iterator query gen --- .../opencadc/inventory/db/SQLGenerator.java | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java index d166f869f..0652a9184 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java @@ -736,13 +736,14 @@ public void setSiteID(UUID siteID) { @Override public ResourceIterator query(DataSource ds) { - StringBuilder select = getSelectFromSQL(Artifact.class, false); - StringBuilder sb = new StringBuilder(" WHERE"); - int whereLen = sb.length(); - + StringBuilder sb = new StringBuilder(); + boolean where = false; if (storageLocationRequired != null && storageLocationRequired) { // ArtifactDAO.storedIterator + sb.append(" WHERE"); + where = true; if (storageBucket != null) { + sb.append(" storageLocation_storageBucket LIKE ? AND"); } sb.append(" storageLocation_storageID IS NOT NULL"); @@ -755,33 +756,51 @@ public ResourceIterator query(DataSource ds) { sb.append(" ORDER BY storageLocation_storageBucket, storageLocation_storageID"); } } else if (storageLocationRequired != null && !storageLocationRequired) { + // ArtifactDAO.unstoredIterator + sb.append(" WHERE"); + where = true; sb.append(" storageLocation_storageID IS NULL"); } + // optional params: + // uriBucket + // siteID + // namespace + // minLastModified if (uriBucket != null) { - // ArtifactDAO.iterator(null, ...) - if (sb.length() > whereLen) { + if (where) { sb.append(" AND"); + } else { + sb.append(" WHERE"); + where = true; } sb.append(" uriBucket LIKE ?"); } if (siteID != null) { // ArtifactDAO.iterator(UUID, ...) - if (sb.length() > whereLen) { + if (where) { sb.append(" AND"); + } else { + sb.append(" WHERE"); + where = true; } sb.append(" siteLocations @> ARRAY[?]"); } if (namespace != null) { - // ArtifactDAO.iterator(Namespace, ...) - if (sb.length() > whereLen) { + if (where) { sb.append(" AND"); + } else { + sb.append(" WHERE"); + where = true; } sb.append(" uri LIKE ?"); } if (minLastModified != null) { - if (sb.length() > whereLen) { + if (where) { sb.append(" AND"); + } else { + sb.append(" WHERE"); + where = true; } sb.append(" lastModified >= ?"); } @@ -793,16 +812,11 @@ public ResourceIterator query(DataSource ds) { } } - if (sb.length() > whereLen) { - select.append(sb.toString()); - } + StringBuilder select = getSelectFromSQL(Artifact.class, false); + select.append(sb.toString()); String sql = select.toString(); - log.warn("sql: " + sql); + log.debug("sql: " + sql); - // params: - // storageBucket OR uriBucket - // siteID - // namespace try { Connection con = ds.getConnection(); log.debug("ArtifactIterator: setAutoCommit(false)"); @@ -1774,7 +1788,7 @@ private class ArtifactResultSetIterator implements ResourceIterator { private final Connection con; private final ResultSet rs; boolean hasRow; - boolean closeWhenDone = false; // not a pooled connection + boolean closeWhenDone = true; // return to pool | assume close suppressed for static connections ArtifactResultSetIterator(Connection con, ResultSet rs) throws SQLException { this.con = con; From 75484715fd8737d37bb6a2c8b992c8929ffe9034 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 12 Mar 2024 13:59:52 -0700 Subject: [PATCH 154/186] vault: fix availability check dao config for singlePool=false --- vault/src/main/java/org/opencadc/vault/VaultInitAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 6ccd72e38..e4e51b418 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -419,7 +419,7 @@ private void initKeyPair() { private void initAvailabilityCheck() { StorageSiteDAO storageSiteDAO = new StorageSiteDAO(); - storageSiteDAO.setConfig(getDaoConfig(props)); + storageSiteDAO.setConfig(getInvConfig(props)); this.jndiSiteAvailabilities = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); terminateAvailabilityCheck(); From 353d8f4008d61481a2f15c4b2ec61f64453ddb86 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Mar 2024 15:09:56 -0700 Subject: [PATCH 155/186] vault-quota: update design docs --- vault-quota/Design.md | 11 ++++++ vault-quota/NodeSize.md | 85 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 vault-quota/NodeSize.md diff --git a/vault-quota/Design.md b/vault-quota/Design.md index 08f81f0b9..769f85039 100644 --- a/vault-quota/Design.md +++ b/vault-quota/Design.md @@ -8,6 +8,17 @@ completed. In the case of a `vault` service co-located with a global SI, the new Artifact is visible in the database once it is synced from the site of the PUT to the global database by `fenwick` (or worst case: `ratik`). +## NOTE +This design was for supporting propagation of space used up the tree so that +allocation space used was dynamically updated as content was modified. While the +algorithm for doing that is nominally viable, the algorithm to validate and repair +incorrect container node sizes in a live system is excessively complex and probably +impossible to implement in practice (deadlocks, excessive database load and query +processing, etc). + +**This design will not be completed and implemented** and is retained here for future +reference. + ## TODO The design below only takes into account incremental propagation of space used by stored files. It is not complete/verified until we also come up with a validation diff --git a/vault-quota/NodeSize.md b/vault-quota/NodeSize.md new file mode 100644 index 000000000..fd48db421 --- /dev/null +++ b/vault-quota/NodeSize.md @@ -0,0 +1,85 @@ +# vault quota design/algorithms + +The definitive source of content-length (file size) of a DataNode comes from the +`inventory.Artifact` table and is not known until a PUT to storage is completed. +In the case of a `vault` service co-located with a single storage site (`minoc`), +the new Artifact is visible in the database as soon as the PUT to `minoc` is +completed. In the case of a `vault` service co-located with a global SI, the new +Artifact is visible in the database once it is synced from the site of the PUT to +the global database by `fenwick` (or worst case: `ratik`). + +## incremental DataNode size algorithm +DataNode(s) require the `bytesUsed` be set so that sizes can be output from listing +container nodes without a join or query to the artifact table. + +This is an event watcher that gets Artifact events (after a PUT) and intiates the +propagation of sizes (space used). +``` +track progress using HarvestState (source, name: TBD) +incremental query for new artifacts in lastModified order +for each new Artifact: + query for DataNode (storageID = artifact.uri) + if Artifact.contentLength != Node.size: + start txn + lock datanode + recheck size diff + set dataNode.size + update HarvestState + commit txn +``` + +## validate DataNode vs Artifact discrepancies +These can be validated in parallel by multiple threads, subdivide work by bucket if we add +DataNode.storageBucket (== Artifact.uriBucket). + +``` +discrepancy: Artifact exists but DataNode does not +explanation: DataNode created, transfer negotiated, DataNode removed, transfer executed +evidence: DeletedNodeEvent exists +action: remove artifact, create DeletedArtifactEvent + +discrepancy: Artifact exists but DataNode does not +explanation: DataNode created, Artifact put, DataNode deleted, Artifact delete failed +evidence: only possible with singlePool==false +action: remove artifact, create DeletedArtifactEvent + +discrepancy: DataNode exists but Artifact does not +explanation: DataNode created, Artifact never (successfully) put (normal) +evidence: DataNode.nodeSize == 0 or null +action: none + +discrepancy: DataNode exists but Artifact does not +explanation: deleted or lost Artifact +evidence: DataNode.nodeSize != 0 (deleted vs lost: DeletedArtifactEvent exists) +action: lock nodes, fix dataNode and propagate delta to parent + +discrepancy: DataNode.nodeSize != Artifact.contentLength +explanation: artifact written (if DataNode.size > 0: replaced) +action: lock nodes, fix DataNode and propagate delta to parent +``` +Required lock order: child-parent or parent-child OK. + +The most generic implementation is a merge join of two iterators (see ratik, tantar): +``` +Iterator aiter = artifactDAO.iterator(vaultNamespace, bucket); // uriBucket,uri order +Iterator niter = nodeDAO.iterator(bucket); // storageBucket,storageID order +``` + +## database changes required +note: all field and column names TBD +* add `transient Long bytesUsed` to ContainerNode and DataNode +* add `bytesUsed` to the `vospace.Node` table +* add `storageBucket` to DataNode?? TBD +* add `storageBucket` to `vospace.Node` table + +## cadc-inventory-db API required immediately +* incremental sync query/iterator: ArtifactDAO.iterator(Namespace ns, String uriBucketPrefix, Date minLastModified, boolean ordered) + order by lastModified if set +* lookup DataNode by storageID: NodeDAO.getDataNode(URI storageID) +* indices to support new queries + +## cadc-inventory-db API required later (tentative) +* validate-by-bucket: use ArtifactDAO.iterator(String uriBucketPrefix, boolean ordered, Namespace ns) +* validate-by-bucket: NodeDAO.dataNodeIterator(String storageBucketPrefix, boolean ordered) +* indices to support new queries + From 819d68f949b5ecb8025d0456cb8c62efe291d91c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Mar 2024 15:55:24 -0700 Subject: [PATCH 156/186] rework remove junk from TransferTest --- vault/README.md | 2 +- .../java/org/opencadc/vault/TransferTest.java | 46 ------------------- .../opencadc/vault/NodePersistenceImpl.java | 2 +- .../vault/VaultTransferGenerator.java | 2 - 4 files changed, 2 insertions(+), 50 deletions(-) diff --git a/vault/README.md b/vault/README.md index 4e3c5fbc9..108357da6 100644 --- a/vault/README.md +++ b/vault/README.md @@ -106,7 +106,7 @@ _all known_ sites. It only makes sense to enable this when `vault` is running in genuine not-found cases: transfer negotiation to GET the file that was never PUT. The _allocationParent_ is a path to a container node (directory) which contains space allocations. An allocation -is owned by a user (uisually different from the _rootOwner_ admin user) who is responsible for the allocation +is owned by a user (usually different from the _rootOwner_ admin user) who is responsible for the allocation and all conntent therein. The owner of an allocation is granted additional permissions within their allocation (they can read/write/delete anything) so the owner cannot be blocked from access to any content within their allocation. This probably only matters for multi-user projects. Multiple _allocationParent_(s) may diff --git a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java index 3122a3af8..14e61f91e 100644 --- a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java @@ -108,50 +108,4 @@ public TransferTest() { super(Constants.RESOURCE_ID, Constants.ADMIN_CERT); enableTestDataNodePermission(Constants.ALT_GROUP, Constants.ALT_CERT); } - - @Before - public void checkGlobal() throws Exception { - // make sure inventory.StorageSite has a readable/writable ivo://opencadc.org/minoc - - try { - DBConfig dbrc = new DBConfig(); - ConnectionConfig cc = dbrc.getConnectionConfig(SERVER, DATABASE); - DBUtil.createJNDIDataSource("jdbc/inventory", cc); - - Map config = new TreeMap(); - config.put(SQLGenerator.class.getName(), SQLGenerator.class); - config.put("jndiDataSourceName", "jdbc/inventory"); - config.put("invSchema", SCHEMA); - config.put("genSchema", SCHEMA); - - StorageSiteDAO dao = new StorageSiteDAO(false); - dao.setConfig(config); - Set sites = dao.list(); - if (sites.isEmpty()) { - StorageSite ss = new StorageSite(URI.create("ivo://opencadc.org/minoc"), "vault-test-minoc", true, true); - dao.put(ss); - log.info("created record in local db: " + ss); - } - sites = dao.list(); - for (StorageSite ss : sites) { - log.info("storage sites in local db: " + ss); - } - } catch (Exception ex) { - log.error("setup failed", ex); - throw ex; - } - } - - @Override - public void asyncMoveTest() { - super.asyncMoveTest(); - } - - @Ignore - @Override - public void syncPushPullTest() { - super.syncPushPullTest(); - } - - } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index ac4b906b1..e3e98ceec 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -304,7 +304,7 @@ public boolean isAllocation(ContainerNode cn) { } ContainerNode p = cn.parent; for (ContainerNode ap : allocationParents) { - if (absoluteEquals(p, ap)) { + if (p.getID().equals(ap.getID())) { return true; } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 0e6a9da79..51fec42a9 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -70,7 +70,6 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.IdentityManager; import ca.nrc.cadc.net.ResourceNotFoundException; -import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.uws.Parameter; import ca.nrc.cadc.vosi.Availability; import java.io.IOException; @@ -193,7 +192,6 @@ private List handleDataNode(DataNode node, String filename, Transfer t log.debug("requested protocol: " + p); if (!protoURIs.contains(p.getUri())) { Protocol anonProto = new Protocol(p.getUri()); - //anonProto.setSecurityMethod(Standards.SECURITY_METHOD_ANON); artifactTrans.getProtocols().add(anonProto); protoURIs.add(p.getUri()); log.debug("Added anon protocol for " + p.getUri()); From 22718c84d6ff4ec9bc857fa6a6eb2be0d57692b1 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Mar 2024 15:56:09 -0700 Subject: [PATCH 157/186] rework: remove unused imports --- .../java/org/opencadc/vault/TransferTest.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java index 14e61f91e..0fe0a6b8e 100644 --- a/vault/src/intTest/java/org/opencadc/vault/TransferTest.java +++ b/vault/src/intTest/java/org/opencadc/vault/TransferTest.java @@ -67,23 +67,9 @@ package org.opencadc.vault; -import ca.nrc.cadc.db.ConnectionConfig; -import ca.nrc.cadc.db.DBConfig; -import ca.nrc.cadc.db.DBUtil; -import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.Log4jInit; -import java.io.File; -import java.net.URI; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.junit.Before; -import org.junit.Ignore; -import org.opencadc.inventory.StorageSite; -import org.opencadc.inventory.db.SQLGenerator; -import org.opencadc.inventory.db.StorageSiteDAO; /** * From 42731460846b55f2e9bea6135f2ec8f5989911c8 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Mar 2024 16:20:42 -0700 Subject: [PATCH 158/186] vault: pull doc improvements from PR-558 --- vault/README.md | 6 +++--- vault/src/intTest/README.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 vault/src/intTest/README.md diff --git a/vault/README.md b/vault/README.md index 108357da6..153232129 100644 --- a/vault/README.md +++ b/vault/README.md @@ -1,8 +1,8 @@ # Storage Inventory VOSpace-2.1 service (vault) -The `vault` servcie is an implementation of the IVOA VOSpace -specification designed to co-exist with other storage-inventory components. It provides a heirarchical data -organization laye on top of the storage management of storage-inventory. +The `vault` service is an implementation of the IVOA VOSpace +specification designed to co-exist with other storage-inventory components. It provides a hierarchical data +organization layer on top of the storage management of storage-inventory. The simplest configuration would be to deploy `vault` with `minoc` with a single metadata database and single back end storage system. Details: TBD. diff --git a/vault/src/intTest/README.md b/vault/src/intTest/README.md new file mode 100644 index 000000000..b03af25ac --- /dev/null +++ b/vault/src/intTest/README.md @@ -0,0 +1,30 @@ +# Storage Inventory VOSpace-2.1 service (vault) + +The simplest configuration that deploys `vault` with `minoc` with a single metadata database and single +back end storage system is sufficient to run the `vault` integration tests. The tests also relly on the +presence of the root owner X509 certificate in `build/classes/java/intTest/vault-test.pem`. +Some tests (primarily permission tests) will be skipped unless the certificate of a second user is present +in `build/classes/java/intTest/vault-auth-test.pem`. This user has to be member of the `ivo://cadc.nrc.ca/gms?opencadc-vospace-test` +group. The names of these certificates and groups are hardcoded in the `vault` int tests classes. + +The int tests suite also relies on a specific configuration of the `vault` service: +### vault.properties +``` +# service identity +org.opencadc.vault.resourceID = ivo://opencadc.org/vault + +# (optional) identify which container nodes are allocations +org.opencadc.vault.allocationParent = / + +# consistency settings +org.opencadc.vault.consistency.preventNotFound=true + +# vault database settings +org.opencadc.vault.inventory.schema = inventory +org.opencadc.vault.vospace.schema = vault +org.opencadc.vault.singlePool = true + +# root container nodes +org.opencadc.vault.root.owner = {owner of root node} + +``` From 4dde6786649a6b7ca125d1e12f911cbbd083aa08 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 14 Mar 2024 10:21:47 -0700 Subject: [PATCH 159/186] Initial version --- .../org/opencadc/ArtifactSyncWorkerTest.java | 6 +++--- .../opencadc/inventory/ArtifactSyncWorker.java | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java index f21463291..608430db8 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java @@ -227,14 +227,14 @@ public void testSyncArtifact() throws Exception { log.info("found: " + actual.getID() + " aka " + actual); Assert.assertEquals(expected.getContentLength(), actual.bytesUsed); - // update - Thread.sleep(20L); + // update the artifact only + artifactDAO.delete(actualArtifact.getID()); expected = new Artifact(expected.getURI(), expected.getMetaChecksum(), new Date(), 333L); artifactDAO.put(expected); actual = (DataNode)nodeDAO.get(orig.getID()); Assert.assertNotEquals(expected.getContentLength(), actual.bytesUsed); - // do the update + // run the update asWorker.run(); actual = (DataNode)nodeDAO.get(orig.getID()); Assert.assertEquals(expected.getContentLength(), actual.bytesUsed); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java index 121a8fd53..fd34d68f4 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java @@ -78,7 +78,7 @@ import org.opencadc.vospace.db.NodeDAO; /** - * This class performs the work of synchronizing the size of Data Node between backend storage and Node Persistence + * This class performs the work of synchronizing the size of Data Nodes between backend storage and Node Persistence * * @author adriand */ @@ -91,7 +91,8 @@ public class ArtifactSyncWorker implements Runnable { private final HarvestStateDAO harvestStateDAO; private final Namespace storageNamespace; - public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, Namespace namespace) { + public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, + Namespace namespace) { this.harvestState = harvestState; this.harvestStateDAO = harvestStateDAO; this.nodeDAO = new NodeDAO(harvestStateDAO); @@ -103,6 +104,7 @@ public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS public void run() { log.debug("Start harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + TransactionManager tm = nodeDAO.getTransactionManager(); try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, null, harvestState.curLastModified, true)) { while (iter.hasNext()) { @@ -110,7 +112,6 @@ public void run() { DataNode node = nodeDAO.getDataNode(artifact.getURI()); if ((node != null) && !artifact.getContentLength().equals(node.bytesUsed)) { node.bytesUsed = artifact.getContentLength(); - TransactionManager tm = nodeDAO.getTransactionManager(); tm.startTransaction(); try { nodeDAO.put(node); @@ -123,14 +124,20 @@ public void run() { log.debug("Failed to update data node size for " + node.getName(), ex); tm.rollbackTransaction(); throw ex; + } finally { + if (tm.isOpen()) { + log.error("BUG: transaction open in finally. Rolling back..."); + tm.rollbackTransaction(); + log.error("Rollback: OK"); + throw new RuntimeException("BUG: transaction open in finally"); + } } } } } catch (IOException ex) { - //log.error("Error closing iterator", ex); + log.error("Error closing iterator", ex); throw new RuntimeException("error while closing ResourceIterator", ex); } log.debug("End harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); - } } From 43f98bd378cf467356a088a1fe3b13aafa8c5817 Mon Sep 17 00:00:00 2001 From: Adrian Damian Date: Thu, 14 Mar 2024 15:45:27 -0700 Subject: [PATCH 160/186] Rework after code review --- .../org/opencadc/inventory/ArtifactSyncWorker.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java index fd34d68f4..7452eca62 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java @@ -110,14 +110,15 @@ public void run() { while (iter.hasNext()) { Artifact artifact = iter.next(); DataNode node = nodeDAO.getDataNode(artifact.getURI()); - if ((node != null) && !artifact.getContentLength().equals(node.bytesUsed)) { - node.bytesUsed = artifact.getContentLength(); + if (node != null && !artifact.getContentLength().equals(node.bytesUsed)) { tm.startTransaction(); try { + node = (DataNode)nodeDAO.lock(node); + if (node == null) { + continue; // node gone - race condition + } + node.bytesUsed = artifact.getContentLength(); nodeDAO.put(node); - harvestState.curLastModified = artifact.getLastModified(); - harvestState.curID = node.getID(); - harvestStateDAO.put(harvestState, true); tm.commitTransaction(); log.debug("Updated size of data node " + node.getName()); } catch (Exception ex) { @@ -133,6 +134,9 @@ public void run() { } } } + harvestState.curLastModified = artifact.getLastModified(); + harvestState.curID = node.getID(); + harvestStateDAO.put(harvestState); } } catch (IOException ex) { log.error("Error closing iterator", ex); From 6a62b1c39def0daec64de4da233e83a13f07a753 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Mar 2024 12:24:37 -0700 Subject: [PATCH 161/186] moved ArtifactSyncWorker to vospace pkg --- .../org/opencadc/{ => vospace/db}/ArtifactSyncWorkerTest.java | 4 ++-- .../{inventory => vospace/db}/ArtifactSyncWorker.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) rename cadc-inventory-db/src/intTest/java/org/opencadc/{ => vospace/db}/ArtifactSyncWorkerTest.java (99%) rename cadc-inventory-db/src/main/java/org/opencadc/{inventory => vospace/db}/ArtifactSyncWorker.java (98%) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java similarity index 99% rename from cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java rename to cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java index 608430db8..fa18912c6 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/ArtifactSyncWorkerTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java @@ -65,7 +65,7 @@ ************************************************************************ */ -package org.opencadc; +package org.opencadc.vospace.db; import ca.nrc.cadc.db.ConnectionConfig; import ca.nrc.cadc.db.DBConfig; @@ -84,7 +84,7 @@ import org.junit.Before; import org.junit.Test; import org.opencadc.inventory.Artifact; -import org.opencadc.inventory.ArtifactSyncWorker; +import org.opencadc.vospace.db.ArtifactSyncWorker; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java similarity index 98% rename from cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java rename to cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java index 7452eca62..29e80384f 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java @@ -65,12 +65,14 @@ ************************************************************************ */ -package org.opencadc.inventory; +package org.opencadc.vospace.db; import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import java.io.IOException; import org.apache.log4j.Logger; +import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; From 5f374cc33bfb9773027aea5f2f010efb7822ec82 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Mar 2024 12:25:02 -0700 Subject: [PATCH 162/186] vospace.Node.sql: explicit text array to match code usage --- cadc-inventory-db/src/main/resources/vospace.Node.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cadc-inventory-db/src/main/resources/vospace.Node.sql b/cadc-inventory-db/src/main/resources/vospace.Node.sql index 3e23b5b2b..51e5d6467 100644 --- a/cadc-inventory-db/src/main/resources/vospace.Node.sql +++ b/cadc-inventory-db/src/main/resources/vospace.Node.sql @@ -8,10 +8,10 @@ create table .Node ( ownerID varchar(256) not null, isPublic boolean, isLocked boolean, - readOnlyGroups text, - readWriteGroups text, + readOnlyGroups text[], + readWriteGroups text[], - -- store all props in a 2D array + -- store misc props in a 2D array properties text[][], -- ContainerNode From 6dd4bb16435fa4a7f9c0aa4a4f6244076c496263 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Mar 2024 14:57:15 -0700 Subject: [PATCH 163/186] vault: integrate ArtifactSyncWorker --- .../vospace/db/ArtifactSyncWorker.java | 45 +++++-- .../org/opencadc/vault/VaultInitAction.java | 26 ++-- .../opencadc/vault/metadata/ArtifactSync.java | 121 ++++++++++++------ vault/src/main/webapp/META-INF/context.xml | 2 +- 4 files changed, 136 insertions(+), 58 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java index 29e80384f..33bf20948 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java @@ -67,9 +67,11 @@ package org.opencadc.vospace.db; +import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; import java.io.IOException; +import java.text.DateFormat; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.Namespace; @@ -77,10 +79,10 @@ import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.db.NodeDAO; /** - * This class performs the work of synchronizing the size of Data Nodes between backend storage and Node Persistence + * This class performs the work of synchronizing the size of Data Nodes from + * inventory (Artifact) to vopsace (Node). * * @author adriand */ @@ -93,8 +95,16 @@ public class ArtifactSyncWorker implements Runnable { private final HarvestStateDAO harvestStateDAO; private final Namespace storageNamespace; - public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, - Namespace namespace) { + /** + * Worker constructor. + * + * @param harvestStateDAO DAO class to persist progress in the vospace database + * @param harvestState current HarvestState instance + * @param artifactDAO DAO class to query for artifacts + * @param namespace artifact namespace + */ + public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, + ArtifactDAO artifactDAO, Namespace namespace) { this.harvestState = harvestState; this.harvestStateDAO = harvestStateDAO; this.nodeDAO = new NodeDAO(harvestStateDAO); @@ -104,11 +114,17 @@ public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS @Override public void run() { - log.debug("Start harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + if (harvestState.curLastModified != null) { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); + } else { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); + } - TransactionManager tm = nodeDAO.getTransactionManager(); - try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, null, - harvestState.curLastModified, true)) { + String uriBucket = null; // process all artifacts in a single thread + try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, harvestState.curLastModified, true)) { + TransactionManager tm = nodeDAO.getTransactionManager(); while (iter.hasNext()) { Artifact artifact = iter.next(); DataNode node = nodeDAO.getDataNode(artifact.getURI()); @@ -122,7 +138,8 @@ public void run() { node.bytesUsed = artifact.getContentLength(); nodeDAO.put(node); tm.commitTransaction(); - log.debug("Updated size of data node " + node.getName()); + log.debug("ArtifactSyncWorker.updateDataNode id=" + node.getID() + + " bytesUsed=" + node.bytesUsed + " artifact.lastModified=" + df.format(artifact.getLastModified())); } catch (Exception ex) { log.debug("Failed to update data node size for " + node.getName(), ex); tm.rollbackTransaction(); @@ -137,13 +154,19 @@ public void run() { } } harvestState.curLastModified = artifact.getLastModified(); - harvestState.curID = node.getID(); + harvestState.curID = artifact.getID(); harvestStateDAO.put(harvestState); } } catch (IOException ex) { log.error("Error closing iterator", ex); throw new RuntimeException("error while closing ResourceIterator", ex); } - log.debug("End harvesting " + harvestState.toString() + " at " + harvestState.curLastModified); + if (harvestState.curLastModified != null) { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); + } else { + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID + " end=true"); + } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index e4e51b418..12e1f2e05 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -88,6 +88,7 @@ import org.apache.log4j.Logger; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.PreauthKeyPair; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestStateDAO; import org.opencadc.inventory.db.PreauthKeyPairDAO; import org.opencadc.inventory.db.SQLGenerator; @@ -111,6 +112,7 @@ public class VaultInitAction extends InitAction { static final String JNDI_VOS_DATASOURCE = "jdbc/nodes"; // context.xml static final String JNDI_INV_DATASOURCE = "jdbc/inventory"; // context.xml + static final String JNDI_INV_ITER_DATASOURCE = "jdbc/inventory-iterator"; // context.xml static final String JNDI_UWS_DATASOURCE = "jdbc/uws"; // context.xml // config keys @@ -290,16 +292,17 @@ static Map getInvConfig(MultiValuedProperties props) { return ret; } - static Map getKeyPairConfig(MultiValuedProperties props) { - return getDaoConfig(props); - /* + static Map getIteratorConfig(MultiValuedProperties props) { Map ret = new TreeMap<>(); ret.put(SQLGenerator.class.getName(), SQLGenerator.class); // not configurable right now - ret.put("jndiDataSourceName", JNDI_VOS_DATASOURCE); - ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // requied but unused - ret.put("genSchema", props.getFirstPropertyValue(VOSPACE_SCHEMA_KEY)); + ret.put("jndiDataSourceName", JNDI_INV_ITER_DATASOURCE); + ret.put("invSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); + ret.put("genSchema", props.getFirstPropertyValue(INVENTORY_SCHEMA_KEY)); // for complete init return ret; - */ + } + + static Map getKeyPairConfig(MultiValuedProperties props) { + return getDaoConfig(props); } private void initConfig() { @@ -453,10 +456,15 @@ private void terminateAvailabilityCheck() { private void initBackgroundWorkers() { HarvestStateDAO hsDAO = new HarvestStateDAO(); - hsDAO.setConfig(getDaoConfig(props)); + hsDAO.setConfig(vosDaoConfig); + + ArtifactDAO artifactDAO = new ArtifactDAO(); + Map iterprops = getIteratorConfig(props); + log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); + artifactDAO.setConfig(iterprops); terminateBackgroundWorkers(); - this.artifactSync = new Thread(new ArtifactSync(hsDAO)); + this.artifactSync = new Thread(new ArtifactSync(hsDAO, artifactDAO, storageNamespace)); artifactSync.setDaemon(true); artifactSync.start(); } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java index ffd1879e6..c0f7d38f5 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java @@ -72,8 +72,11 @@ import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; +import org.opencadc.inventory.Namespace; +import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; +import org.opencadc.vospace.db.ArtifactSyncWorker; /** * Main artifact-sync agent that enables incremental sync of Artifact @@ -84,67 +87,91 @@ public class ArtifactSync implements Runnable { private static final Logger log = Logger.getLogger(ArtifactSync.class); - private static final long SHORT_SLEEP = 12000L; - private static final long LONG_SLEEP = 2 * SHORT_SLEEP; - private static final long EVICT_AGE = 3 * LONG_SLEEP; + private static final long ROUNDS = 6000L; // 6 sec + private static final long SHORT_SLEEP = 2 * ROUNDS; + private static final long LONG_SLEEP = 5 * ROUNDS; + private static final long EVICT_AGE = 12 * ROUNDS; + + private static final long FAIL_SLEEP = 10 * ROUNDS; // slightly smaller than evict private final UUID instanceID = UUID.randomUUID(); - private final HarvestStateDAO dao; - private String name = Artifact.class.getSimpleName(); - private URI resourceID = URI.create("jdbc/inventory"); + private final HarvestStateDAO stateDAO; + private final ArtifactDAO artifactDAO; + private final Namespace artifactNamespace; - public ArtifactSync(HarvestStateDAO dao) { - this.dao = dao; + public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { + this.stateDAO = stateDAO; + this.artifactDAO = artifactDAO; + this.artifactNamespace = artifactNamespace; // fenwick setup for production workload: //dao.setUpdateBufferCount(99); // buffer 99 updates, do every 100 //dao.setMaintCount(999); // buffer 999 so every 1000 real updates aka every 1e5 events - // here, we need timestamp updates to retain leader status, so - // dao.setMaintCount(9999); // every 1e4 + // we need continuous timestamp updates to retain leader status, so only schedule maintenance + stateDAO.setUpdateBufferCount(0); + stateDAO.setMaintCount(9999); // every 1e4 } @Override public void run() { + String name = Artifact.class.getSimpleName(); + URI resourceID = URI.create("jdbc:inventory"); try { Thread.sleep(SHORT_SLEEP); while (true) { - boolean leader = false; log.debug("check leader " + instanceID); - HarvestState state = dao.get(name, resourceID); + HarvestState state = stateDAO.get(name, resourceID); log.debug("check leader " + instanceID + " found: " + state); if (state.instanceID == null) { state.instanceID = instanceID; - dao.put(state); - state = dao.get(state.getID()); + stateDAO.put(state); + state = stateDAO.get(state.getID()); log.debug("created: " + state); } - if (instanceID.equals(state.instanceID)) { - log.debug("still the leader..."); - dao.put(state, true); - leader = true; - } else { - // see if we should perform a coup... - Date now = new Date(); - long age = now.getTime() - state.getLastModified().getTime(); - if (age > EVICT_AGE) { - - state.instanceID = instanceID; - dao.put(state); - state = dao.get(state.getID()); - leader = true; - log.debug("EVICTED " + state.instanceID + " because age " + age + " > " + EVICT_AGE); - } - } + + // determine leader + boolean leader = checkLeaderStatus(state); if (leader) { - log.debug("leader " + state.instanceID + " starting worker..."); - // TODO - dao.flushBufferedState(); - Thread.sleep(SHORT_SLEEP / 2L); // for testing - log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); - Thread.sleep(SHORT_SLEEP); + log.info("LEADER " + state.instanceID); + boolean fail = false; + try { + ArtifactSyncWorker worker = new ArtifactSyncWorker(stateDAO, state, artifactDAO, artifactNamespace); + worker.run(); + } catch (Exception ex) { + log.error("unexpected worker fail", ex); + fail = true; + } + + leader = checkLeaderStatus(state); + if (!fail && leader) { + try { + stateDAO.flushBufferedState(); + } catch (Exception ex) { + log.error("unexpected HarvestState flush fail", ex); + fail = true; + } + } + + if (!fail && leader) { + try { + // update leader timestamp before sleeping + stateDAO.put(state, true); + } catch (Exception ex) { + log.error("unexpected HarvestState force update fail", ex); + fail = true; + } + } + + if (fail) { + log.debug("failed leader " + state.instanceID + " sleep=" + FAIL_SLEEP); + Thread.sleep(FAIL_SLEEP); + } else { + log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); + Thread.sleep(SHORT_SLEEP); + } } else { log.debug("not leader: sleep=" + LONG_SLEEP); Thread.sleep(LONG_SLEEP); @@ -154,4 +181,24 @@ public void run() { log.debug("interrupted - assuming shutdown", ex); } } + + private boolean checkLeaderStatus(HarvestState state) { + boolean leader = false; + if (instanceID.equals(state.instanceID)) { + stateDAO.put(state, true); + leader = true; + } else { + // see if we should perform a coup... + Date now = new Date(); + long age = now.getTime() - state.getLastModified().getTime(); + if (age > EVICT_AGE) { + log.info("EVICTING " + state.instanceID + " because age " + age + " > " + EVICT_AGE); + state.instanceID = instanceID; + stateDAO.put(state); + state = stateDAO.get(state.getID()); + leader = true; + } + } + return leader; + } } diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index 94ec7b42f..0e787176d 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -28,7 +28,7 @@ testOnBorrow="true" validationQuery="select 123" /> - Date: Mon, 18 Mar 2024 15:43:04 -0700 Subject: [PATCH 164/186] vault: update DataNode.bytesUsed from artifact in get use transaction and lock node before update in move --- .../opencadc/vault/NodePersistenceImpl.java | 119 +++++++++++++----- 1 file changed, 86 insertions(+), 33 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index e3e98ceec..b22ae52bb 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -358,10 +358,6 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (ret == null) { return null; } - ret.parent = parent; - IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); - ret.owner = identityManager.toSubject(ret.ownerID); - ret.ownerDisplay = identityManager.toDisplayString(ret.owner); // in principle we could have queried vospace.Node join inventory.Artifact above // and avoid this query.... simplicity for now @@ -371,6 +367,46 @@ public Node get(ContainerNode parent, String name) throws TransientException { Artifact a = artifactDAO.get(dn.storageID); DateFormat df = NodeWriter.getDateFormat(); if (a != null) { + // DataNode.bytesUsed is an optimization (cache): + // if DataNode.bytesUsed != Artifact.contentLength we update the cache + // this retains put+get consistency in a single-site deployed (with minoc) + // and may help hide some inconsistencies in child listing sizes + if (!a.getContentLength().equals(dn.bytesUsed)) { + TransactionManager txn = dao.getTransactionManager(); + try { + log.debug("starting node transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + DataNode locked = (DataNode) dao.lock(dn); + if (locked != null) { + dn = locked; // safer than accidentally using the wrong variable + dn.bytesUsed = a.getContentLength(); + dao.put(dn); + ret = dn; + } + + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + if (locked == null) { + return null; // gone + } + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to update bytesUsed on " + dn.getID() + " aka " + dn.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); + } + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); + } + } + } + Date d = ret.getLastModified(); Date cd = null; if (ret.getLastModified().before(a.getLastModified())) { @@ -396,19 +432,17 @@ public Node get(ContainerNode parent, String name) throws TransientException { if (a.contentType != null) { ret.getProperties().add(new NodeProperty(VOS.PROPERTY_URI_TYPE, a.contentType)); } - // TODO: a.getContentLength() is correct, but might differ from dn.bytesUsed due to eventual consistency - // child listing iterator can only report dn.bytesUsed - // correct or consistency with listing?? - - // currently needed for consistency-requiring FilesTest - log.warn("DataNode.bytesUsed: " + dn.bytesUsed + " -> " + a.getContentLength()); - dn.bytesUsed = a.getContentLength(); } if (dn.bytesUsed == null) { - log.warn("DataNode.bytesUsed: 0 (no artifact)"); dn.bytesUsed = 0L; // no data stored } } + + ret.parent = parent; + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + ret.owner = identityManager.toSubject(ret.ownerID); + ret.ownerDisplay = identityManager.toDisplayString(ret.owner); + return ret; } @@ -645,7 +679,7 @@ public void move(Node node, ContainerNode dest, String newName) { if (node.parent == null || dest.parent == null) { throw new IllegalArgumentException("args must both be peristent nodes before move"); } - // try to detect attempt to dsconnect from path to root: node is a parent of dest + // try to detect attempt to disconnect from path to root: node is a parent of dest ContainerNode cur = dest; while (!cur.getID().equals(root.getID())) { cur = cur.parent; @@ -654,28 +688,48 @@ public void move(Node node, ContainerNode dest, String newName) { } } - - Subject caller = AuthenticationUtil.getCurrentSubject(); - node.owner = caller; - node.ownerID = null; - node.ownerDisplay = null; - node.parent = dest; - node.parentID = null; - if (newName != null) { - node.setName(newName); - } + + NodeDAO dao = getDAO(); + TransactionManager txn = dao.getTransactionManager(); try { - Node result = put(node); - log.debug("moved: " + result); - } catch (NodeNotSupportedException ex) { - LocalServiceURI loc = new LocalServiceURI(getResourceID()); - VOSURI srcURI = loc.getURI(node); - throw new RuntimeException("BUG: failed to move node because of detected type change: " - + srcURI.getPath() + " type=" + node.getClass().getSimpleName(), ex); + log.debug("starting node transaction"); + txn.startTransaction(); + log.debug("start txn: OK"); + + // lock the source node + Node locked = dao.lock(node); + if (locked != null) { + node = locked; // safer than having two vars and accidentally using the wrong one + Subject caller = AuthenticationUtil.getCurrentSubject(); + node.owner = caller; + node.ownerID = null; + node.ownerDisplay = null; + node.parent = dest; + node.parentID = null; + if (newName != null) { + node.setName(newName); + } + Node result = put(node); + log.debug("moved: " + result); + } + log.debug("commit txn..."); + txn.commitTransaction(); + log.debug("commit txn: OK"); + } catch (Exception ex) { + if (txn.isOpen()) { + log.error("failed to move " + node.getID() + " aka " + node.getName(), ex); + txn.rollbackTransaction(); + log.debug("rollback txn: OK"); + } + } finally { + if (txn.isOpen()) { + log.error("BUG - open transaction in finally"); + txn.rollbackTransaction(); + log.error("rollback txn: OK"); + } } } - /** * Delete the specified node. * @@ -721,8 +775,7 @@ public void delete(Node node) throws TransientException { } } else if (node instanceof DataNode) { DataNode dn = (DataNode) node; - NodeProperty len = dn.getProperty(VOS.PROPERTY_URI_CONTENTLENGTH); - if (len != null) { + if (dn.bytesUsed != null) { // artifact exists storageID = dn.storageID; } From e284c6ed25aa3b320ae06b1df87690288a8563b1 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 19 Mar 2024 12:10:53 -0700 Subject: [PATCH 165/186] vault: enable availability modes to disable and enable background ArtifactSync thread --- .../vospace/db/ArtifactSyncWorker.java | 12 ++-- .../opencadc/vault/NodePersistenceImpl.java | 10 ++- .../opencadc/vault/ServiceAvailability.java | 45 ++++++++++++- .../org/opencadc/vault/VaultInitAction.java | 64 +++++++++++++------ .../vault/VaultTransferGenerator.java | 5 +- .../org/opencadc/vault/files/GetAction.java | 2 - .../opencadc/vault/metadata/ArtifactSync.java | 20 ++++-- vault/src/main/webapp/META-INF/context.xml | 5 +- vault/src/main/webapp/WEB-INF/web.xml | 4 -- 9 files changed, 125 insertions(+), 42 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java index 33bf20948..96709c050 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java @@ -117,9 +117,11 @@ public void run() { DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); if (harvestState.curLastModified != null) { log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); + + " instance=" + harvestState.instanceID + + " start=" + df.format(harvestState.curLastModified)); } else { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); + log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + + " instance=" + harvestState.instanceID); } String uriBucket = null; // process all artifacts in a single thread @@ -163,10 +165,12 @@ public void run() { } if (harvestState.curLastModified != null) { log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); + + " instance=" + harvestState.instanceID + + " end=" + df.format(harvestState.curLastModified)); } else { log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID + " end=true"); + + " instance=" + harvestState.instanceID + + " end=true"); } } } diff --git a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java index b22ae52bb..bf2fbb388 100644 --- a/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java +++ b/vault/src/main/java/org/opencadc/vault/NodePersistenceImpl.java @@ -166,18 +166,22 @@ public class NodePersistenceImpl implements NodePersistence { private final Namespace storageNamespace; private final boolean localGroupsOnly; - private URI resourceID; + private final URI resourceID; private final boolean preventNotFound; + final String appName; // access by VaultTransferGenerator + // possibly temporary hack so migration tool can set this to false and // preserve lastModified timestamps on nodes public boolean nodeOrigin = true; - public NodePersistenceImpl(URI resourceID) { + public NodePersistenceImpl(URI resourceID, String appName) { if (resourceID == null) { throw new IllegalArgumentException("resource ID required"); } this.resourceID = resourceID; + this.appName = appName; + MultiValuedProperties config = VaultInitAction.getConfig(); this.nodeDaoConfig = VaultInitAction.getDaoConfig(config); this.invDaoConfig = VaultInitAction.getInvConfig(config); @@ -259,7 +263,7 @@ public TransferGenerator getTransferGenerator() { keyDAO.setConfig(kpDaoConfig); PreauthKeyPair kp = keyDAO.get(VaultInitAction.KEY_PAIR_NAME); TokenTool tt = new TokenTool(kp.getPublicKey(), kp.getPrivateKey()); - return new VaultTransferGenerator(this, getArtifactDAO(), tt, preventNotFound); + return new VaultTransferGenerator(this, appName, getArtifactDAO(), tt, preventNotFound); } private NodeDAO getDAO() { diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index 0b0005a42..e750e2bea 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -67,11 +67,17 @@ package org.opencadc.vault; +import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.vosi.Availability; import ca.nrc.cadc.vosi.AvailabilityPlugin; +import ca.nrc.cadc.vosi.avail.CheckDataSource; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.vault.metadata.ArtifactSync; /** * This class performs the work of determining if the executing artifact @@ -130,7 +136,31 @@ public Availability getStatus() { return new Availability(false, RestAction.STATE_READ_ONLY_MSG); } - //TODO add availability checks for dependent services + // check database pools + DataSource ds; + String testSQL; + CheckDataSource cds; + + ds = DBUtil.findJNDIDataSource("jdbc/nodes"); + testSQL = "select * from vospace.ModelVersion"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/inventory"); + testSQL = "select * from inventory.Artifact limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/inventory-iterator"); + testSQL = "select * from inventory.Artifact limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + + ds = DBUtil.findJNDIDataSource("jdbc/uws"); + testSQL = "select * from uws.Job limit 1"; + cds = new CheckDataSource(ds, testSQL); + cds.check(); + } catch (Throwable t) { // the test itself failed log.debug("failure", t); @@ -149,10 +179,13 @@ public void setState(String state) { String key = appName + RestAction.STATE_MODE_KEY; if (RestAction.STATE_OFFLINE.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_OFFLINE); + setOffline(true); } else if (RestAction.STATE_READ_ONLY.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_READ_ONLY); + setOffline(true); } else if (RestAction.STATE_READ_WRITE.equalsIgnoreCase(state)) { System.setProperty(key, RestAction.STATE_READ_WRITE); + setOffline(false); } else { throw new IllegalArgumentException("invalid state: " + state + " expected: " + RestAction.STATE_READ_WRITE + "|" @@ -170,4 +203,14 @@ private String getState() { return ret; } + private void setOffline(boolean offline) { + String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + try { + InitialContext initialContext = new InitialContext(); + ArtifactSync async = (ArtifactSync) initialContext.lookup(jndiArtifactSync); + async.setOffline(offline); + } catch (NamingException e) { + log.debug(String.format("unable to unbind %s - %s", jndiArtifactSync, e.getMessage())); + } + } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index 12e1f2e05..f10298f7f 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2023. (c) 2023. +* (c) 2024. (c) 2024. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -134,10 +134,11 @@ public class VaultInitAction extends InitAction { private List allocationParents = new ArrayList<>(); private String jndiNodePersistence; - private String jndiPreauthKeys; + private String jndiPreauthKeys; // store pubkey in JNDI for download via GetKeyAction + private String jndiArtifactSync; // store in JNDI to support availability mode change private String jndiSiteAvailabilities; private Thread availabilityCheck; - private Thread artifactSync; + private Thread artifactSyncThread; public VaultInitAction() { super(); @@ -369,7 +370,7 @@ private void initNodePersistence() { } catch (NamingException ignore) { log.debug("unbind previous JNDI key (" + jndiNodePersistence + ") failed... ignoring"); } - NodePersistence npi = new NodePersistenceImpl(resourceID); + NodePersistence npi = new NodePersistenceImpl(resourceID, appName); ctx.bind(jndiNodePersistence, npi); log.info("initNodePersistence: created JNDI key: " + jndiNodePersistence + " impl: " + npi.getClass().getName()); @@ -455,31 +456,54 @@ private void terminateAvailabilityCheck() { } private void initBackgroundWorkers() { - HarvestStateDAO hsDAO = new HarvestStateDAO(); - hsDAO.setConfig(vosDaoConfig); - - ArtifactDAO artifactDAO = new ArtifactDAO(); - Map iterprops = getIteratorConfig(props); - log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); - artifactDAO.setConfig(iterprops); - - terminateBackgroundWorkers(); - this.artifactSync = new Thread(new ArtifactSync(hsDAO, artifactDAO, storageNamespace)); - artifactSync.setDaemon(true); - artifactSync.start(); + try { + HarvestStateDAO hsDAO = new HarvestStateDAO(); + hsDAO.setConfig(vosDaoConfig); + + ArtifactDAO artifactDAO = new ArtifactDAO(); + Map iterprops = getIteratorConfig(props); + log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); + artifactDAO.setConfig(iterprops); + + terminateBackgroundWorkers(); + ArtifactSync async = new ArtifactSync(hsDAO, artifactDAO, storageNamespace); + this.artifactSyncThread = new Thread(async); + artifactSyncThread.setDaemon(true); + artifactSyncThread.start(); + + // store in JNDI so availability can set offline + String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + InitialContext ctx = new InitialContext(); + try { + ctx.unbind(jndiArtifactSync); + } catch (NamingException ignore) { + log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); + } + ctx.bind(jndiArtifactSync, async); + log.info("initBackgroundWorkers: created JNDI key: " + jndiArtifactSync); + } catch (Exception ex) { + throw new RuntimeException("check/init ArtifactSync failed", ex); + } } private void terminateBackgroundWorkers() { - if (this.artifactSync != null) { + if (this.artifactSyncThread != null) { try { log.info("terminating ArtifactSync Thread..."); - this.artifactSync.interrupt(); - this.artifactSync.join(); + this.artifactSyncThread.interrupt(); + this.artifactSyncThread.join(); log.info("terminating ArtifactSync Thread... [OK]"); } catch (Throwable t) { log.info("failed to terminate ArtifactSync thread", t); } finally { - this.artifactSync = null; + this.artifactSyncThread = null; + } + + try { + InitialContext initialContext = new InitialContext(); + initialContext.unbind(this.jndiArtifactSync); + } catch (NamingException e) { + log.debug(String.format("unable to unbind %s - %s", this.jndiArtifactSync, e.getMessage())); } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java index 51fec42a9..3b0884887 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java +++ b/vault/src/main/java/org/opencadc/vault/VaultTransferGenerator.java @@ -117,7 +117,8 @@ public class VaultTransferGenerator implements TransferGenerator { private final Map siteAvailabilities; @SuppressWarnings("unchecked") - public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { + public VaultTransferGenerator(NodePersistenceImpl nodePersistence, String appName, + ArtifactDAO artifactDAO, TokenTool tokenTool, boolean preventNotFound) { this.nodePersistence = nodePersistence; this.authorizer = new VOSpaceAuthorizer(nodePersistence); this.artifactDAO = artifactDAO; @@ -125,7 +126,7 @@ public VaultTransferGenerator(NodePersistenceImpl nodePersistence, ArtifactDAO a this.preventNotFound = preventNotFound; // TODO: get appname from ??? - String siteAvailabilitiesKey = "vault" + "-" + StorageSiteAvailabilityCheck.class.getName(); + String siteAvailabilitiesKey = appName + "-" + StorageSiteAvailabilityCheck.class.getName(); log.debug("siteAvailabilitiesKey: " + siteAvailabilitiesKey); try { Context initContext = new InitialContext(); diff --git a/vault/src/main/java/org/opencadc/vault/files/GetAction.java b/vault/src/main/java/org/opencadc/vault/files/GetAction.java index d3967e5c2..231cad51e 100644 --- a/vault/src/main/java/org/opencadc/vault/files/GetAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/GetAction.java @@ -70,12 +70,10 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.net.TransientException; import java.net.HttpURLConnection; -import java.net.URI; import java.util.List; import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.opencadc.vospace.DataNode; -import org.opencadc.vospace.NodeProperty; import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.server.NodeFault; diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java index c0f7d38f5..023a2d2b4 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java @@ -88,8 +88,8 @@ public class ArtifactSync implements Runnable { private static final Logger log = Logger.getLogger(ArtifactSync.class); private static final long ROUNDS = 6000L; // 6 sec - private static final long SHORT_SLEEP = 2 * ROUNDS; - private static final long LONG_SLEEP = 5 * ROUNDS; + private static final long SHORT_SLEEP = 5 * ROUNDS; + private static final long LONG_SLEEP = 10 * ROUNDS; private static final long EVICT_AGE = 12 * ROUNDS; private static final long FAIL_SLEEP = 10 * ROUNDS; // slightly smaller than evict @@ -99,6 +99,8 @@ public class ArtifactSync implements Runnable { private final ArtifactDAO artifactDAO; private final Namespace artifactNamespace; + private boolean offline = false; + public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { this.stateDAO = stateDAO; this.artifactDAO = artifactDAO; @@ -113,14 +115,24 @@ public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace stateDAO.setMaintCount(9999); // every 1e4 } + public void setOffline(boolean offline) { + this.offline = offline; + + } + @Override public void run() { String name = Artifact.class.getSimpleName(); URI resourceID = URI.create("jdbc:inventory"); try { - Thread.sleep(SHORT_SLEEP); + Thread.sleep(1 * ROUNDS); // delay startup a bit while (true) { + while (offline) { + log.warn("disabled: sleep=" + LONG_SLEEP); + Thread.sleep(LONG_SLEEP); + } + log.debug("check leader " + instanceID); HarvestState state = stateDAO.get(name, resourceID); log.debug("check leader " + instanceID + " found: " + state); @@ -135,7 +147,7 @@ public void run() { boolean leader = checkLeaderStatus(state); if (leader) { - log.info("LEADER " + state.instanceID); + log.debug("leader: " + state); boolean fail = false; try { ArtifactSyncWorker worker = new ArtifactSyncWorker(stateDAO, state, artifactDAO, artifactNamespace); diff --git a/vault/src/main/webapp/META-INF/context.xml b/vault/src/main/webapp/META-INF/context.xml index 0e787176d..02f5f7090 100644 --- a/vault/src/main/webapp/META-INF/context.xml +++ b/vault/src/main/webapp/META-INF/context.xml @@ -27,14 +27,15 @@ removeAbandoned="false" testOnBorrow="true" validationQuery="select 123" /> - + ca.nrc.cadc.vosi.AvailabilityPlugin org.opencadc.vault.ServiceAvailability - - availabilityProperties - vault-availability.properties - 3 From ea52a02eb6d4909d86a1e90cc6a02002176ce4f9 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 19 Mar 2024 15:21:58 -0700 Subject: [PATCH 166/186] update VERSIONs for service images and add change log --- ChangeLog.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ luskan/README.md | 4 +++ luskan/VERSION | 2 +- minoc/README.md | 25 +++-------------- minoc/VERSION | 2 +- raven/README.md | 4 +++ raven/VERSION | 2 +- vault/VERSION | 2 +- 8 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 ChangeLog.md diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 000000000..7fca1ef3a --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,70 @@ +# Change Log + +This is a cursory summary of changes to various storage-inventory components. +Check the README in specific modules for details. + +## minoc:1.0.0 +``` +added optional `org.opencadc.minoc.trust.preauth` config +removed optional `org.opencadc.minoc.publicKeyFile` config +``` +A `minoc` instance will download a public key from each trusted service and +use the public key(s) to validate URLs that include a _preauth_ token. + +``` +added optional `org.opencadc.minoc.readable` and `org.opencadc.minoc.writable` config +``` +A `minoc` service will advertise (via inventory.StorageSite record in the database) the +_readable_ and _writable_ status; this information is synced to global inventory and +used by `raven` to determine if it should generate PUT or GET URLs that use the `minoc` +service(s) at that site. By default, the configuration of _readGrantProvider_(s) and +_writeGrantProvider_(s) is used to determine the default status; configuration of any +_trust.preauth_ will also make make the status _readable_ and _writable_ even if the +service is only intending to accept one or the other. + +The explicit _readable_ and _writable_ configuration options will override all other +logic and set the status accordingly. This is currently optional but may be required +in a future version. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## raven:1.0.0 +``` +added org.opencadc.raven.inventory connection pool +``` +A `raven` service uses this pool to perform database initialization. This pool is +configured in the `catalina.properties` file. + +``` +added optional `org.opencadc.raven.keys.preauth` config +removed optional `org.opencadc.raven.publicKeyFile` and `org.opencadc.minoc.privateKeyFile` config +``` +When configured to do so, a `raven` service will generate it's own public/private key pair +and use the private key to _sign_ `minoc` URLs. All the `minoc` services known to the global +`raven` service must also be configured to _trust_ `raven`. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## luskan:1.0.0 +``` +changed config file: cadc-tap-tmp.properties +``` +A `luskan` service now uses the DelegatingStorageManager` so this config file must +specify which storage manager implementation to use along with existing +implementation-specific configuration options. + +``` +added optional config file: cadc-log.properties +added optional config file: cadc-vosi.properties +``` + +## vault:0.5.0 (NEW) +This is a new service that implements the IVOA VOSpace 2.1 REST API to provide user-managed +hierarchical storage. `vault` is deployed with it's own database and an associated inventory +database, uses inventory services (`minoc`) for file storage and management. diff --git a/luskan/README.md b/luskan/README.md index 0d4dd2093..90f4ccdf9 100644 --- a/luskan/README.md +++ b/luskan/README.md @@ -104,6 +104,10 @@ Assuming instances are restarted regularly, this would cause rollover approximat See cadc-log for common dynamic logging control. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. + ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/luskan/VERSION b/luskan/VERSION index e894843e4..2b8664584 100644 --- a/luskan/VERSION +++ b/luskan/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=0.6.8 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/minoc/README.md b/minoc/README.md index 0f27e2b08..e0292b941 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -144,16 +144,9 @@ will be able to read files. See cadc-log for common dynamic logging control. -### minoc-availability.properties (optional) -WARN: This config file name is going to change to a common one for consistency across multiple services. - -The minoc-availability.properties file specifies which users have the authority to change the availability state of the minoc service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical user name. - -Example: -``` -users = {user identity} -``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user identities (X500 distingushed name), one line per user. Optional: if the `minoc-availability.properties` is not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the state will not be changeable. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. @@ -174,15 +167,3 @@ docker run --rm -it minoc:latest /bin/bash docker run --rm --user tomcat:tomcat --volume=/path/to/external/config:/config:ro --name minoc minoc:latest ``` -## using it - -Using `cURL` is possible with Minoc to put a file for testing. - -**Note:** The `content-type` header is important! -```bash -$ curl -v -X PUT \ - --header "content-type: application/fits" \ - --data-binary @myfile.fits \ - -E ~/.ssl/cadcproxy.pem \ - https://myhost.com/minoc/files/test:TEST/myfile.fits -``` diff --git a/minoc/VERSION b/minoc/VERSION index a8d5aa3a2..3d4fab565 100644 --- a/minoc/VERSION +++ b/minoc/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.9.10 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/raven/README.md b/raven/README.md index e4ea4f0d9..cac828ddc 100644 --- a/raven/README.md +++ b/raven/README.md @@ -112,6 +112,10 @@ will be able to read files. See cadc-log for common dynamic logging control. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. + ### cadcproxy.pem (optional) This client certificate is used to make authenticated server-to-server calls for system-level A&A purposes. diff --git a/raven/VERSION b/raven/VERSION index ee967b01d..6a39bdba3 100644 --- a/raven/VERSION +++ b/raven/VERSION @@ -2,6 +2,6 @@ # semantic version tag: major.minor[.patch] # build version tag: timestamp # tag: {semantic}-{build} -VER=0.8.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/vault/VERSION b/vault/VERSION index 9d05f397f..17ac68c1c 100644 --- a/vault/VERSION +++ b/vault/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.2.0 +VER=0.5.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER From ba3bd5cc62462c274ef8b66cb4cf42ea80a04f72 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 09:55:46 -0700 Subject: [PATCH 167/186] update image versions --- critwall/VERSION | 2 +- fenwick/VERSION | 2 +- ratik/VERSION | 2 +- ringhold/VERSION | 4 +++- tantar/VERSION | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/critwall/VERSION b/critwall/VERSION index ba99f57c6..7fbddd116 100644 --- a/critwall/VERSION +++ b/critwall/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.4.5 +VER=0.5.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/fenwick/VERSION b/fenwick/VERSION index fed2d5959..a77484760 100644 --- a/fenwick/VERSION +++ b/fenwick/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.5.7 +VER=0.6.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/ratik/VERSION b/ratik/VERSION index 79fec5904..d9e5d6047 100644 --- a/ratik/VERSION +++ b/ratik/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.1.9 +VER=0.2.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/ringhold/VERSION b/ringhold/VERSION index 51807fa89..b3562c8a8 100644 --- a/ringhold/VERSION +++ b/ringhold/VERSION @@ -1,4 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -TAGS="0.2-$(date --utc +"%Y%m%dT%H%M%S")" +VER=0.2.0 +TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" +unset VER diff --git a/tantar/VERSION b/tantar/VERSION index ba99f57c6..7fbddd116 100644 --- a/tantar/VERSION +++ b/tantar/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.4.5 +VER=0.5.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER From 3ca95773baa4a75a8ba7960338b0ac34f81edd01 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 14:01:29 -0700 Subject: [PATCH 168/186] cadc-inventory and cadc-inventory-db versions 1.0.0 inventory and vospace DDL version 1.0.0 remove intermediate vospace DDL upgrade minoc: fix pubkey loading bug --- cadc-inventory-db/build.gradle | 6 +- .../inventory/db/version/InitDatabaseSI.java | 4 +- .../opencadc/vospace/db/InitDatabaseVOS.java | 7 +-- ...e-0.15.sql => inventory.upgrade-1.0.0.sql} | 0 .../main/resources/vospace.upgrade-0.15.sql | 7 --- cadc-inventory/build.gradle | 4 +- .../java/org/opencadc/minoc/MinocConfig.java | 59 +++++++++---------- tantar/build.gradle | 4 +- vault/build.gradle | 8 +-- 9 files changed, 43 insertions(+), 56 deletions(-) rename cadc-inventory-db/src/main/resources/{inventory.upgrade-0.15.sql => inventory.upgrade-1.0.0.sql} (100%) delete mode 100644 cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql diff --git a/cadc-inventory-db/build.gradle b/cadc-inventory-db/build.gradle index 1b5a7ca8f..91e5e51d8 100644 --- a/cadc-inventory-db/build.gradle +++ b/cadc-inventory-db/build.gradle @@ -17,7 +17,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.15.0' +version = '1.0.0' description = 'OpenCADC Storage Inventory database library' def git_url = 'https://github.com/opencadc/storage-inventory' @@ -27,8 +27,8 @@ mainClassName = 'org.opencadc.inventory.db.version.Main' dependencies { compile 'org.opencadc:cadc-util:[1.11.0,2.0)' compile 'org.opencadc:cadc-gms:[1.0.0,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,)' - compile 'org.opencadc:cadc-vos:[2.0.5,3.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,)' + compile 'org.opencadc:cadc-vos:[2.0.6,3.0)' testCompile 'junit:junit:[4.0,)' diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java index 6e810b8f3..f59d7d767 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java @@ -80,7 +80,7 @@ public class InitDatabaseSI extends InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseSI.class); public static final String MODEL_NAME = "storage-inventory"; - public static final String MODEL_VERSION = "0.15"; + public static final String MODEL_VERSION = "1.0.0"; public static final String PREV_MODEL_VERSION = "0.14"; //public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT"; @@ -98,7 +98,7 @@ public class InitDatabaseSI extends InitDatabase { }; static String[] UPGRADE_SQL = new String[] { - "inventory.upgrade-0.15.sql", + "inventory.upgrade-1.0.0.sql", "generic.PreauthKeyPair.sql", "generic.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java index 05d8c45b4..40d7f29de 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java @@ -79,9 +79,9 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { private static final Logger log = Logger.getLogger(InitDatabaseVOS.class); - public static final String MODEL_NAME = "vospace-inventory"; - public static final String MODEL_VERSION = "0.15"; - public static final String PREV_MODEL_VERSION = "0.3"; + public static final String MODEL_NAME = "storage-vospace"; + public static final String MODEL_VERSION = "1.0.0"; + public static final String PREV_MODEL_VERSION = "n/a"; static String[] CREATE_SQL = new String[] { "generic.ModelVersion.sql", @@ -93,7 +93,6 @@ public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase { }; static String[] UPGRADE_SQL = new String[] { - "vospace.upgrade-0.15.sql", "generic.permissions.sql" }; diff --git a/cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql similarity index 100% rename from cadc-inventory-db/src/main/resources/inventory.upgrade-0.15.sql rename to cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql diff --git a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql b/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql deleted file mode 100644 index bfb33166e..000000000 --- a/cadc-inventory-db/src/main/resources/vospace.upgrade-0.15.sql +++ /dev/null @@ -1,7 +0,0 @@ - -alter table .Node - add column bytesUsed bigint, - add column storageBucket varchar(5) -; - -create unique index node_storageID on .Node(storageID); \ No newline at end of file diff --git a/cadc-inventory/build.gradle b/cadc-inventory/build.gradle index 55f43e500..cfe772697 100644 --- a/cadc-inventory/build.gradle +++ b/cadc-inventory/build.gradle @@ -14,13 +14,13 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '0.10.0' +version = '1.0.0' description = 'OpenCADC Storage Inventory core library' def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { - compile 'org.opencadc:cadc-util:[1.10.6,2.0)' + compile 'org.opencadc:cadc-util:[1.11.0,2.0)' testCompile 'junit:junit:[4.0,)' } diff --git a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java index d4b27d9b9..437804b4a 100644 --- a/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java +++ b/minoc/src/main/java/org/opencadc/minoc/MinocConfig.java @@ -116,7 +116,6 @@ public class MinocConfig { private final MultiValuedProperties configProperties; private final Map trustedServices = new TreeMap<>(); - private boolean trustedServiceKeySync = false; private final List readGrantServices = new ArrayList<>(); private final List writeGrantServices = new ArrayList<>(); private final boolean readable; @@ -180,7 +179,7 @@ public MinocConfig() { throw new IllegalStateException("invalid config: " + TRUST_KEY + "=" + s + " INVALID", ex); } } - trustedServiceKeySync = false; + // try to sync keys on startup syncKeys(); } @@ -351,44 +350,40 @@ public MultiValuedProperties getProperties() { } public Map getTrustedServices() { + // check and try to sync missing keys before request syncKeys(); return trustedServices; } private void syncKeys() { - if (!trustedServiceKeySync) { - int numFails = 0; - RegistryClient reg = new RegistryClient(); - // check map for null keys and try to retrieve them - for (Map.Entry me : trustedServices.entrySet()) { - if (me.getValue() == null) { - try { - log.info("get trusted pubkey: " + me.getKey()); - URL capURL = reg.getAccessURL(RegistryClient.Query.CAPABILITIES, me.getKey()); - String s = capURL.toExternalForm().replace("/capabilities", "/pubkey"); - URL keyURL = new URL(s); - log.info("get trusted pubkey: " + me.getKey() + " -> " + keyURL); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - HttpGet get = new HttpGet(keyURL, bos); - get.setConnectionTimeout(6000); - get.setReadTimeout(6000); - get.setRetry(0, 0, HttpTransfer.RetryReason.NONE); - get.run(); - if (get.getThrowable() != null) { - throw (Exception) get.getThrowable(); - } - byte[] key = bos.toByteArray(); - trustedServices.put(me.getKey(), key); - log.info("get trusted pubkey: " + me.getKey() + " OK"); - } catch (Exception ex) { - log.warn("failed to get public key from " + me.getKey() + ": " + ex); - numFails++; + RegistryClient reg = new RegistryClient(); + // check map for null keys and try to retrieve them + // ASSUMPTION: keys never change once generated so if they do then minoc + // needs to be restarted + for (Map.Entry me : trustedServices.entrySet()) { + if (me.getValue() == null) { + try { + log.info("get trusted pubkey: " + me.getKey()); + URL capURL = reg.getAccessURL(RegistryClient.Query.CAPABILITIES, me.getKey()); + String s = capURL.toExternalForm().replace("/capabilities", "/pubkey"); + URL keyURL = new URL(s); + log.info("get trusted pubkey: " + me.getKey() + " -> " + keyURL); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(keyURL, bos); + get.setConnectionTimeout(6000); + get.setReadTimeout(6000); + get.setRetry(0, 0, HttpTransfer.RetryReason.NONE); + get.run(); + if (get.getThrowable() != null) { + throw (Exception) get.getThrowable(); } + byte[] key = bos.toByteArray(); + trustedServices.put(me.getKey(), key); + log.info("get trusted pubkey: " + me.getKey() + " OK"); + } catch (Exception ex) { + log.warn("failed to get public key from " + me.getKey() + ": " + ex); } } - if (numFails == 0) { - trustedServiceKeySync = true; - } } } diff --git a/tantar/build.gradle b/tantar/build.gradle index c99751a61..154e5309d 100644 --- a/tantar/build.gradle +++ b/tantar/build.gradle @@ -21,8 +21,8 @@ mainClassName = 'org.opencadc.tantar.Main' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' compile 'org.opencadc:cadc-log:[1.1.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-storage-adapter:[0.11.1,1.0)' diff --git a/vault/build.gradle b/vault/build.gradle index a11549995..e6ac8689f 100644 --- a/vault/build.gradle +++ b/vault/build.gradle @@ -31,11 +31,11 @@ def git_url = 'https://github.com/opencadc/storage-inventory' dependencies { compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' - compile 'org.opencadc:cadc-util:[1.9.10,2.0)' + compile 'org.opencadc:cadc-util:[1.11.0,2.0)' compile 'org.opencadc:cadc-log:[1.1.6,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,)' compile 'org.opencadc:cadc-rest:[1.3.16,)' - compile 'org.opencadc:cadc-vos:[2.0.3,)' + compile 'org.opencadc:cadc-vos:[2.0.6,)' compile 'org.opencadc:cadc-vos-server:[2.0.9,)' compile 'org.opencadc:cadc-vosi:[1.3.2,)' compile 'org.opencadc:cadc-uws:[1.0,)' @@ -43,8 +43,8 @@ dependencies { compile 'org.opencadc:cadc-access-control:[1.1.1,2.0)' compile 'org.opencadc:cadc-cdp:[1.2.3,)' compile 'org.opencadc:cadc-registry:[1.7.6,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,1.0)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.3,1.0)' compile 'org.opencadc:cadc-permissions:[0.3.5,1.0)' From 8a729871bc4f2f905e0206921dd199bf91658607 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 14:59:48 -0700 Subject: [PATCH 169/186] update cadc-inventory lib dependencies for 1.0 milestone --- critwall/build.gradle | 5 ++--- fenwick/build.gradle | 5 ++--- minoc/build.gradle | 4 ++-- ratik/build.gradle | 5 ++--- raven/build.gradle | 4 ++-- ringhold/build.gradle | 4 ++-- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/critwall/build.gradle b/critwall/build.gradle index 9b9a74cd9..83716cb15 100644 --- a/critwall/build.gradle +++ b/critwall/build.gradle @@ -21,9 +21,8 @@ mainClassName = 'org.opencadc.critwall.Main' dependencies { compile 'org.opencadc:cadc-storage-adapter:[0.8,1.0)' compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - // cadc-inventory-db-0.15 is in the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15.0,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-registry:[1.7,2.0)' compile 'org.opencadc:cadc-vosi:[1.3.6,2.0)' compile 'org.opencadc:cadc-vos:[2.0,)' diff --git a/fenwick/build.gradle b/fenwick/build.gradle index eb9cce1e1..f7feee09f 100644 --- a/fenwick/build.gradle +++ b/fenwick/build.gradle @@ -16,9 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.14,1.2)' // 1.2 upper bound is correct #reasons diff --git a/minoc/build.gradle b/minoc/build.gradle index 45705d4f2..add0a28c7 100644 --- a/minoc/build.gradle +++ b/minoc/build.gradle @@ -34,8 +34,8 @@ dependencies { compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-data-ops-fits:[0.3.0,)' compile 'org.opencadc:cadc-gms:[1.0.0,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.2.1,)' compile 'org.opencadc:cadc-soda-server:[1.2.0,2.0.0)' compile 'org.opencadc:cadc-storage-adapter:[0.11.2,)' diff --git a/ratik/build.gradle b/ratik/build.gradle index 6eef59f99..79137835c 100644 --- a/ratik/build.gradle +++ b/ratik/build.gradle @@ -16,9 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.10.2,2.0)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - // cadc-inventory-db-0.15.0 is from the vos2 feature branch - compile 'org.opencadc:cadc-inventory-db:[0.15,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-util:[0.1.8,1.0)' compile 'org.opencadc:cadc-registry:[1.5,2.0)' compile 'org.opencadc:cadc-tap:[1.1.15,2.0)' diff --git a/raven/build.gradle b/raven/build.gradle index a20a7319a..fec3cf4f1 100644 --- a/raven/build.gradle +++ b/raven/build.gradle @@ -33,8 +33,8 @@ dependencies { compile 'org.opencadc:cadc-rest:[1.3.14,)' compile 'org.opencadc:cadc-cdp:[1.0,)' compile 'org.opencadc:cadc-gms:[1.0.4,)' - compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)' - compile 'org.opencadc:cadc-inventory-db:[0.15.0,)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' compile 'org.opencadc:cadc-inventory-server:[0.3.0,)' compile 'org.opencadc:cadc-permissions:[0.3.5,)' compile 'org.opencadc:cadc-permissions-client:[0.3,)' diff --git a/ringhold/build.gradle b/ringhold/build.gradle index e29f2e50c..685a569da 100644 --- a/ringhold/build.gradle +++ b/ringhold/build.gradle @@ -16,8 +16,8 @@ group = 'org.opencadc' dependencies { compile 'org.opencadc:cadc-util:[1.6,2.0)' - compile 'org.opencadc:cadc-inventory:[0.10,)' - compile 'org.opencadc:cadc-inventory-db:[0.15,1.0)' + compile 'org.opencadc:cadc-inventory:[1.0,2.0)' + compile 'org.opencadc:cadc-inventory-db:[1.0,2.0)' testCompile 'junit:junit:[4.12,5.0)' } From d762437a8ee9c20e57798ce5c8c0f9a886155b0c Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 20 Mar 2024 15:41:59 -0700 Subject: [PATCH 170/186] rename ArtifactSync to DataNodeSizeSync for clarity --- ...rTest.java => DataNodeSizeWorkerTest.java} | 10 ++--- ...yncWorker.java => DataNodeSizeWorker.java} | 15 +++---- .../opencadc/vault/ServiceAvailability.java | 6 +-- .../org/opencadc/vault/VaultInitAction.java | 42 ++++++++++--------- ...rtifactSync.java => DataNodeSizeSync.java} | 10 ++--- 5 files changed, 43 insertions(+), 40 deletions(-) rename cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/{ArtifactSyncWorkerTest.java => DataNodeSizeWorkerTest.java} (97%) rename cadc-inventory-db/src/main/java/org/opencadc/vospace/db/{ArtifactSyncWorker.java => DataNodeSizeWorker.java} (93%) rename vault/src/main/java/org/opencadc/vault/metadata/{ArtifactSync.java => DataNodeSizeSync.java} (95%) diff --git a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java similarity index 97% rename from cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java rename to cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java index fa18912c6..649800b31 100644 --- a/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/ArtifactSyncWorkerTest.java +++ b/cadc-inventory-db/src/intTest/java/org/opencadc/vospace/db/DataNodeSizeWorkerTest.java @@ -84,7 +84,7 @@ import org.junit.Before; import org.junit.Test; import org.opencadc.inventory.Artifact; -import org.opencadc.vospace.db.ArtifactSyncWorker; +import org.opencadc.vospace.db.DataNodeSizeWorker; import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; @@ -101,8 +101,8 @@ * * @author adriand */ -public class ArtifactSyncWorkerTest { - private static final Logger log = Logger.getLogger(ArtifactSyncWorkerTest.class); +public class DataNodeSizeWorkerTest { + private static final Logger log = Logger.getLogger(DataNodeSizeWorkerTest.class); static { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); @@ -117,7 +117,7 @@ public class ArtifactSyncWorkerTest { ArtifactDAO artifactDAO; - public ArtifactSyncWorkerTest() throws Exception { + public DataNodeSizeWorkerTest() throws Exception { DBConfig dbrc = new DBConfig(); ConnectionConfig cc = dbrc.getConnectionConfig(TestUtil.SERVER, TestUtil.DATABASE); DBUtil.PoolConfig pool = new DBUtil.PoolConfig(cc, 1, 6000L, "select 123"); @@ -219,7 +219,7 @@ public void testSyncArtifact() throws Exception { harvestStateDAO.put(hs); hs = harvestStateDAO.get(hsName, resourceID); - ArtifactSyncWorker asWorker = new ArtifactSyncWorker(harvestStateDAO, hs, artifactDAO, siNamespace); + DataNodeSizeWorker asWorker = new DataNodeSizeWorker(harvestStateDAO, hs, artifactDAO, siNamespace); asWorker.run(); actual = (DataNode)nodeDAO.get(orig.getID()); diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java similarity index 93% rename from cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java rename to cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index 96709c050..0b40f8484 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/ArtifactSyncWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -86,8 +86,8 @@ * * @author adriand */ -public class ArtifactSyncWorker implements Runnable { - private static final Logger log = Logger.getLogger(ArtifactSyncWorker.class); +public class DataNodeSizeWorker implements Runnable { + private static final Logger log = Logger.getLogger(DataNodeSizeWorker.class); private final HarvestState harvestState; private final NodeDAO nodeDAO; @@ -103,7 +103,7 @@ public class ArtifactSyncWorker implements Runnable { * @param artifactDAO DAO class to query for artifacts * @param namespace artifact namespace */ - public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, + public DataNodeSizeWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState, ArtifactDAO artifactDAO, Namespace namespace) { this.harvestState = harvestState; this.harvestStateDAO = harvestStateDAO; @@ -114,13 +114,14 @@ public ArtifactSyncWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS @Override public void run() { + String opName = DataNodeSizeWorker.class.getSimpleName() + ".artifactQuery"; DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); if (harvestState.curLastModified != null) { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); } else { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); } @@ -164,11 +165,11 @@ public void run() { throw new RuntimeException("error while closing ResourceIterator", ex); } if (harvestState.curLastModified != null) { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); } else { - log.info("ArtifactSyncWorker.artifactQuery source=" + harvestState.getResourceID() + log.info(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=true"); } diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index e750e2bea..6e22d8c83 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -77,7 +77,7 @@ import javax.naming.NamingException; import javax.sql.DataSource; import org.apache.log4j.Logger; -import org.opencadc.vault.metadata.ArtifactSync; +import org.opencadc.vault.metadata.DataNodeSizeSync; /** * This class performs the work of determining if the executing artifact @@ -204,10 +204,10 @@ private String getState() { } private void setOffline(boolean offline) { - String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + String jndiArtifactSync = appName + "-" + DataNodeSizeSync.class.getName(); try { InitialContext initialContext = new InitialContext(); - ArtifactSync async = (ArtifactSync) initialContext.lookup(jndiArtifactSync); + DataNodeSizeSync async = (DataNodeSizeSync) initialContext.lookup(jndiArtifactSync); async.setOffline(offline); } catch (NamingException e) { log.debug(String.format("unable to unbind %s - %s", jndiArtifactSync, e.getMessage())); diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index f10298f7f..b61f34705 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -95,7 +95,7 @@ import org.opencadc.inventory.db.StorageSiteDAO; import org.opencadc.inventory.db.version.InitDatabaseSI; import org.opencadc.inventory.transfer.StorageSiteAvailabilityCheck; -import org.opencadc.vault.metadata.ArtifactSync; +import org.opencadc.vault.metadata.DataNodeSizeSync; import org.opencadc.vospace.db.InitDatabaseVOS; import org.opencadc.vospace.server.NodePersistence; import org.springframework.dao.DataIntegrityViolationException; @@ -135,10 +135,12 @@ public class VaultInitAction extends InitAction { private String jndiNodePersistence; private String jndiPreauthKeys; // store pubkey in JNDI for download via GetKeyAction - private String jndiArtifactSync; // store in JNDI to support availability mode change + private String jndiSiteAvailabilities; private Thread availabilityCheck; - private Thread artifactSyncThread; + + private String jndiDataNodeSizeSync; // store in JNDI to support availability mode change + private Thread dataNodeSizeSyncThread; public VaultInitAction() { super(); @@ -466,44 +468,44 @@ private void initBackgroundWorkers() { artifactDAO.setConfig(iterprops); terminateBackgroundWorkers(); - ArtifactSync async = new ArtifactSync(hsDAO, artifactDAO, storageNamespace); - this.artifactSyncThread = new Thread(async); - artifactSyncThread.setDaemon(true); - artifactSyncThread.start(); + DataNodeSizeSync async = new DataNodeSizeSync(hsDAO, artifactDAO, storageNamespace); + this.dataNodeSizeSyncThread = new Thread(async); + dataNodeSizeSyncThread.setDaemon(true); + dataNodeSizeSyncThread.start(); // store in JNDI so availability can set offline - String jndiArtifactSync = appName + "-" + ArtifactSync.class.getName(); + this.jndiDataNodeSizeSync = appName + "-" + DataNodeSizeSync.class.getName(); InitialContext ctx = new InitialContext(); try { - ctx.unbind(jndiArtifactSync); + ctx.unbind(jndiDataNodeSizeSync); } catch (NamingException ignore) { log.debug("unbind previous JNDI key (" + jndiPreauthKeys + ") failed... ignoring"); } - ctx.bind(jndiArtifactSync, async); - log.info("initBackgroundWorkers: created JNDI key: " + jndiArtifactSync); + ctx.bind(jndiDataNodeSizeSync, async); + log.info("initBackgroundWorkers: created JNDI key: " + jndiDataNodeSizeSync); } catch (Exception ex) { throw new RuntimeException("check/init ArtifactSync failed", ex); } } private void terminateBackgroundWorkers() { - if (this.artifactSyncThread != null) { + if (this.dataNodeSizeSyncThread != null) { try { - log.info("terminating ArtifactSync Thread..."); - this.artifactSyncThread.interrupt(); - this.artifactSyncThread.join(); - log.info("terminating ArtifactSync Thread... [OK]"); + log.info("terminating " + DataNodeSizeSync.class.getSimpleName() + " Thread..."); + this.dataNodeSizeSyncThread.interrupt(); + this.dataNodeSizeSyncThread.join(); + log.info("terminating " + DataNodeSizeSync.class.getSimpleName() + " Thread... [OK]"); } catch (Throwable t) { - log.info("failed to terminate ArtifactSync thread", t); + log.info("failed to terminate " + DataNodeSizeSync.class.getSimpleName() + " thread", t); } finally { - this.artifactSyncThread = null; + this.dataNodeSizeSyncThread = null; } try { InitialContext initialContext = new InitialContext(); - initialContext.unbind(this.jndiArtifactSync); + initialContext.unbind(this.jndiDataNodeSizeSync); } catch (NamingException e) { - log.debug(String.format("unable to unbind %s - %s", this.jndiArtifactSync, e.getMessage())); + log.debug(String.format("unable to unbind %s - %s", this.jndiDataNodeSizeSync, e.getMessage())); } } } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java similarity index 95% rename from vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java rename to vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 023a2d2b4..2bee54116 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/ArtifactSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -76,7 +76,7 @@ import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.HarvestState; import org.opencadc.inventory.db.HarvestStateDAO; -import org.opencadc.vospace.db.ArtifactSyncWorker; +import org.opencadc.vospace.db.DataNodeSizeWorker; /** * Main artifact-sync agent that enables incremental sync of Artifact @@ -84,8 +84,8 @@ * * @author pdowler */ -public class ArtifactSync implements Runnable { - private static final Logger log = Logger.getLogger(ArtifactSync.class); +public class DataNodeSizeSync implements Runnable { + private static final Logger log = Logger.getLogger(DataNodeSizeSync.class); private static final long ROUNDS = 6000L; // 6 sec private static final long SHORT_SLEEP = 5 * ROUNDS; @@ -101,7 +101,7 @@ public class ArtifactSync implements Runnable { private boolean offline = false; - public ArtifactSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { + public DataNodeSizeSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Namespace artifactNamespace) { this.stateDAO = stateDAO; this.artifactDAO = artifactDAO; this.artifactNamespace = artifactNamespace; @@ -150,7 +150,7 @@ public void run() { log.debug("leader: " + state); boolean fail = false; try { - ArtifactSyncWorker worker = new ArtifactSyncWorker(stateDAO, state, artifactDAO, artifactNamespace); + DataNodeSizeWorker worker = new DataNodeSizeWorker(stateDAO, state, artifactDAO, artifactNamespace); worker.run(); } catch (Exception ex) { log.error("unexpected worker fail", ex); From 5bb6d74f9529139f3628c172725b3525ee8b1654 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 10:00:35 -0700 Subject: [PATCH 171/186] fix open connection leak in HarvestStateDAO table maintenance code --- .../opencadc/inventory/db/HarvestStateDAO.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java index 5df563354..a0f8d8f74 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java @@ -68,6 +68,7 @@ package org.opencadc.inventory.db; import java.net.URI; +import java.sql.Connection; import java.sql.SQLException; import java.util.UUID; import org.apache.log4j.Logger; @@ -194,13 +195,15 @@ public void put(HarvestState val, boolean forceTimestampUpdate) { if (curMaintCount == maintCount) { String sql = "VACUUM " + gen.getTable(HarvestState.class); log.warn("maintenance: " + curMaintCount + "==" + maintCount + " " + sql); - //JdbcTemplate jdbc = new JdbcTemplate(dataSource); - //jdbc.execute(sql); try { - dataSource.getConnection().createStatement().execute(sql); - } catch (SQLException ex) { - log.error("ERROR: " + sql + " FAILED", ex); - // yes, log and proceed + try (Connection c = dataSource.getConnection()) { + c.createStatement().execute(sql); + } catch (SQLException ex) { + log.error("maintenance failed: " + sql, ex); + // yes, log and proceed + } // auto-close to return to pool + } catch (Exception ex) { + log.error("failed to close connection after maintenance: " + sql, ex); } curMaintCount = 0; } else { From 209a95eb5e41108ab0f4e71bdb71575ed78fb1ef Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 10:10:17 -0700 Subject: [PATCH 172/186] import cleanup --- vault/src/main/java/org/opencadc/vault/ServiceAvailability.java | 1 - 1 file changed, 1 deletion(-) diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index 6e22d8c83..b1e53f109 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -71,7 +71,6 @@ import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.vosi.Availability; import ca.nrc.cadc.vosi.AvailabilityPlugin; - import ca.nrc.cadc.vosi.avail.CheckDataSource; import javax.naming.InitialContext; import javax.naming.NamingException; From af92038185258d5dbb0f8445c2ef5107afbca5b8 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 10:50:10 -0700 Subject: [PATCH 173/186] temporarily remove ringhold from CI --- .github/workflows/gradle.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 25d39bd68..64748cf5b 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -66,19 +66,11 @@ jobs: - name: java build -- raven run: cd raven && ../gradlew --info clean build javadoc checkstyleMain - - - name: java build -- ringhold - run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain - name: java build -- ratik run: cd ratik && ../gradlew --info clean build javadoc checkstyleMain -## TODO: docker build depends on cadc-tomcat base image from docker-base.git -# - name: docker build -- baldur -# run: cd baldur && docker build . --file Dockerfile --tag baldur:$(date +%s) -# - name: docker build -- minoc -# run: cd minoc && docker build . --file Dockerfile --tag minoc:$(date +%s) -# - name: docker build -- luskan -# run: cd luskan && docker build . --file Dockerfile --tag luskan:$(date +%s) -# - name: docker build -- raven -# run: cd raven && docker build . --file Dockerfile --tag raven:$(date +%s) +## disabled until updated for cadc-inventory-db API changes +# - name: java build -- ringhold +# run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain + From f737deecf57e6156c61dd7d3cde4b2d54f9c7353 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 11:15:33 -0700 Subject: [PATCH 174/186] vault code cleanup --- .../java/org/opencadc/vault/metadata/DataNodeSizeSync.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 2bee54116..9b917b45d 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -106,10 +106,6 @@ public DataNodeSizeSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Names this.artifactDAO = artifactDAO; this.artifactNamespace = artifactNamespace; - // fenwick setup for production workload: - //dao.setUpdateBufferCount(99); // buffer 99 updates, do every 100 - //dao.setMaintCount(999); // buffer 999 so every 1000 real updates aka every 1e5 events - // we need continuous timestamp updates to retain leader status, so only schedule maintenance stateDAO.setUpdateBufferCount(0); stateDAO.setMaintCount(9999); // every 1e4 @@ -117,7 +113,6 @@ public DataNodeSizeSync(HarvestStateDAO stateDAO, ArtifactDAO artifactDAO, Names public void setOffline(boolean offline) { this.offline = offline; - } @Override From c2c650b3f9ac3cf44dcc10cbc4359d3b7235d056 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 12:52:03 -0700 Subject: [PATCH 175/186] DataNodeSizeSync logging in machine readable format move info logging from DataNodeSizeWorker to DataNodeSizeSync --- .../vospace/db/DataNodeSizeWorker.java | 16 ++++++++--- .../vault/metadata/DataNodeSizeSync.java | 27 ++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index 0b40f8484..aeabf6e36 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -94,6 +94,8 @@ public class DataNodeSizeWorker implements Runnable { private final ArtifactDAO artifactDAO; private final HarvestStateDAO harvestStateDAO; private final Namespace storageNamespace; + + private long numArtifactsProcessed; /** * Worker constructor. @@ -112,16 +114,21 @@ public DataNodeSizeWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestS this.storageNamespace = namespace; } + public long getNumArtifactsProcessed() { + return numArtifactsProcessed; + } + @Override public void run() { + this.numArtifactsProcessed = 0L; String opName = DataNodeSizeWorker.class.getSimpleName() + ".artifactQuery"; DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); if (harvestState.curLastModified != null) { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " start=" + df.format(harvestState.curLastModified)); } else { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID); } @@ -159,17 +166,18 @@ public void run() { harvestState.curLastModified = artifact.getLastModified(); harvestState.curID = artifact.getID(); harvestStateDAO.put(harvestState); + numArtifactsProcessed++; } } catch (IOException ex) { log.error("Error closing iterator", ex); throw new RuntimeException("error while closing ResourceIterator", ex); } if (harvestState.curLastModified != null) { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=" + df.format(harvestState.curLastModified)); } else { - log.info(opName + " source=" + harvestState.getResourceID() + log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID + " end=true"); } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 9b917b45d..8abf2b8a1 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -138,15 +138,25 @@ public void run() { log.debug("created: " + state); } + long t1 = System.currentTimeMillis(); + BackgroundLogInfo logInfo = new BackgroundLogInfo(instanceID.toString()); + logInfo.setSuccess(false); + // determine leader boolean leader = checkLeaderStatus(state); - + logInfo.leader = leader; + + log.info(logInfo.start()); + long sleep = LONG_SLEEP; // default for not leader if (leader) { log.debug("leader: " + state); boolean fail = false; try { DataNodeSizeWorker worker = new DataNodeSizeWorker(stateDAO, state, artifactDAO, artifactNamespace); worker.run(); + logInfo.setLastModified(state.curLastModified); + logInfo.processed = worker.getNumArtifactsProcessed(); + logInfo.setSuccess(true); } catch (Exception ex) { log.error("unexpected worker fail", ex); fail = true; @@ -173,16 +183,19 @@ public void run() { } if (fail) { - log.debug("failed leader " + state.instanceID + " sleep=" + FAIL_SLEEP); - Thread.sleep(FAIL_SLEEP); + sleep = FAIL_SLEEP; } else { - log.debug("idle leader " + state.instanceID + " sleep=" + SHORT_SLEEP); - Thread.sleep(SHORT_SLEEP); + sleep = SHORT_SLEEP; } + logInfo.setSuccess(!fail); + logInfo.setElapsedTime(System.currentTimeMillis() - t1); } else { - log.debug("not leader: sleep=" + LONG_SLEEP); - Thread.sleep(LONG_SLEEP); + // not leader success + logInfo.setSuccess(true); } + logInfo.sleep = sleep; + log.info(logInfo.end()); + Thread.sleep(sleep); } } catch (InterruptedException ex) { log.debug("interrupted - assuming shutdown", ex); From 0c4c36ec0f575e5bf5d8773ea1c23e3a59fdf4b7 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 25 Mar 2024 13:38:13 -0700 Subject: [PATCH 176/186] vault: machine-readable logging in DataNodeSizeSync incremental lookback in DataNodeSizeWorker --- .../vospace/db/DataNodeSizeWorker.java | 32 +++++- .../vault/metadata/BackgroundLogInfo.java | 106 ++++++++++++++++++ .../vault/metadata/DataNodeSizeSync.java | 9 +- 3 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index aeabf6e36..236da5ca1 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -72,6 +72,7 @@ import ca.nrc.cadc.io.ResourceIterator; import java.io.IOException; import java.text.DateFormat; +import java.util.Date; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.Namespace; @@ -89,6 +90,10 @@ public class DataNodeSizeWorker implements Runnable { private static final Logger log = Logger.getLogger(DataNodeSizeWorker.class); + // lookback when doing incremental harvest because head of sequence is + // not monotonic over short timescales (events arrive out of sequence) + private static final long LOOKBACK_TIME_MS = 60 * 1000L; + private final HarvestState harvestState; private final NodeDAO nodeDAO; private final ArtifactDAO artifactDAO; @@ -132,12 +137,21 @@ public void run() { + " instance=" + harvestState.instanceID); } + final Date now = new Date(); + final Date lookBack = new Date(now.getTime() - LOOKBACK_TIME_MS); + Date startTime = getQueryLowerBound(lookBack, harvestState.curLastModified); + if (lookBack != null && harvestState.curLastModified != null) { + log.debug("lookBack=" + df.format(lookBack) + " curLastModified=" + df.format(harvestState.curLastModified) + + " -> " + df.format(startTime)); + } + String uriBucket = null; // process all artifacts in a single thread - try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, harvestState.curLastModified, true)) { + try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, startTime, true)) { TransactionManager tm = nodeDAO.getTransactionManager(); while (iter.hasNext()) { Artifact artifact = iter.next(); DataNode node = nodeDAO.getDataNode(artifact.getURI()); + log.debug(artifact.getURI() + " len=" + artifact.getContentLength() + " -> " + node.getName()); if (node != null && !artifact.getContentLength().equals(node.bytesUsed)) { tm.startTransaction(); try { @@ -182,4 +196,20 @@ public void run() { + " end=true"); } } + + private Date getQueryLowerBound(Date lookBack, Date lastModified) { + if (lookBack == null) { + // feature not enabled + return lastModified; + } + if (lastModified == null) { + // first harvest + return null; + } + if (lookBack.before(lastModified)) { + return lookBack; + } + return lastModified; + + } } diff --git a/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java b/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java new file mode 100644 index 000000000..88ac11a93 --- /dev/null +++ b/vault/src/main/java/org/opencadc/vault/metadata/BackgroundLogInfo.java @@ -0,0 +1,106 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2024. (c) 2024. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.vault.metadata; + +import ca.nrc.cadc.date.DateUtil; +import ca.nrc.cadc.log.WebServiceLogInfo; +import java.text.DateFormat; +import java.util.Date; +import org.apache.log4j.Logger; + +/** + * Log structure for background threads. + * + * @author pdowler + */ +public class BackgroundLogInfo extends WebServiceLogInfo { + private static final Logger log = Logger.getLogger(BackgroundLogInfo.class); + + public Boolean leader; + public String instance; + public String lastmodified; + public Long processed; + public Long sleep; + + + public BackgroundLogInfo(String instance) { + super.serviceName = "vault"; + this.instance = instance; + } + + public void setOperation(String op) { + super.method = op; + } + + public void setLastModified(Date ts) { + DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); + this.lastmodified = df.format(ts); + } + + +} diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index 8abf2b8a1..a63eb83c9 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -140,17 +140,18 @@ public void run() { long t1 = System.currentTimeMillis(); BackgroundLogInfo logInfo = new BackgroundLogInfo(instanceID.toString()); + logInfo.setOperation(DataNodeSizeWorker.class.getSimpleName()); logInfo.setSuccess(false); // determine leader boolean leader = checkLeaderStatus(state); logInfo.leader = leader; - - log.info(logInfo.start()); - long sleep = LONG_SLEEP; // default for not leader + logInfo.setLastModified(state.curLastModified); + long sleep = LONG_SLEEP; if (leader) { log.debug("leader: " + state); boolean fail = false; + log.info(logInfo.start()); try { DataNodeSizeWorker worker = new DataNodeSizeWorker(stateDAO, state, artifactDAO, artifactNamespace); worker.run(); @@ -191,6 +192,7 @@ public void run() { logInfo.setElapsedTime(System.currentTimeMillis() - t1); } else { // not leader success + sleep = LONG_SLEEP; logInfo.setSuccess(true); } logInfo.sleep = sleep; @@ -215,7 +217,6 @@ private boolean checkLeaderStatus(HarvestState state) { log.info("EVICTING " + state.instanceID + " because age " + age + " > " + EVICT_AGE); state.instanceID = instanceID; stateDAO.put(state); - state = stateDAO.get(state.getID()); leader = true; } } From 7aa2a807c2058cabb980c9566e376fb80a4d341c Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Tue, 26 Mar 2024 08:59:33 -0700 Subject: [PATCH 177/186] CADC-13234 update ringhold README for new config properties --- ringhold/README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ringhold/README.md b/ringhold/README.md index 9f23c1b8f..ea36b55cd 100644 --- a/ringhold/README.md +++ b/ringhold/README.md @@ -1,6 +1,6 @@ -# Storage Inventory local artifact removal process (ringhold) +# Storage Inventory local artifact deletion process (ringhold) -Process to remove local artifacts that are no longer being synchronised by fenwick. This tool is used +Process to delete local artifacts that are no longer being synchronised by fenwick. This tool is used to perform quick cleanup at a storage site after changing the fenwick artifact-filter policy. ## configuration @@ -18,17 +18,25 @@ org.opencadc.ringhold.inventory.schema={schema for inventory database objects} org.opencadc.ringhold.inventory.username={username for inventory admin} org.opencadc.ringhold.inventory.password={password for inventory admin} org.opencadc.ringhold.inventory.url=jdbc:postgresql://{server}/{database} + +# artifact namespace +org.opencadc.ringhold.namespace={storage site namespace} + +# artifact uri bucket filter (optional) +org.opencadc.ringhold.buckets={uriBucket prefix or range of prefixes} ``` The `inventory` account owns and manages all the content (insert, update, delete) in the inventory schema. Unlike other components that modify inventory content, this component **does not initialise** the database objects because it never makes sense to run this in a new/empty database. The database is specified in the JDBC URL. Failure to connect to a pre-initialised database will show up in logs. -### artifact-deselector.sql -Contains a SQL clause used as a WHERE constraint. The clause returns Artifact's that match the URI pattern. -``` -WHERE uri LIKE 'cadc:CFHT/%' -``` +The `namespace` is the prefix of the Artifact URI's to be deleted. The `namespace` must end with a colon (:) +or slash (/) so one namespace cannot accidentally match (be a prefix of) another namespace. Multiple values +of `namespace` may be specified, one per line. + +The `buckets` value indicates a subset of artifacts to delete. The range of uri bucket prefixes is specified +with two values separated by a single - (dash) character; whitespace is ignored. Multiple instances of `ringhold` +can be run (in parallel) to subdivide the work as long as the range of buckets do not overlap. ## building it ``` From f6bb3790d1d8e0d103587c360958cacd27993feb Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Tue, 26 Mar 2024 11:14:13 -0700 Subject: [PATCH 178/186] CADC-13234 update README after review --- ringhold/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ringhold/README.md b/ringhold/README.md index ea36b55cd..6c919fde2 100644 --- a/ringhold/README.md +++ b/ringhold/README.md @@ -1,7 +1,8 @@ # Storage Inventory local artifact deletion process (ringhold) -Process to delete local artifacts that are no longer being synchronised by fenwick. This tool is used -to perform quick cleanup at a storage site after changing the fenwick artifact-filter policy. +Process to remove the local copy of artifacts from a storage site inventory database and +generate DeletedStorageLocationEvent(s) so the removal will propagate correctly to a global inventory. +This does not remove the files from storage (see `tantar`). ## configuration See the [cadc-java](https://github.com/opencadc/docker-base/tree/master/cadc-java) image docs for general config requirements. @@ -19,7 +20,7 @@ org.opencadc.ringhold.inventory.username={username for inventory admin} org.opencadc.ringhold.inventory.password={password for inventory admin} org.opencadc.ringhold.inventory.url=jdbc:postgresql://{server}/{database} -# artifact namespace +# artifact namespace(s) to remove org.opencadc.ringhold.namespace={storage site namespace} # artifact uri bucket filter (optional) From 419ef8306cdfbe8818afcb21b290864ed209bdfa Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 26 Mar 2024 14:19:01 -0700 Subject: [PATCH 179/186] vault: cleanup and doc update --- vault/README.md | 37 ++++------ .../opencadc/vault/ServiceAvailability.java | 8 +-- .../org/opencadc/vault/VaultInitAction.java | 29 +++++--- vault/src/main/webapp/WEB-INF/web.xml | 70 +++++++++---------- 4 files changed, 74 insertions(+), 70 deletions(-) diff --git a/vault/README.md b/vault/README.md index 153232129..f2ef10863 100644 --- a/vault/README.md +++ b/vault/README.md @@ -41,7 +41,6 @@ org.opencadc.vault.nodes.password={password for vospace pool} org.opencadc.vault.nodes.url=jdbc:postgresql://{server}/{database} org.opencadc.vault.inventory.maxActive={max connections for inventory pool} -# optional: config for separate inventory pool org.opencadc.vault.inventory.username={username for inventory pool} org.opencadc.vault.inventory.password={password for inventory pool} org.opencadc.vault.inventory.url=jdbc:postgresql://{server}/{database} @@ -58,13 +57,12 @@ VOSI-availability output. The _inventory_ account owns and manages (create, alter, drop) inventory database objects and manages all the content (update and delete Artifact, insert DeletedArtifactEvent). The database is specified -in the JDBC URL and the schema name is specified in the minoc.properties (below). Failure to connect or +in the JDBC URL and the schema name is specified in the vault.properties (below). Failure to connect or initialize the database will show up in logs and in the VOSI-availability output. The _inventory_ content may be in the same database as the _nodes_, in a different database in the same server, or in a different -server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. Note: it is a good -idea to set `maxActive` to a valid integer (e.g. 1 because the tomcat connection pool doesn't like 0 and -decides to make it 100 instead) when using a single pool; this avoids an ugly but meaningless stack trace -in the logs at startup. +server entirely. See `org.opencadc.vault.singlePool` below for the pros and cons. The _inventory_ pool must +be functional for initialization, availability checks (`maxActive` = 1 with `singlePool` is sufficient), and +the connection information is re-used by an internal background thread that synchronizes data node sizes. The _uws_ account owns and manages (create, alter, drop) uws database objects in the `uws` schema and manages all the content (insert, update, delete). The database is specified in the JDBC URLFailure to connect or initialize the @@ -128,28 +126,21 @@ DeletedArtifactEvent are done in a separate transaction and if that fails the Ar orphaned until the vault validation (see ???) runs and fixes such a discrepancy. However, _singlePool_ = `false` allows the content to be stored in two separate databases or servers. -The _root.owner_ owns the root node and has full read and write permission in the root container, so it can -create and delete container nodes at the root and assign container node properties that are normally read-only -to normal users: owner, quota, etc. This must be set to the username of the admin. +The _root.owner_ key configures the owner of the root node; the owner and has full read and write permission +in the root container, so it can create and delete container nodes at the root and assign container node properties +that are normally read-only to normal users: owner, quota, etc. This must be set to the username of the admin. -The _storage.namespace_ configures `vault` to use the specified namespace in storage-inventory to store files. +The _storage.namespace_ key configures `vault` to use the specified namespace in storage-inventory to store files. This only applies to new data nodes that are created and will not effect previously created nodes and artifacts. Probably don't want to change this... prevent change? TBD. -### vault-availability.properties (optional) +### cadc-log.properties (optional) +See cadc-log for common +dynamic logging control. -The vault-availability.properties file specifies which users have the authority to change the availability state of -the vault service. Each entry consists of a key=value pair. The key is always "users". The value is the x500 canonical -user name. - -Example: -``` -users = {user identity} -``` -`users` specifies the user(s) who are authorized to make calls to the service. The value is a list of user -identities (X500 distingushed name), one line per user. Optional: if the `vault-availability.properties` is -not found or does not list any `users`, the service will function in the default mode (ReadWrite) and the -state will not be changeable. +### cadc-vosi.properties (optional) +See cadc-vosi for common +service state control. ## building it ``` diff --git a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java index b1e53f109..5a76fb1bc 100644 --- a/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java +++ b/vault/src/main/java/org/opencadc/vault/ServiceAvailability.java @@ -201,15 +201,15 @@ private String getState() { } return ret; } - + private void setOffline(boolean offline) { - String jndiArtifactSync = appName + "-" + DataNodeSizeSync.class.getName(); + String jndiKey = appName + "-" + DataNodeSizeSync.class.getName(); try { InitialContext initialContext = new InitialContext(); - DataNodeSizeSync async = (DataNodeSizeSync) initialContext.lookup(jndiArtifactSync); + DataNodeSizeSync async = (DataNodeSizeSync) initialContext.lookup(jndiKey); async.setOffline(offline); } catch (NamingException e) { - log.debug(String.format("unable to unbind %s - %s", jndiArtifactSync, e.getMessage())); + log.debug(String.format("unable to find %s - %s", jndiKey, e.getMessage())); } } } diff --git a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java index b61f34705..a80f04526 100644 --- a/vault/src/main/java/org/opencadc/vault/VaultInitAction.java +++ b/vault/src/main/java/org/opencadc/vault/VaultInitAction.java @@ -69,6 +69,7 @@ import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.rest.InitAction; +import ca.nrc.cadc.rest.RestAction; import ca.nrc.cadc.util.InvalidConfigException; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; @@ -131,12 +132,11 @@ public class VaultInitAction extends InitAction { private Namespace storageNamespace; private Map vosDaoConfig; private Map invDaoConfig; - private List allocationParents = new ArrayList<>(); - private String jndiNodePersistence; + private String jndiNodePersistence; // store in JNDI for cadc-vos-server lib private String jndiPreauthKeys; // store pubkey in JNDI for download via GetKeyAction - private String jndiSiteAvailabilities; + private String jndiSiteAvailabilities; // store in JNDI to share with ProtocolsGenerator private Thread availabilityCheck; private String jndiDataNodeSizeSync; // store in JNDI to support availability mode change @@ -149,8 +149,9 @@ public VaultInitAction() { @Override public void doInit() { initConfig(); - initDatabase(); - initUWSDatabase(); + initDatabaseVOS(); + initDatabaseINV(); + initDatabaseUWS(); initNodePersistence(); initKeyPair(); initAvailabilityCheck(); @@ -324,7 +325,7 @@ private void initConfig() { } } - private void initDatabase() { + private void initDatabaseVOS() { try { String dsname = (String) vosDaoConfig.get("jndiDataSourceName"); String schema = (String) vosDaoConfig.get("vosSchema"); @@ -336,7 +337,9 @@ private void initDatabase() { } catch (Exception ex) { throw new IllegalStateException("check/init vospace database failed", ex); } - + } + + private void initDatabaseINV() { try { String dsname = (String) invDaoConfig.get("jndiDataSourceName"); String schema = (String) invDaoConfig.get("invSchema"); @@ -350,7 +353,7 @@ private void initDatabase() { } } - private void initUWSDatabase() { + private void initDatabaseUWS() { try { log.info("initDatabase: " + JNDI_UWS_DATASOURCE + " uws START"); DataSource uws = DBUtil.findJNDIDataSource(JNDI_UWS_DATASOURCE); @@ -466,9 +469,19 @@ private void initBackgroundWorkers() { Map iterprops = getIteratorConfig(props); log.warn("iterator pool: " + iterprops.get("jndiDataSourceName")); artifactDAO.setConfig(iterprops); + + // determine startup mode + boolean offline = false; // normal + String key = appName + RestAction.STATE_MODE_KEY; + String ret = System.getProperty(key); + if (ret != null + && (RestAction.STATE_READ_ONLY.equals(ret) || RestAction.STATE_OFFLINE.equals(ret))) { + offline = true; + } terminateBackgroundWorkers(); DataNodeSizeSync async = new DataNodeSizeSync(hsDAO, artifactDAO, storageNamespace); + async.setOffline(offline); this.dataNodeSizeSyncThread = new Thread(async); dataNodeSizeSyncThread.setDaemon(true); dataNodeSizeSyncThread.start(); diff --git a/vault/src/main/webapp/WEB-INF/web.xml b/vault/src/main/webapp/WEB-INF/web.xml index fffacd300..f57bad3bf 100644 --- a/vault/src/main/webapp/WEB-INF/web.xml +++ b/vault/src/main/webapp/WEB-INF/web.xml @@ -37,6 +37,40 @@ 1 + + + AvailabilityServlet + ca.nrc.cadc.vosi.AvailabilityServlet + + ca.nrc.cadc.vosi.AvailabilityPlugin + org.opencadc.vault.ServiceAvailability + + 2 + + + + + CapabilitiesServlet + ca.nrc.cadc.rest.RestServlet + + init + ca.nrc.cadc.vosi.CapInitAction + + + head + ca.nrc.cadc.vosi.CapHeadAction + + + get + ca.nrc.cadc.vosi.CapGetAction + + + input + /capabilities.xml + + 2 + + NodesServlet ca.nrc.cadc.rest.RestServlet @@ -64,7 +98,7 @@ delete org.opencadc.vospace.server.actions.DeleteNodeAction - 2 + 3 @@ -208,40 +242,6 @@ - - - CapabilitiesServlet - ca.nrc.cadc.rest.RestServlet - - init - ca.nrc.cadc.vosi.CapInitAction - - - head - ca.nrc.cadc.vosi.CapHeadAction - - - get - ca.nrc.cadc.vosi.CapGetAction - - - input - /capabilities.xml - - 3 - - - - - AvailabilityServlet - ca.nrc.cadc.vosi.AvailabilityServlet - - ca.nrc.cadc.vosi.AvailabilityPlugin - org.opencadc.vault.ServiceAvailability - - 3 - - From 15b86dd22b10145f8457d2388a8ce9b63fb5beb6 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 28 Mar 2024 12:35:36 -0700 Subject: [PATCH 180/186] vault: improve server header info for files endpoint --- .../main/java/org/opencadc/vault/files/HeadAction.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java index 69b68f00a..902c4c9a6 100644 --- a/vault/src/main/java/org/opencadc/vault/files/HeadAction.java +++ b/vault/src/main/java/org/opencadc/vault/files/HeadAction.java @@ -70,6 +70,7 @@ import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.rest.RestAction; +import ca.nrc.cadc.rest.Version; import java.net.URI; import java.net.URISyntaxException; import java.util.Date; @@ -107,6 +108,13 @@ protected final InlineContentHandler getInlineContentHandler() { return null; } + @Override + protected String getServerImpl() { + // no null version checking because fail to build correctly can't get past basic testing + Version v = getVersionFromResource(); + return "storage-inventory/vault-" + v.getMajorMinor(); + } + @Override public void initAction() throws Exception { String jndiNodePersistence = super.appName + "-" + NodePersistence.class.getName(); From b53037b22bc3bf1009e44ec4ad5db8c0e17ea767 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 28 Mar 2024 13:56:10 -0700 Subject: [PATCH 181/186] CADC-13234 update ringhold to query with configured namespace and uri buckets instead of select clause. --- ringhold/VERSION | 2 +- .../ringhold/InventoryValidatorTest.java | 325 ++++++++++-------- .../opencadc/ringhold/ArtifactDeselector.java | 3 + .../opencadc/ringhold/InventoryValidator.java | 88 +++-- .../main/java/org/opencadc/ringhold/Main.java | 25 +- 5 files changed, 275 insertions(+), 168 deletions(-) diff --git a/ringhold/VERSION b/ringhold/VERSION index 51807fa89..9229240a7 100644 --- a/ringhold/VERSION +++ b/ringhold/VERSION @@ -1,4 +1,4 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -TAGS="0.2-$(date --utc +"%Y%m%dT%H%M%S")" +TAGS="0.3-$(date --utc +"%Y%m%dT%H%M%S")" diff --git a/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java b/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java index 0a809c6fb..47e963b1d 100644 --- a/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java +++ b/ringhold/src/intTest/java/org/opencadc/ringhold/InventoryValidatorTest.java @@ -73,6 +73,7 @@ import ca.nrc.cadc.db.DBConfig; import ca.nrc.cadc.db.DBUtil; import ca.nrc.cadc.db.version.InitDatabase; +import ca.nrc.cadc.util.BucketSelector; import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.HexUtil; import ca.nrc.cadc.util.Log4jInit; @@ -84,20 +85,17 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Date; -import java.util.Map; -import java.util.MissingResourceException; -import java.util.Properties; -import java.util.TreeMap; -import java.util.UUID; +import java.util.*; import javax.sql.DataSource; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedStorageLocationEvent; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.StorageLocation; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedStorageLocationEventDAO; @@ -108,7 +106,7 @@ /** * Various versions of: * Insert artifacts more than uri pattern - * Run tool with one uri deselector + * Run tool with different Namespaces specified * Confirm delete storage location event creation and absence of artifacts in inventory */ public class InventoryValidatorTest { @@ -119,7 +117,7 @@ public class InventoryValidatorTest { Log4jInit.setLevel("org.opencadc.inventory", Level.INFO); Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO); Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); - Log4jInit.setLevel("org.opencadc.ringhold", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.ringhold", Level.INFO); } static String INVENTORY_SERVER = "RINGHOLD_TEST"; @@ -188,80 +186,12 @@ public InventoryValidatorTest() throws Exception { @Before public void setup() throws Exception { - writeConfig(); truncateTables(); } @Test - public void missingConfigTest() throws Exception { - final Path includePath = new File(TMP_DIR + "/config").toPath(); - Files.createDirectories(includePath); - final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); - boolean deleted = includeFile.delete(); - Assert.assertTrue("include file not deleted", deleted); - - configTest(); - } - - @Test - public void emptyConfigTest() throws Exception { - final Path includePath = new File(TMP_DIR + "/config").toPath(); - Files.createDirectories(includePath); - final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); - - final FileWriter fileWriter = new FileWriter(includeFile); - fileWriter.write(""); - fileWriter.flush(); - fileWriter.close(); - - configTest(); - } - - @Test - public void onlyCommentsConfigTest() throws Exception { - final Path includePath = new File(TMP_DIR + "/config").toPath(); - Files.createDirectories(includePath); - final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); - - final FileWriter fileWriter = new FileWriter(includeFile); - fileWriter.write("# WHERE uri LIKE 'cadc:INTTEST/%'"); - fileWriter.flush(); - fileWriter.close(); - - configTest(); - } - - @Test - public void doesNotStartWithWhereConfigTest() throws Exception { - final Path includePath = new File(TMP_DIR + "/config").toPath(); - Files.createDirectories(includePath); - final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); - - final FileWriter fileWriter = new FileWriter(includeFile); - fileWriter.write("uri LIKE 'cadc:INTTEST/%'\r\n"); - fileWriter.flush(); - fileWriter.close(); - - configTest(); - } - - @Test - public void multipleWhereConfigTest() throws Exception { - final Path includePath = new File(TMP_DIR + "/config").toPath(); - Files.createDirectories(includePath); - final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); - - final FileWriter fileWriter = new FileWriter(includeFile); - fileWriter.write("WHERE uri LIKE 'cadc:INTTEST/%'\r\n"); - fileWriter.write("WHERE uri LIKE 'cadc:TEST/%'"); - fileWriter.flush(); - fileWriter.close(); - - configTest(); - } - - public void configTest() { - StorageLocation storageLocation = new StorageLocation(URI.create("ivo://cadc.nrc.ca/foo")); + public void noArtifactsMatchNamespace() throws Exception { + StorageLocation storageLocation = new StorageLocation(URI.create("cadc:foo")); Artifact a1 = getTestArtifact("cadc:TEST/one.txt"); a1.storageLocation = storageLocation; @@ -273,48 +203,11 @@ public void configTest() { a3.storageLocation = storageLocation; this.artifactDAO.put(a3); - try { - System.setProperty("user.home", TMP_DIR); - InventoryValidator testSubject = new InventoryValidator(this.daoConfig, this.daoConfig); - testSubject.run(); - Assert.fail("should throw an exception for invalid config"); - } catch (Exception expected) { - // exception expected - } finally { - System.setProperty("user.home", USER_HOME); - } - - a1 = this.artifactDAO.get(a1.getID()); - Assert.assertNotNull(a1); - a2 = this.artifactDAO.get(a2.getID()); - Assert.assertNotNull(a2); - a3 = this.artifactDAO.get(a3.getID()); - Assert.assertNotNull(a3); - - DeletedStorageLocationEvent dsle1 = this.deletedStorageLocationEventDAO.get(a1.getID()); - Assert.assertNull(dsle1); - DeletedStorageLocationEvent dsle2 = this.deletedStorageLocationEventDAO.get(a2.getID()); - Assert.assertNull(dsle2); - DeletedStorageLocationEvent dsle3 = this.deletedStorageLocationEventDAO.get(a3.getID()); - } - - @Test - public void noArtifactsMatchFilter() throws Exception { - StorageLocation storageLocation = new StorageLocation(URI.create("ivo://cadc.nrc.ca/foo")); - - Artifact a1 = getTestArtifact("cadc:TEST/one.txt"); - a1.storageLocation = storageLocation; - this.artifactDAO.put(a1); - Artifact a2 = getTestArtifact("cadc:INT/two.txt"); - a2.storageLocation = storageLocation; - this.artifactDAO.put(a2); - Artifact a3 = getTestArtifact("cadc:CADC/three.txt"); - a3.storageLocation = storageLocation; - this.artifactDAO.put(a3); + List namespaces = Collections.singletonList(new Namespace("cadc:NOMATCH/")); try { System.setProperty("user.home", TMP_DIR); - InventoryValidator testSubject = new InventoryValidator(this.daoConfig, this.daoConfig); + InventoryValidator testSubject = new InventoryValidator(daoConfig, daoConfig, namespaces, null); testSubject.run(); } finally { System.setProperty("user.home", USER_HOME); @@ -336,9 +229,9 @@ public void noArtifactsMatchFilter() throws Exception { } @Test - public void someArtifactsMatchFilter() throws Exception { - StorageLocation a_storageLocation = new StorageLocation(URI.create("ivo://cadc.nrc.ca/foo")); - StorageLocation b_storageLocation = new StorageLocation(URI.create("ivo://cadc.nrc.ca/bar")); + public void someArtifactsMatchNamespace() throws Exception { + StorageLocation a_storageLocation = new StorageLocation(URI.create("cadc:foo")); + StorageLocation b_storageLocation = new StorageLocation(URI.create("cadc:bar")); Artifact b1 = getTestArtifact("cadc:INT/one.txt"); b1.storageLocation = b_storageLocation; @@ -359,9 +252,12 @@ public void someArtifactsMatchFilter() throws Exception { b3.storageLocation = b_storageLocation; this.artifactDAO.put(b3); + List namespaces = Collections.singletonList(new Namespace("cadc:INTTEST/")); + BucketSelector buckets = new BucketSelector("0-f"); + try { System.setProperty("user.home", TMP_DIR); - InventoryValidator testSubject = new InventoryValidator(this.daoConfig, this.daoConfig); + InventoryValidator testSubject = new InventoryValidator(daoConfig, daoConfig, namespaces, buckets); testSubject.run(); } finally { System.setProperty("user.home", USER_HOME); @@ -397,51 +293,67 @@ public void someArtifactsMatchFilter() throws Exception { } @Test - public void allArtifactsMatchFilter() throws Exception { - StorageLocation storageLocation = new StorageLocation(URI.create("ivo://cadc.nrc.ca/foo")); + public void allArtifactsMatchNamespace() throws Exception { + StorageLocation a_storageLocation = new StorageLocation(URI.create("cadc:foo")); + StorageLocation b_storageLocation = new StorageLocation(URI.create("cadc:bar")); - Artifact a1 = getTestArtifact("cadc:INTTEST/one.txt"); - a1.storageLocation = storageLocation; + Artifact b1 = getTestArtifact("cadc:INT/one.txt"); + b1.storageLocation = b_storageLocation; + this.artifactDAO.put(b1); + Artifact b2 = getTestArtifact("cadc:INT_TEST/two.txt"); + b2.storageLocation = b_storageLocation; + this.artifactDAO.put(b2); + Artifact a1 = getTestArtifact("cadc:INTTEST/three.txt"); + a1.storageLocation = a_storageLocation; this.artifactDAO.put(a1); - Artifact a2 = getTestArtifact("cadc:INTTEST/two.txt"); - a2.storageLocation = storageLocation; + Artifact a2 = getTestArtifact("cadc:INTTEST/four.txt"); + a2.storageLocation = a_storageLocation; this.artifactDAO.put(a2); - Artifact a3 = getTestArtifact("cadc:INTTEST/three.txt"); - a3.storageLocation = storageLocation; + Artifact a3 = getTestArtifact("cadc:INTTEST/five.txt"); + a3.storageLocation = a_storageLocation; this.artifactDAO.put(a3); + Artifact b3 = getTestArtifact("cadc:TEST/six.txt"); + b3.storageLocation = b_storageLocation; + this.artifactDAO.put(b3); + + List namespaces = Arrays.asList(new Namespace("cadc:INT/"), + new Namespace("cadc:INT_TEST/"), new Namespace("cadc:INTTEST/"), + new Namespace("cadc:TEST/")); + BucketSelector buckets = new BucketSelector("0-f"); try { System.setProperty("user.home", TMP_DIR); - InventoryValidator testSubject = new InventoryValidator(this.daoConfig, this.daoConfig); + InventoryValidator testSubject = new InventoryValidator(daoConfig, daoConfig, namespaces, buckets); testSubject.run(); } finally { System.setProperty("user.home", USER_HOME); } + DeletedStorageLocationEvent b_dsle1 = this.deletedStorageLocationEventDAO.get(b1.getID()); + Assert.assertNotNull(b_dsle1); + DeletedStorageLocationEvent b_dsle2 = this.deletedStorageLocationEventDAO.get(b2.getID()); + Assert.assertNotNull(b_dsle2); DeletedStorageLocationEvent a_dsle1 = this.deletedStorageLocationEventDAO.get(a1.getID()); Assert.assertNotNull(a_dsle1); DeletedStorageLocationEvent a_dsle2 = this.deletedStorageLocationEventDAO.get(a2.getID()); Assert.assertNotNull(a_dsle2); DeletedStorageLocationEvent a_dsle3 = this.deletedStorageLocationEventDAO.get(a3.getID()); Assert.assertNotNull(a_dsle3); + DeletedStorageLocationEvent b_dsle3 = this.deletedStorageLocationEventDAO.get(b3.getID()); + Assert.assertNotNull(b_dsle3); + b1 = this.artifactDAO.get(b1.getID()); + Assert.assertNull(b1); + b2 = this.artifactDAO.get(b2.getID()); + Assert.assertNull(b2); a1 = this.artifactDAO.get(a1.getID()); Assert.assertNull(a1); a2 = this.artifactDAO.get(a2.getID()); Assert.assertNull(a2); a3 = this.artifactDAO.get(a3.getID()); Assert.assertNull(a3); - } - - private void writeConfig() throws IOException { - final Path includePath = new File(TMP_DIR + "/config").toPath(); - Files.createDirectories(includePath); - final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); - - final FileWriter fileWriter = new FileWriter(includeFile); - fileWriter.write("WHERE uri LIKE 'cadc:INTTEST/%'"); - fileWriter.flush(); - fileWriter.close(); + b3 = this.artifactDAO.get(b3.getID()); + Assert.assertNull(b3); } private Artifact getTestArtifact(final String uri) { @@ -450,7 +362,6 @@ private Artifact getTestArtifact(final String uri) { return new Artifact(URI.create(uri), checkSum, new Date(), 512L); } - private void truncateTables() throws Exception { final JdbcTemplate jdbcTemplate = new JdbcTemplate(DBUtil.findJNDIDataSource(jndiPath)); jdbcTemplate.execute("TRUNCATE TABLE " + INVENTORY_SCHEMA + ".deletedArtifactEvent"); @@ -461,4 +372,134 @@ private void truncateTables() throws Exception { jdbcTemplate.execute("TRUNCATE TABLE " + INVENTORY_SCHEMA + ".Artifact"); } + + // below are tests for the ArtifactDeselector, which is not currently used, + // but preserved in case one day it is again. + + @Ignore + @Test + public void missingConfigTest() throws Exception { + final Path includePath = new File(TMP_DIR + "/config").toPath(); + Files.createDirectories(includePath); + final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); + boolean deleted = includeFile.delete(); + Assert.assertTrue("include file not deleted", deleted); + + configTest(); + } + + @Ignore + @Test + public void emptyConfigTest() throws Exception { + final Path includePath = new File(TMP_DIR + "/config").toPath(); + Files.createDirectories(includePath); + final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); + + final FileWriter fileWriter = new FileWriter(includeFile); + fileWriter.write(""); + fileWriter.flush(); + fileWriter.close(); + + configTest(); + } + + @Ignore + @Test + public void onlyCommentsConfigTest() throws Exception { + final Path includePath = new File(TMP_DIR + "/config").toPath(); + Files.createDirectories(includePath); + final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); + + final FileWriter fileWriter = new FileWriter(includeFile); + fileWriter.write("# WHERE uri LIKE 'cadc:INTTEST/%'"); + fileWriter.flush(); + fileWriter.close(); + + configTest(); + } + + @Ignore + @Test + public void doesNotStartWithWhereConfigTest() throws Exception { + final Path includePath = new File(TMP_DIR + "/config").toPath(); + Files.createDirectories(includePath); + final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); + + final FileWriter fileWriter = new FileWriter(includeFile); + fileWriter.write("uri LIKE 'cadc:INTTEST/%'\r\n"); + fileWriter.flush(); + fileWriter.close(); + + configTest(); + } + + @Ignore + @Test + public void multipleWhereConfigTest() throws Exception { + final Path includePath = new File(TMP_DIR + "/config").toPath(); + Files.createDirectories(includePath); + final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); + + final FileWriter fileWriter = new FileWriter(includeFile); + fileWriter.write("WHERE uri LIKE 'cadc:INTTEST/%'\r\n"); + fileWriter.write("WHERE uri LIKE 'cadc:TEST/%'"); + fileWriter.flush(); + fileWriter.close(); + + configTest(); + } + + @Ignore + @Test + public void configTest() { + StorageLocation storageLocation = new StorageLocation(URI.create("ivo://cadc.nrc.ca/foo")); + + Artifact a1 = getTestArtifact("cadc:TEST/one.txt"); + a1.storageLocation = storageLocation; + this.artifactDAO.put(a1); + Artifact a2 = getTestArtifact("cadc:INT/two.txt"); + a2.storageLocation = storageLocation; + this.artifactDAO.put(a2); + Artifact a3 = getTestArtifact("cadc:CADC/three.txt"); + a3.storageLocation = storageLocation; + this.artifactDAO.put(a3); + + try { + System.setProperty("user.home", TMP_DIR); + List namespaces = Collections.singletonList(new Namespace("cadc:FOO/")); + BucketSelector buckets = new BucketSelector("0-f"); + InventoryValidator testSubject = new InventoryValidator(this.daoConfig, this.daoConfig, namespaces, buckets); + testSubject.run(); + Assert.fail("should throw an exception for invalid config"); + } catch (Exception expected) { + // exception expected + } finally { + System.setProperty("user.home", USER_HOME); + } + + a1 = this.artifactDAO.get(a1.getID()); + Assert.assertNotNull(a1); + a2 = this.artifactDAO.get(a2.getID()); + Assert.assertNotNull(a2); + a3 = this.artifactDAO.get(a3.getID()); + Assert.assertNotNull(a3); + + DeletedStorageLocationEvent dsle1 = this.deletedStorageLocationEventDAO.get(a1.getID()); + Assert.assertNull(dsle1); + DeletedStorageLocationEvent dsle2 = this.deletedStorageLocationEventDAO.get(a2.getID()); + Assert.assertNull(dsle2); + DeletedStorageLocationEvent dsle3 = this.deletedStorageLocationEventDAO.get(a3.getID()); + } + + private void writeConfig() throws IOException { + final Path includePath = new File(TMP_DIR + "/config").toPath(); + Files.createDirectories(includePath); + final File includeFile = new File(includePath.toFile(), "artifact-deselector.sql"); + + final FileWriter fileWriter = new FileWriter(includeFile); + fileWriter.write("WHERE uri LIKE 'cadc:INTTEST/%'"); + fileWriter.flush(); + fileWriter.close(); + } + } diff --git a/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java b/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java index 893331669..e2ad6d33c 100644 --- a/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java +++ b/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java @@ -81,6 +81,9 @@ import org.apache.log4j.Logger; /** + * Class is no longer used, switched to using configured namespace and bucketUri to query + * for artifacts to remove. But left in this package if a use case is found at a future date. + * * Implementation of ArtifactSelector that includes artifacts via selective queries. * This class requires one or more fragments of SQL (a WHERE clause), each in a separate * file located in {user.home}/config/include and named {something}.sql -- see the diff --git a/ringhold/src/main/java/org/opencadc/ringhold/InventoryValidator.java b/ringhold/src/main/java/org/opencadc/ringhold/InventoryValidator.java index 5956e368d..cda660098 100644 --- a/ringhold/src/main/java/org/opencadc/ringhold/InventoryValidator.java +++ b/ringhold/src/main/java/org/opencadc/ringhold/InventoryValidator.java @@ -69,12 +69,17 @@ import ca.nrc.cadc.db.TransactionManager; import ca.nrc.cadc.io.ResourceIterator; -import ca.nrc.cadc.net.ResourceNotFoundException; +import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.util.BucketSelector; import java.io.IOException; +import java.util.Iterator; +import java.util.List; import java.util.Map; import org.apache.log4j.Logger; import org.opencadc.inventory.Artifact; import org.opencadc.inventory.DeletedStorageLocationEvent; +import org.opencadc.inventory.InventoryUtil; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.ArtifactDAO; import org.opencadc.inventory.db.DeletedStorageLocationEventDAO; @@ -89,28 +94,27 @@ public class InventoryValidator implements Runnable { private final ArtifactDAO artifactIteratorDAO; private final ArtifactDAO artifactDAO; - private final String deselector; + private final List namespaces; + private final BucketSelector bucketSelector; - public InventoryValidator(Map txnConfig, Map iterConfig) { + public InventoryValidator(Map txnConfig, Map iterConfig, + List namespaces, BucketSelector bucketSelector) { + InventoryUtil.assertNotNull(InventoryValidator.class, "txnConfig", txnConfig); + InventoryUtil.assertNotNull(InventoryValidator.class, "iterConfig", iterConfig); + InventoryUtil.assertNotNull(InventoryValidator.class, "namespaces", namespaces); + this.artifactDAO = new ArtifactDAO(); artifactDAO.setConfig(txnConfig); this.artifactIteratorDAO = new ArtifactDAO(); artifactIteratorDAO.setConfig(iterConfig); - ArtifactDeselector artifactDeselector = new ArtifactDeselector(); - try { - this.deselector = artifactDeselector.getConstraint(); - } catch (ResourceNotFoundException ex) { - throw new IllegalArgumentException("missing required configuration: " - + ArtifactDeselector.SQL_FILTER_FILE_NAME, ex); - } catch (IOException ex) { - throw new IllegalArgumentException("unable to read config: " + ArtifactDeselector.SQL_FILTER_FILE_NAME, ex); - } + this.namespaces = namespaces; + this.bucketSelector = bucketSelector; } /** - * Find an artifact with a uri pattern in the deselector, + * Find an artifact for the given namespace(s) and optional bucketUri, * delete the artifact and generate a deleted storage location event. */ @Override @@ -119,31 +123,68 @@ public void run() { final DeletedStorageLocationEventDAO deletedStorageLocationEventDAO = new DeletedStorageLocationEventDAO(this.artifactDAO); - try (final ResourceIterator artifactIterator = - this.artifactIteratorDAO.iterator(this.deselector, null, false)) { + for (Namespace namespace : namespaces) { + if (bucketSelector == null) { + iterateBucket(transactionManager, deletedStorageLocationEventDAO, namespace,null); + } else { + Iterator bucketIter = bucketSelector.getBucketIterator(); + while (bucketIter.hasNext()) { + String bucket = bucketIter.next(); + log.info(InventoryValidator.class.getSimpleName() + ".START bucket=" + bucket); + int retries = 0; + boolean done = false; + while (!done && retries < 3) { + try { + iterateBucket(transactionManager, deletedStorageLocationEventDAO, namespace, bucket); + log.info(InventoryValidator.class.getSimpleName() + ".END bucket=" + bucket); + done = true; + } catch (TransientException ex) { + log.error(InventoryValidator.class.getSimpleName() + ".FAIL bucket=" + bucket, ex); + retries++; + } catch (IllegalArgumentException ex) { + log.error(InventoryValidator.class.getSimpleName() + ".FAIL bucket=" + bucket, ex); + throw ex; + } catch (RuntimeException ex) { + // TODO: probably not a great idea to retry on these... + log.error(InventoryValidator.class.getSimpleName() + ".FAIL bucket=" + bucket, ex); + retries++; + } catch (Exception ex) { + log.error(InventoryValidator.class.getSimpleName() + ".FAIL bucket=" + bucket, ex); + throw ex; + } + } + } + } + } + } + + private void iterateBucket(TransactionManager transactionManager, + DeletedStorageLocationEventDAO deletedStorageLocationEventDAO, + Namespace namespace, String bucket) { + try (final ResourceIterator artifactIterator = this.artifactIteratorDAO.iterator(namespace, bucket, false)) { while (artifactIterator.hasNext()) { - Artifact deselectorArtifact = artifactIterator.next(); - log.debug("START: Process Artifact " + deselectorArtifact.getID() + " " + deselectorArtifact.getURI()); + Artifact artifact = artifactIterator.next(); + log.debug("START: Process Artifact " + artifact.getID() + " " + artifact.getURI()); try { transactionManager.startTransaction(); - Artifact cur = this.artifactDAO.lock(deselectorArtifact); + Artifact cur = this.artifactDAO.lock(artifact); if (cur != null) { DeletedStorageLocationEvent deletedStorageLocationEvent = new DeletedStorageLocationEvent(cur.getID()); deletedStorageLocationEventDAO.put(deletedStorageLocationEvent); - + this.artifactDAO.delete(cur.getID()); - + transactionManager.commitTransaction(); log.info("DELETE: Artifact " + cur.getID() + " " + cur.getURI()); } else { transactionManager.rollbackTransaction(); log.debug("Artifact not found"); } - - log.debug("END: Process Artifact " + deselectorArtifact.getID() + " " - + deselectorArtifact.getURI()); + + log.debug("END: Process Artifact " + artifact.getID() + " " + + artifact.getURI()); } catch (Exception exception) { if (transactionManager.isOpen()) { log.error("Exception in transaction. Rolling back..."); @@ -164,4 +205,5 @@ public void run() { log.error("Error closing iterator: " + e.getMessage()); } } + } diff --git a/ringhold/src/main/java/org/opencadc/ringhold/Main.java b/ringhold/src/main/java/org/opencadc/ringhold/Main.java index b774ba256..f4d64954d 100644 --- a/ringhold/src/main/java/org/opencadc/ringhold/Main.java +++ b/ringhold/src/main/java/org/opencadc/ringhold/Main.java @@ -69,17 +69,21 @@ import ca.nrc.cadc.db.ConnectionConfig; import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.util.BucketSelector; import ca.nrc.cadc.util.Log4jInit; import ca.nrc.cadc.util.MultiValuedProperties; import ca.nrc.cadc.util.PropertiesReader; import ca.nrc.cadc.util.StringUtil; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import javax.naming.NamingException; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.opencadc.inventory.Namespace; import org.opencadc.inventory.db.SQLGenerator; /** @@ -99,6 +103,8 @@ public class Main { private static final String DB_USERNAME_CONFIG_KEY = CONFIG_PREFIX + ".inventory.username"; private static final String DB_PASSWORD_CONFIG_KEY = CONFIG_PREFIX + ".inventory.password"; private static final String DB_URL_CONFIG_KEY = CONFIG_PREFIX + ".inventory.url"; + private static final String NAMESPACE_CONFIG_KEY = CONFIG_PREFIX + ".namespace"; + private static final String BUCKETS_CONFIG_KEY = CONFIG_PREFIX + ".buckets"; // Used to verify configuration items. See the README for descriptions. private static final String[] MANDATORY_PROPERTY_KEYS = { @@ -107,7 +113,8 @@ public class Main { DB_URL_CONFIG_KEY, DB_USERNAME_CONFIG_KEY, LOGGING_CONFIG_KEY, - SQLGENERATOR_CONFIG_KEY + SQLGENERATOR_CONFIG_KEY, + NAMESPACE_CONFIG_KEY }; public static void main(final String[] args) { @@ -163,7 +170,21 @@ public static void main(final String[] args) { iterConfig.put("jndiDataSourceName", "jdbc/inventory-iter"); iterConfig.put(SQLGENERATOR_CONFIG_KEY, Class.forName(configuredSQLGenerator)); - final InventoryValidator doit = new InventoryValidator(daoConfig, iterConfig); + // check namespaces are valid + final List configuredNamespaces = props.getProperty(NAMESPACE_CONFIG_KEY); + final List namespaces = new ArrayList<>(); + for (String namespace : configuredNamespaces) { + namespaces.add(new Namespace(namespace)); + } + + // uri buckets + BucketSelector bucketSelector = null; + final String buckets = props.getFirstPropertyValue(BUCKETS_CONFIG_KEY); + if (buckets != null) { + bucketSelector = new BucketSelector(buckets); + } + + final InventoryValidator doit = new InventoryValidator(daoConfig, iterConfig, namespaces, bucketSelector); doit.run(); } catch (Throwable unexpected) { log.fatal("Unexpected failure", unexpected); From 8c91699d7bedca367c39d4abcacb316ef27eddfa Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 Apr 2024 12:48:14 -0700 Subject: [PATCH 182/186] rework after review --- ChangeLog.md | 12 ++++++------ .../org/opencadc/vospace/db/DataNodeSizeWorker.java | 5 +++-- minoc/README.md | 9 +++++---- .../opencadc/vault/metadata/DataNodeSizeSync.java | 1 - 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 7fca1ef3a..77d824e33 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,13 +17,13 @@ added optional `org.opencadc.minoc.readable` and `org.opencadc.minoc.writable` c A `minoc` service will advertise (via inventory.StorageSite record in the database) the _readable_ and _writable_ status; this information is synced to global inventory and used by `raven` to determine if it should generate PUT or GET URLs that use the `minoc` -service(s) at that site. By default, the configuration of _readGrantProvider_(s) and -_writeGrantProvider_(s) is used to determine the default status; configuration of any -_trust.preauth_ will also make make the status _readable_ and _writable_ even if the -service is only intending to accept one or the other. +service(s) at that site. The configuration of _readGrantProvider_(s) and +_writeGrantProvider_(s) implicitly determines the status (_readable_ and _writable_ +respectively); configuration of any _trust.preauth_ will also implicitly make make the +status _readable_ and _writable_. -The explicit _readable_ and _writable_ configuration options will override all other -logic and set the status accordingly. This is currently optional but may be required +The explicit _readable_ and _writable_ configuration options will override the above +implicit logic and set the status accordingly. This is currently optional but may be required in a future version. ``` diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java index 236da5ca1..5bfcb30e3 100644 --- a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java +++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java @@ -134,7 +134,8 @@ public void run() { + " start=" + df.format(harvestState.curLastModified)); } else { log.debug(opName + " source=" + harvestState.getResourceID() - + " instance=" + harvestState.instanceID); + + " instance=" + harvestState.instanceID + + " start=null"); } final Date now = new Date(); @@ -193,7 +194,7 @@ public void run() { } else { log.debug(opName + " source=" + harvestState.getResourceID() + " instance=" + harvestState.instanceID - + " end=true"); + + " end=null"); } } diff --git a/minoc/README.md b/minoc/README.md index e0292b941..99831467e 100644 --- a/minoc/README.md +++ b/minoc/README.md @@ -91,7 +91,8 @@ org.opencadc.minoc.recoverableNamespace = {namespace} The optional _trust.preauth_ key(s) configure `minoc` to trust external service(s) to have performed authorization checks. Such services may include a signed token in the URL and `minoc` will validate the request using a public key retrieved from the service instead of performing authorization checks -itself. Example: +itself. Currently, only `raven` and `vault` can generate such URLs and provide access to their +public keys. Example: ``` # trust a SI global inventory org.opencadc.minoc.trust.preauth = ivo://example.net/raven @@ -99,13 +100,13 @@ org.opencadc.minoc.trust.preauth = ivo://example.net/raven # trust a SI VOSpace service org.opencadc.minoc.trust.preauth = ivo://example.net/vault ``` -Setting _trust.preauth_ one or more times also implies _readable_ and _writable_ are _true_. +Setting _trust.preauth_ one or more times implicitly sets _readable_ and _writable_ to _true_. The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for operations. Multiple values of the granting service resourceID(s) may be provided by including multiple property settings (one per line). All services will be consulted but a single positive -result is sufficient to grant permission for an action. Setting these values also sets the implied _readable_ -and _writable_ is _true_ respectively. +result is sufficient to grant permission for an action. Setting these values implicitly sets _readable_ +and _writable_ to _true_ respectively. The optional _readable_ and _writable_ keys configure minoc explicitly rather than relying on one or more of the above trust or grant provider settings. For example, this allows one to configure a read-only minoc diff --git a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java index a63eb83c9..3bce32e99 100644 --- a/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java +++ b/vault/src/main/java/org/opencadc/vault/metadata/DataNodeSizeSync.java @@ -157,7 +157,6 @@ public void run() { worker.run(); logInfo.setLastModified(state.curLastModified); logInfo.processed = worker.getNumArtifactsProcessed(); - logInfo.setSuccess(true); } catch (Exception ex) { log.error("unexpected worker fail", ex); fail = true; From adbadf203e9d0cd44685d33ec4d767457c4513c6 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Tue, 2 Apr 2024 13:23:35 -0700 Subject: [PATCH 183/186] CADC-13234 checkstyle fix --- .../main/java/org/opencadc/ringhold/ArtifactDeselector.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java b/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java index e2ad6d33c..2570e29b3 100644 --- a/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java +++ b/ringhold/src/main/java/org/opencadc/ringhold/ArtifactDeselector.java @@ -81,9 +81,9 @@ import org.apache.log4j.Logger; /** - * Class is no longer used, switched to using configured namespace and bucketUri to query + *

Class is no longer used, switched to using configured namespace and bucketUri to query * for artifacts to remove. But left in this package if a use case is found at a future date. - * +

* Implementation of ArtifactSelector that includes artifacts via selective queries. * This class requires one or more fragments of SQL (a WHERE clause), each in a separate * file located in {user.home}/config/include and named {something}.sql -- see the From 47800127fbf01bb0ad06a2d867776333a29f7446 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 2 Apr 2024 13:48:28 -0700 Subject: [PATCH 184/186] remaining SI image version to 1.0.0 --- critwall/VERSION | 2 +- fenwick/VERSION | 2 +- ratik/VERSION | 2 +- tantar/VERSION | 2 +- vault/VERSION | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/critwall/VERSION b/critwall/VERSION index 7fbddd116..d849b8f34 100644 --- a/critwall/VERSION +++ b/critwall/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.5.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/fenwick/VERSION b/fenwick/VERSION index a77484760..38b1547ae 100644 --- a/fenwick/VERSION +++ b/fenwick/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.6.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/ratik/VERSION b/ratik/VERSION index d9e5d6047..38b1547ae 100644 --- a/ratik/VERSION +++ b/ratik/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.2.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/tantar/VERSION b/tantar/VERSION index 7fbddd116..d849b8f34 100644 --- a/tantar/VERSION +++ b/tantar/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor[.patch] # build version tag: timestamp -VER=0.5.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/vault/VERSION b/vault/VERSION index 17ac68c1c..3d4fab565 100644 --- a/vault/VERSION +++ b/vault/VERSION @@ -4,6 +4,6 @@ # tags with and without build number so operators use the versioned # tag but we always keep a timestamped tag in case a semantic tag gets # replaced accidentally -VER=0.5.0 +VER=1.0.0 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER From 593bb1e9154fabab1df1fc5e211b26011794fa5e Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 3 Apr 2024 08:05:15 -0700 Subject: [PATCH 185/186] CADC-13234 restore ringhold in github build --- .github/workflows/gradle.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 64748cf5b..e3771b65b 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -70,7 +70,6 @@ jobs: - name: java build -- ratik run: cd ratik && ../gradlew --info clean build javadoc checkstyleMain -## disabled until updated for cadc-inventory-db API changes -# - name: java build -- ringhold -# run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain + - name: java build -- ringhold + run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain From a0f5f27dd2a8b0d1b8f6e45ed43474f93312cf66 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 3 Apr 2024 08:20:13 -0700 Subject: [PATCH 186/186] CADC-13234 fix github build formatting --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index e3771b65b..86f24decc 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -70,6 +70,6 @@ jobs: - name: java build -- ratik run: cd ratik && ../gradlew --info clean build javadoc checkstyleMain - - name: java build -- ringhold - run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain + - name: java build -- ringhold + run: cd ringhold && ../gradlew --info clean build javadoc checkstyleMain