iterator(String uriBucketPrefix, boolean order
*
* Use case: local cleanup by arbitrary criteria
*
- * @param criteria conditions for selecting artifacts
+ * @param ns namespace for selecting artifacts
* @param uriBucketPrefix null, prefix, or complete Artifact.uriBucket string
* @param ordered order by Artifact.uri (true) or not ordered (false)
* @return iterator over artifacts matching criteria
*/
- public ResourceIterator iterator(String criteria, String uriBucketPrefix, boolean ordered) {
+ public ResourceIterator iterator(Namespace ns, String uriBucketPrefix, boolean ordered) {
+ return iterator(ns, uriBucketPrefix, null, ordered);
+ }
+
+ /**
+ * Iterate over artifacts that match criteria. This method adds an optional Date argument to
+ * support incremental processing. In this case, ordered will be in timestamp order rather than
+ * uri order.
+ *
+ * Use case: process artifact events directly in the database
+ *
+ * @param ns namespace for selecting artifacts
+ * @param uriBucketPrefix null, prefix, or complete Artifact.uriBucket string
+ * @param minLastModified minimum Artifact.lastModified to consider (incremental mode)
+ * @param ordered order by Artifact.uri (true) or not ordered (false)
+ * @return iterator over artifacts matching criteria
+ */
+ public ResourceIterator iterator(Namespace ns, String uriBucketPrefix, Date minLastModified, boolean ordered) {
checkInit();
long t = System.currentTimeMillis();
try {
SQLGenerator.ArtifactIteratorQuery iter = (SQLGenerator.ArtifactIteratorQuery) gen.getEntityIteratorQuery(Artifact.class);
- iter.setPrefix(uriBucketPrefix);
- iter.setCriteria(criteria);
+ iter.setUriBucket(uriBucketPrefix);
+ iter.setNamespace(ns);
+ iter.setMinLastModified(minLastModified);
iter.setOrderedOutput(ordered);
return iter.query(dataSource);
} catch (BadSqlGrammarException ex) {
@@ -273,7 +293,7 @@ public ResourceIterator iterator(String criteria, String uriBucketPref
}
throw new RuntimeException("BUG: should be unreachable");
}
-
+
/**
* Iterate over Artifacts from a specific site. If a siteID is specified, only artifacts where
* artifact.siteLocations includes that siteID are returned; this is only applicable in a global
@@ -292,7 +312,7 @@ public ResourceIterator iterator(UUID siteID, String uriBucketPrefix,
try {
SQLGenerator.ArtifactIteratorQuery iter = (SQLGenerator.ArtifactIteratorQuery) gen.getEntityIteratorQuery(Artifact.class);
- iter.setPrefix(uriBucketPrefix);
+ iter.setUriBucket(uriBucketPrefix);
iter.setSiteID(siteID);
iter.setOrderedOutput(ordered);
return iter.query(dataSource);
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java
index c065b58a6..029ca037b 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityGet.java
@@ -68,7 +68,7 @@
package org.opencadc.inventory.db;
import java.util.UUID;
-import org.opencadc.inventory.Entity;
+import org.opencadc.persist.Entity;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java
index 782de3129..04e065bb8 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityIteratorQuery.java
@@ -67,7 +67,7 @@
package org.opencadc.inventory.db;
-import java.util.Iterator;
+import ca.nrc.cadc.io.ResourceIterator;
import javax.sql.DataSource;
/**
@@ -76,5 +76,5 @@
* @param entity subclass
*/
public interface EntityIteratorQuery {
- Iterator query(DataSource ds);
+ ResourceIterator query(DataSource ds);
}
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java
index 238518d78..d99e41742 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityLock.java
@@ -68,7 +68,7 @@
package org.opencadc.inventory.db;
import java.util.UUID;
-import org.opencadc.inventory.Entity;
+import org.opencadc.persist.Entity;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java
index 383347eba..e236c7a2b 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/EntityPut.java
@@ -69,7 +69,7 @@
package org.opencadc.inventory.db;
-import org.opencadc.inventory.Entity;
+import org.opencadc.persist.Entity;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.PreparedStatementCreator;
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java
index 210030c66..4d012cc24 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestState.java
@@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
-* (c) 2020. (c) 2020.
+* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -70,7 +70,6 @@
import java.net.URI;
import java.util.Date;
import java.util.UUID;
-
import org.apache.log4j.Logger;
import org.opencadc.inventory.Entity;
import org.opencadc.inventory.InventoryUtil;
@@ -95,6 +94,12 @@ public class HarvestState extends Entity {
*/
public UUID curID;
+ /**
+ * The ID of the current running instance. This is optional and only used by applications
+ * that share workload between instances.
+ */
+ public UUID instanceID;
+
public HarvestState(String name, URI resourceID) {
super();
InventoryUtil.assertNotNull(HarvestState.class, "name", name);
@@ -117,6 +122,7 @@ public String getName() {
}
public void setName(String name) {
+ InventoryUtil.assertNotNull(HarvestState.class, "name", name);
this.name = name;
}
@@ -131,6 +137,9 @@ public void setResourceID(URI resourceID) {
@Override
public String toString() {
+ if (instanceID != null) {
+ return HarvestState.class.getSimpleName() + "[" + instanceID + "," + name + "," + resourceID + "]";
+ }
return HarvestState.class.getSimpleName() + "[" + name + "," + resourceID + "]";
}
}
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java
index d7dd81e2b..a0f8d8f74 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/HarvestStateDAO.java
@@ -68,6 +68,7 @@
package org.opencadc.inventory.db;
import java.net.URI;
+import java.sql.Connection;
import java.sql.SQLException;
import java.util.UUID;
import org.apache.log4j.Logger;
@@ -176,12 +177,16 @@ public HarvestState get(String name, URI resourceID) {
@Override
public void put(HarvestState val) {
- if (curBufferCount < updateBufferCount) {
+ put(val, false);
+ }
+
+ public void put(HarvestState val, boolean forceTimestampUpdate) {
+ if (curBufferCount < updateBufferCount && !forceTimestampUpdate) {
log.debug("buffering: " + curBufferCount + " < " + updateBufferCount + " " + val);
curBufferCount++;
bufferedState = val;
} else {
- super.put(val);
+ super.put(val, false, forceTimestampUpdate);
curBufferCount = 0;
bufferedState = null;
@@ -190,13 +195,15 @@ public void put(HarvestState val) {
if (curMaintCount == maintCount) {
String sql = "VACUUM " + gen.getTable(HarvestState.class);
log.warn("maintenance: " + curMaintCount + "==" + maintCount + " " + sql);
- //JdbcTemplate jdbc = new JdbcTemplate(dataSource);
- //jdbc.execute(sql);
try {
- dataSource.getConnection().createStatement().execute(sql);
- } catch (SQLException ex) {
- log.error("ERROR: " + sql + " FAILED", ex);
- // yes, log and proceed
+ try (Connection c = dataSource.getConnection()) {
+ c.createStatement().execute(sql);
+ } catch (SQLException ex) {
+ log.error("maintenance failed: " + sql, ex);
+ // yes, log and proceed
+ } // auto-close to return to pool
+ } catch (Exception ex) {
+ log.error("failed to close connection after maintenance: " + sql, ex);
}
curMaintCount = 0;
} else {
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/PreauthKeyPairDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/PreauthKeyPairDAO.java
new file mode 100644
index 000000000..aa8085594
--- /dev/null
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/PreauthKeyPairDAO.java
@@ -0,0 +1,144 @@
+/*
+************************************************************************
+******************* CANADIAN ASTRONOMY DATA CENTRE *******************
+************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
+*
+* (c) 2022. (c) 2022.
+* Government of Canada Gouvernement du Canada
+* National Research Council Conseil national de recherches
+* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
+* All rights reserved Tous droits réservés
+*
+* NRC disclaims any warranties, Le CNRC dénie toute garantie
+* expressed, implied, or énoncée, implicite ou légale,
+* statutory, of any kind with de quelque nature que ce
+* respect to the software, soit, concernant le logiciel,
+* including without limitation y compris sans restriction
+* any warranty of merchantability toute garantie de valeur
+* or fitness for a particular marchande ou de pertinence
+* purpose. NRC shall not be pour un usage particulier.
+* liable in any event for any Le CNRC ne pourra en aucun cas
+* damages, whether direct or être tenu responsable de tout
+* indirect, special or general, dommage, direct ou indirect,
+* consequential or incidental, particulier ou général,
+* arising from the use of the accessoire ou fortuit, résultant
+* software. Neither the name de l'utilisation du logiciel. Ni
+* of the National Research le nom du Conseil National de
+* Council of Canada nor the Recherches du Canada ni les noms
+* names of its contributors may de ses participants ne peuvent
+* be used to endorse or promote être utilisés pour approuver ou
+* products derived from this promouvoir les produits dérivés
+* software without specific prior de ce logiciel sans autorisation
+* written permission. préalable et particulière
+* par écrit.
+*
+* This file is part of the Ce fichier fait partie du projet
+* OpenCADC project. OpenCADC.
+*
+* OpenCADC is free software: OpenCADC est un logiciel libre ;
+* you can redistribute it and/or vous pouvez le redistribuer ou le
+* modify it under the terms of modifier suivant les termes de
+* the GNU Affero General Public la “GNU Affero General Public
+* License as published by the License” telle que publiée
+* Free Software Foundation, par la Free Software Foundation
+* either version 3 of the : soit la version 3 de cette
+* License, or (at your option) licence, soit (à votre gré)
+* any later version. toute version ultérieure.
+*
+* OpenCADC is distributed in the OpenCADC est distribué
+* hope that it will be useful, dans l’espoir qu’il vous
+* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE
+* without even the implied GARANTIE : sans même la garantie
+* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ
+* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF
+* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence
+* General Public License for Générale Publique GNU Affero
+* more details. pour plus de détails.
+*
+* You should have received Vous devriez avoir reçu une
+* a copy of the GNU Affero copie de la Licence Générale
+* General Public License along Publique GNU Affero avec
+* with OpenCADC. If not, see OpenCADC ; si ce n’est
+* . pas le cas, consultez :
+* .
+*
+************************************************************************
+*/
+
+package org.opencadc.inventory.db;
+
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.log4j.Logger;
+import org.opencadc.inventory.PreauthKeyPair;
+import org.springframework.jdbc.BadSqlGrammarException;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+/**
+ * Simple DAO class to store and retrieve public-private key pairs.
+ *
+ * @author pdowler
+ */
+public class PreauthKeyPairDAO extends AbstractDAO {
+ private static final Logger log = Logger.getLogger(PreauthKeyPairDAO.class);
+
+ public PreauthKeyPairDAO() {
+ super(true);
+ }
+
+ public PreauthKeyPairDAO(AbstractDAO src) {
+ super(src);
+ }
+
+ public PreauthKeyPair get(UUID id) {
+ return super.get(PreauthKeyPair.class, id);
+ }
+
+ public PreauthKeyPair get(String name) {
+ if (name == null) {
+ throw new IllegalArgumentException("name cannot be null");
+ }
+ checkInit();
+ log.debug("GET: " + name);
+ long t = System.currentTimeMillis();
+
+ try {
+ JdbcTemplate jdbc = new JdbcTemplate(dataSource);
+
+ SQLGenerator.KeyPairGet get = ( SQLGenerator.KeyPairGet) gen.getEntityGet(PreauthKeyPair.class);
+ get.setName(name);
+ PreauthKeyPair ret = get.execute(jdbc);
+ return ret;
+ } catch (BadSqlGrammarException ex) {
+ handleInternalFail(ex);
+ } finally {
+ long dt = System.currentTimeMillis() - t;
+ log.debug("GET: " + name + " " + dt + "ms");
+ }
+ throw new RuntimeException("BUG: should be unreachable");
+ }
+
+ public void delete(UUID id) {
+ super.delete(PreauthKeyPair.class, id);
+ }
+
+ public Set list() {
+ checkInit();
+ log.debug("LIST");
+ long t = System.currentTimeMillis();
+
+ try {
+ JdbcTemplate jdbc = new JdbcTemplate(dataSource);
+ EntityList get = gen.getEntityList(PreauthKeyPair.class);
+ Set result = get.query(jdbc);
+ return result;
+ } catch (BadSqlGrammarException ex) {
+ handleInternalFail(ex);
+ } finally {
+ long dt = System.currentTimeMillis() - t;
+ log.debug("LIST: " + dt + "ms");
+ }
+ throw new RuntimeException("BUG: should be unreachable");
+ }
+}
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java
index fb5da9fec..0652a9184 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/SQLGenerator.java
@@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
-* (c) 2022. (c) 2022.
+* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -89,16 +89,25 @@
import java.util.UUID;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
+import org.opencadc.gms.GroupURI;
import org.opencadc.inventory.Artifact;
import org.opencadc.inventory.DeletedArtifactEvent;
import org.opencadc.inventory.DeletedStorageLocationEvent;
-import org.opencadc.inventory.Entity;
import org.opencadc.inventory.InventoryUtil;
+import org.opencadc.inventory.Namespace;
import org.opencadc.inventory.ObsoleteStorageLocation;
+import org.opencadc.inventory.PreauthKeyPair;
import org.opencadc.inventory.SiteLocation;
import org.opencadc.inventory.StorageLocation;
import org.opencadc.inventory.StorageLocationEvent;
import org.opencadc.inventory.StorageSite;
+import org.opencadc.persist.Entity;
+import org.opencadc.vospace.ContainerNode;
+import org.opencadc.vospace.DataNode;
+import org.opencadc.vospace.DeletedNodeEvent;
+import org.opencadc.vospace.LinkNode;
+import org.opencadc.vospace.Node;
+import org.opencadc.vospace.NodeProperty;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.ResultSetExtractor;
@@ -111,43 +120,57 @@
public class SQLGenerator {
private static final Logger log = Logger.getLogger(SQLGenerator.class);
- private final Map tableMap = new TreeMap(new ClassComp());
- private final Map columnMap = new TreeMap(new ClassComp());
+ private final Map tableMap = new TreeMap<>(new ClassComp());
+ private final Map columnMap = new TreeMap<>(new ClassComp());
protected final String database; // currently not used in SQL
- protected final String schema; // may be null
+ protected final String invSchema;
+ protected final String genSchema;
+ protected final String vosSchema;
/**
* Constructor. The database name is currently not used in any generated SQL; code assumes
* that the DataSource is connected to the right database already and cross-database statements
- * are not supported. The schema name is used to qualify table names (if set). The optional table
- * name prefix is pre-pended to the table name (after the optional {schema}.) and is provided as
- * a work around in shared (usually test) databases where schema name is a username and table
- * names may collide with other content. Normal "production" use would specify the schema only.
+ * are not supported.
*
- * @param database database name (may be null)
- * @param schema schema name (may be null)
+ * @param database database name: not used; future-proof
+ * @param invSchema inventory schema name (required, implies genSchema = invSchema)
+ * @param genSchema generic schema name for internal tables (PreauthKeys, HarvestState) - optional
*/
- public SQLGenerator(String database, String schema) {
+ public SQLGenerator(String database, String invSchema, String genSchema) {
+ this(database, invSchema, genSchema, null);
+ }
+
+ /**
+ * Constructor. The database name is currently not used in any generated SQL; code assumes
+ * that the DataSource is connected to the right database already and cross-database statements
+ * are not supported. The genSchema is optional (required for HarvestState and PreauthKeyPair).
+ * The vosSchema is optional (used for Node and DeletedNodeEvent).
+ *
+ * @param database database name: not used; future-proof
+ * @param invSchema inventory schema name - required
+ * @param genSchema generic schema name - optional
+ * @param vosSchema vospace schema name - optional
+ */
+ public SQLGenerator(String database, String invSchema, String genSchema, String vosSchema) {
this.database = database;
- this.schema = schema;
+ InventoryUtil.assertNotNull(SQLGenerator.class, "invSchema", invSchema); // required for all uses
+ this.invSchema = invSchema;
+ InventoryUtil.assertNotNull(SQLGenerator.class, "genSchema", genSchema); // required for correct init
+ this.genSchema = genSchema;
+ this.vosSchema = vosSchema; // only required for vospace
init();
}
protected void init() {
- String pref = "";
- if (schema != null) {
- pref = schema + ".";
- }
// inventory model
- this.tableMap.put(Artifact.class, pref + Artifact.class.getSimpleName());
- this.tableMap.put(StorageSite.class, pref + StorageSite.class.getSimpleName());
- this.tableMap.put(DeletedArtifactEvent.class, pref + DeletedArtifactEvent.class.getSimpleName());
- this.tableMap.put(DeletedStorageLocationEvent.class, pref + DeletedStorageLocationEvent.class.getSimpleName());
- this.tableMap.put(StorageLocationEvent.class, pref + StorageLocationEvent.class.getSimpleName());
+ this.tableMap.put(Artifact.class, invSchema + "." + Artifact.class.getSimpleName());
+ this.tableMap.put(StorageSite.class, invSchema + "." + StorageSite.class.getSimpleName());
+ this.tableMap.put(DeletedArtifactEvent.class, invSchema + "." + DeletedArtifactEvent.class.getSimpleName());
+ this.tableMap.put(DeletedStorageLocationEvent.class, invSchema + "." + DeletedStorageLocationEvent.class.getSimpleName());
+ this.tableMap.put(StorageLocationEvent.class, invSchema + "." + StorageLocationEvent.class.getSimpleName());
// internal
- this.tableMap.put(ObsoleteStorageLocation.class, pref + ObsoleteStorageLocation.class.getSimpleName());
- this.tableMap.put(HarvestState.class, pref + HarvestState.class.getSimpleName());
+ this.tableMap.put(ObsoleteStorageLocation.class, invSchema + "." + ObsoleteStorageLocation.class.getSimpleName());
String[] cols = new String[] {
"uri", // first column is logical key
@@ -193,16 +216,71 @@ protected void init() {
};
this.columnMap.put(ObsoleteStorageLocation.class, cols);
- cols = new String[] {
- "name",
- "resourceID",
- "curLastModified",
- "curID",
- "lastModified",
- "metaChecksum",
- "id" // last column is always PK
- };
- this.columnMap.put(HarvestState.class, cols);
+ log.debug("genSchema: " + genSchema);
+ if (genSchema != null) {
+ // generic support
+ this.tableMap.put(HarvestState.class, genSchema + "." + HarvestState.class.getSimpleName());
+ this.tableMap.put(PreauthKeyPair.class, genSchema + "." + PreauthKeyPair.class.getSimpleName());
+ cols = new String[] {
+ "name",
+ "resourceID",
+ "curLastModified",
+ "curID",
+ "instanceID",
+ "lastModified",
+ "metaChecksum",
+ "id" // last column is always PK
+ };
+ this.columnMap.put(HarvestState.class, cols);
+
+ cols = new String[] {
+ "name",
+ "publicKey",
+ "privateKey",
+ "lastModified",
+ "metaChecksum",
+ "id" // last column is always PK
+ };
+ this.columnMap.put(PreauthKeyPair.class, cols);
+ }
+
+ // optional vospace
+ log.debug("vosSchema: " + vosSchema);
+ if (vosSchema != null) {
+ tableMap.put(Node.class, vosSchema + "." + Node.class.getSimpleName());
+ tableMap.put(DeletedNodeEvent.class, vosSchema + "." + DeletedNodeEvent.class.getSimpleName());
+
+ cols = new String[] {
+ "parentID",
+ "name",
+ "nodeType",
+ "ownerID",
+ "isPublic",
+ "isLocked",
+ "readOnlyGroups",
+ "readWriteGroups",
+ "properties",
+ "inheritPermissions",
+ "bytesUsed",
+ "busy",
+ "storageID",
+ "storageBucket",
+ "target",
+ "lastModified",
+ "metaChecksum",
+ "id" // last column is always PK
+ };
+ this.columnMap.put(Node.class, cols);
+
+ cols = new String[] {
+ "nodeType",
+ "storageID",
+ "lastModified",
+ "metaChecksum",
+ "id" // last column is always PK
+ };
+ this.columnMap.put(DeletedNodeEvent.class, cols);
+ }
}
private static class ClassComp implements Comparator {
@@ -218,9 +296,28 @@ public String getCurrentTimeSQL() {
return "SELECT now()";
}
- // test usage
- String getTable(Class c) {
- return tableMap.get(c);
+ public String getTable(Class c) {
+ Class targetClass = c;
+ String ret = tableMap.get(targetClass);
+ if (ret == null) {
+ // enable finding a common table that stores subclass instances
+ targetClass = targetClass.getSuperclass();
+ ret = tableMap.get(targetClass);
+ }
+ log.debug("table: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + ret);
+ return ret;
+ }
+
+ private String[] getColumns(Class c) {
+ Class targetClass = c;
+ String[] ret = columnMap.get(targetClass);
+ if (ret == null) {
+ // enable finding a common table that stores subclass instances
+ targetClass = targetClass.getSuperclass();
+ ret = columnMap.get(targetClass);
+ }
+ log.debug("columns: " + c.getSimpleName() + " -> " + targetClass.getSimpleName() + " -> " + (ret == null ? null : ret.length));
+ return ret;
}
public EntityGet getEntityGet(Class c) {
@@ -234,11 +331,18 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) {
if (StorageSite.class.equals(c)) {
return new StorageSiteGet(forUpdate);
}
+ if (PreauthKeyPair.class.equals(c)) {
+ return new KeyPairGet(forUpdate);
+ }
+ if (Node.class.equals(c)) {
+ return new NodeGet(forUpdate);
+ }
if (forUpdate) {
throw new UnsupportedOperationException("entity-get + forUpdate: " + c.getSimpleName());
}
+ // raw events are never locked for update
if (DeletedArtifactEvent.class.equals(c)) {
return new DeletedArtifactEventGet();
}
@@ -251,29 +355,58 @@ public EntityGet getEntityGet(Class c, boolean forUpdate) {
if (ObsoleteStorageLocation.class.equals(c)) {
return new ObsoleteStorageLocationGet();
}
+
+ if (DeletedNodeEvent.class.equals(c)) {
+ //return new DeletedNodeGet();
+ }
+
if (HarvestState.class.equals(c)) {
return new HarvestStateGet();
}
+
+
+
throw new UnsupportedOperationException("entity-get: " + c.getName());
}
+ public NodeCount getNodeCount() {
+ return new NodeCount();
+ }
+
+ public class NodeCount {
+ private UUID id;
+
+ public void setID(UUID id) {
+ this.id = id;
+ }
+
+ public int execute(JdbcTemplate jdbc) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("SELECT count(*) FROM ").append(getTable(Node.class));
+ sb.append(" WHERE parentID = '").append(id.toString()).append("'");
+ String sql = sb.toString();
+ log.debug("NodeCount: " + sql);
+ int ret = jdbc.queryForObject(sql, Integer.class);
+ return ret;
+ }
+ }
+
public EntityIteratorQuery getEntityIteratorQuery(Class c) {
if (Artifact.class.equals(c)) {
return new ArtifactIteratorQuery();
}
- throw new UnsupportedOperationException("entity-list: " + c.getName());
+ if (Node.class.equals(c)) {
+ return new NodeIteratorQuery();
+ }
+ throw new UnsupportedOperationException("entity-iterator: " + c.getName());
}
public EntityList getEntityList(Class c) {
if (StorageSite.class.equals(c)) {
return new StorageSiteList();
}
- throw new UnsupportedOperationException("entity-list: " + c.getName());
- }
-
- public EntityLock getEntityLock(Class c) {
- if (Artifact.class.equals(c)) {
- return new EntityLockImpl(c);
+ if (PreauthKeyPair.class.equals(c)) {
+ return new KeyPairList();
}
throw new UnsupportedOperationException("entity-list: " + c.getName());
}
@@ -305,6 +438,15 @@ public EntityPut getEntityPut(Class c, boolean update) {
if (HarvestState.class.equals(c)) {
return new HarvestStatePut(update);
}
+ if (PreauthKeyPair.class.equals(c)) {
+ return new KeyPairPut(update);
+ }
+ if (Node.class.isAssignableFrom(c)) {
+ return new NodePut(update);
+ }
+ if (DeletedNodeEvent.class.equals(c)) {
+ //return new DeletedNodePut(update);
+ }
throw new UnsupportedOperationException("entity-put: " + c.getName());
}
@@ -312,41 +454,6 @@ public EntityDelete getEntityDelete(Class c) {
return new EntityDeleteImpl(c);
}
- private class EntityLockImpl implements EntityLock {
- private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
- private final Class entityClass;
- private UUID id;
-
- EntityLockImpl(Class entityClass) {
- this.entityClass = entityClass;
- }
-
- @Override
- public void setID(UUID id) {
- this.id = id;
- }
-
- @Override
- public void execute(JdbcTemplate jdbc) throws EntityNotFoundException {
- int n = jdbc.update(this);
- if (n == 0) {
- throw new EntityNotFoundException("not found: " + id);
- }
- }
-
- @Override
- public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
- String sql = getLockSQL(entityClass);
- log.debug("EntityLockImpl: " + sql);
- PreparedStatement prep = conn.prepareStatement(sql);
- int col = 1;
- prep.setObject(col++, id);
- prep.setObject(col++, id);
-
- return prep;
- }
- }
-
private class SkeletonGet implements EntityGet {
private UUID id;
private final Class entityClass;
@@ -428,7 +535,7 @@ public void setLocation(StorageLocation loc) {
@Override
public ObsoleteStorageLocation execute(JdbcTemplate jdbc) {
- return (ObsoleteStorageLocation) jdbc.query(this, new ObsoleteStorageLocationExtractor());
+ return jdbc.query(this, new ObsoleteStorageLocationExtractor());
}
@Override
@@ -443,7 +550,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
String col = getKeyColumn(ObsoleteStorageLocation.class, true);
sb.append(col).append(" = ?");
} else {
- String[] cols = columnMap.get(ObsoleteStorageLocation.class);
+ String[] cols = getColumns(ObsoleteStorageLocation.class);
sb.append(cols[0]).append(" = ?");
sb.append(" AND ");
if (loc.storageBucket != null) {
@@ -500,7 +607,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
String col = getKeyColumn(HarvestState.class, true);
sb.append(col).append(" = ?");
} else {
- String[] cols = columnMap.get(HarvestState.class);
+ String[] cols = getColumns(HarvestState.class);
sb.append(cols[0]).append(" = ?");
sb.append(" AND ");
sb.append(cols[1]).append(" = ?");
@@ -571,10 +678,15 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
class ArtifactIteratorQuery implements EntityIteratorQuery {
private Boolean storageLocationRequired;
- private String prefix;
+ private String storageBucket;
+
private UUID siteID;
- private String whereClause;
+ private String uriBucket;
+ private Namespace namespace;
+ private Date minLastModified;
private boolean ordered;
+
+ private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
public ArtifactIteratorQuery() {
}
@@ -590,22 +702,30 @@ public void setStorageLocationRequired(Boolean slr) {
this.storageLocationRequired = slr;
}
- public void setPrefix(String prefix) {
+ public void setStorageBucket(String prefix) {
if (StringUtil.hasText(prefix)) {
- this.prefix = prefix.trim();
+ this.storageBucket = prefix.trim();
} else {
- this.prefix = null;
+ this.storageBucket = null;
}
}
- public void setCriteria(String whereClause) {
- if (StringUtil.hasText(whereClause)) {
- this.whereClause = whereClause.trim();
+ public void setUriBucket(String uriBucket) {
+ if (StringUtil.hasText(uriBucket)) {
+ this.uriBucket = uriBucket.trim();
} else {
- this.whereClause = null;
+ uriBucket = null;
}
}
+ public void setNamespace(Namespace namespace) {
+ this.namespace = namespace;
+ }
+
+ public void setMinLastModified(Date minLastModified) {
+ this.minLastModified = minLastModified;
+ }
+
public void setOrderedOutput(boolean ordered) {
this.ordered = ordered;
}
@@ -616,13 +736,14 @@ public void setSiteID(UUID siteID) {
@Override
public ResourceIterator query(DataSource ds) {
-
- StringBuilder sb = getSelectFromSQL(Artifact.class, false);
- sb.append(" WHERE");
-
+ StringBuilder sb = new StringBuilder();
+ boolean where = false;
if (storageLocationRequired != null && storageLocationRequired) {
// ArtifactDAO.storedIterator
- if (StringUtil.hasText(prefix)) {
+ sb.append(" WHERE");
+ where = true;
+ if (storageBucket != null) {
+
sb.append(" storageLocation_storageBucket LIKE ? AND");
}
sb.append(" storageLocation_storageID IS NOT NULL");
@@ -636,42 +757,64 @@ public ResourceIterator query(DataSource ds) {
}
} else if (storageLocationRequired != null && !storageLocationRequired) {
// ArtifactDAO.unstoredIterator
- if (StringUtil.hasText(prefix)) {
- sb.append(" uriBucket LIKE ? AND");
- }
+ sb.append(" WHERE");
+ where = true;
sb.append(" storageLocation_storageID IS NULL");
- if (ordered) {
- sb.append(" ORDER BY uri");
- }
- } else if (siteID != null) {
- if (prefix != null && siteID != null) {
- sb.append(" uriBucket LIKE ? AND ").append("siteLocations @> ARRAY[?]");
+ }
+
+ // optional params:
+ // uriBucket
+ // siteID
+ // namespace
+ // minLastModified
+ if (uriBucket != null) {
+ if (where) {
+ sb.append(" AND");
} else {
- sb.append(" siteLocations @> ARRAY[?]");
+ sb.append(" WHERE");
+ where = true;
}
- if (ordered) {
- sb.append(" ORDER BY uri");
+ sb.append(" uriBucket LIKE ?");
+ }
+ if (siteID != null) {
+ // ArtifactDAO.iterator(UUID, ...)
+ if (where) {
+ sb.append(" AND");
+ } else {
+ sb.append(" WHERE");
+ where = true;
}
- } else if (whereClause != null) {
- if (prefix != null && whereClause != null) {
- sb.append(" uriBucket LIKE ? AND ( ").append(whereClause).append(" )");
+ sb.append(" siteLocations @> ARRAY[?]");
+ }
+ if (namespace != null) {
+ if (where) {
+ sb.append(" AND");
} else {
- sb.append(" (").append(whereClause).append(" )");
+ sb.append(" WHERE");
+ where = true;
}
- if (ordered) {
- sb.append(" ORDER BY uri");
+ sb.append(" uri LIKE ?");
+ }
+ if (minLastModified != null) {
+ if (where) {
+ sb.append(" AND");
+ } else {
+ sb.append(" WHERE");
+ where = true;
}
- } else if (prefix != null) {
- sb.append(" uriBucket LIKE ?");
- if (ordered) {
+ sb.append(" lastModified >= ?");
+ }
+ if (ordered && !(storageLocationRequired != null && storageLocationRequired)) {
+ if (minLastModified != null) {
+ sb.append(" ORDER BY lastModified ASC");
+ } else {
sb.append(" ORDER BY uri");
}
- } else {
- // trim off " WHERE"
- sb.delete(sb.length() - 6, sb.length());
}
- String sql = sb.toString();
+ StringBuilder select = getSelectFromSQL(Artifact.class, false);
+ select.append(sb.toString());
+ String sql = select.toString();
log.debug("sql: " + sql);
try {
@@ -683,15 +826,29 @@ public ResourceIterator query(DataSource ds) {
ps.setFetchSize(1000);
ps.setFetchDirection(ResultSet.FETCH_FORWARD);
int col = 1;
- if (prefix != null) {
- String val = prefix + "%";
+ if (storageBucket != null) {
+ String val = storageBucket + "%";
+ log.debug("bucket prefix: " + val);
+ ps.setString(col++, val);
+ } else if (uriBucket != null) {
+ String val = uriBucket + "%";
log.debug("bucket prefix: " + val);
ps.setString(col++, val);
}
if (siteID != null) {
log.debug("siteID: " + siteID);
ps.setObject(col++, siteID);
+ }
+ if (namespace != null) {
+ String val = namespace.getNamespace() + "%";
+ log.debug("namespace prefix: " + val);
+ ps.setString(col++, val);
+ }
+ if (minLastModified != null) {
+ log.debug("min lastModified: " + minLastModified);
+ ps.setTimestamp(col++, new Timestamp(minLastModified.getTime()), utc);
}
+
ResultSet rs = ps.executeQuery();
return new ArtifactResultSetIterator(con, rs);
@@ -704,7 +861,6 @@ public ResourceIterator query(DataSource ds) {
private class StorageSiteGet implements EntityGet {
private UUID id;
- private URI uri;
private final boolean forUpdate;
public StorageSiteGet(boolean forUpdate) {
@@ -731,6 +887,9 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
} else {
throw new IllegalStateException("primary key is null");
}
+ if (forUpdate) {
+ sb.append(" FOR UPDATE");
+ }
String sql = sb.toString();
log.debug("StorageSiteGet: " + sql);
PreparedStatement prep = conn.prepareStatement(sql);
@@ -759,6 +918,228 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
}
}
+ class KeyPairGet implements EntityGet {
+ private UUID id;
+ private String name;
+ private final boolean forUpdate;
+
+ public KeyPairGet(boolean forUpdate) {
+ this.forUpdate = forUpdate;
+ }
+
+ @Override
+ public void setID(UUID id) {
+ this.id = id;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public PreauthKeyPair execute(JdbcTemplate jdbc) {
+ return (PreauthKeyPair) jdbc.query(this, new KeyPairExtractor());
+ }
+
+ @Override
+ public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
+ StringBuilder sb = getSelectFromSQL(PreauthKeyPair.class, false);
+ sb.append(" WHERE ");
+ if (id != null) {
+ String col = getKeyColumn(PreauthKeyPair.class, true);
+ sb.append(col).append(" = ?");
+ } else if (name != null) {
+ sb.append("name = ?");
+ } else {
+ throw new IllegalStateException("primary key and name are both null");
+ }
+ if (forUpdate) {
+ sb.append(" FOR UPDATE");
+ }
+ String sql = sb.toString();
+ log.debug("KeyPairGet: " + sql);
+ PreparedStatement prep = conn.prepareStatement(sql);
+ if (id != null) {
+ prep.setObject(1, id);
+ } else {
+ prep.setString(1, name);
+ }
+ return prep;
+ }
+ }
+
+ private class KeyPairList implements EntityList {
+
+ @Override
+ public Set query(JdbcTemplate jdbc) {
+ List keys = (List) jdbc.query(this, new KeyPairRowMapper());
+ Set ret = new TreeSet<>();
+ ret.addAll(keys);
+ return ret;
+ }
+
+ @Override
+ public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
+ StringBuilder sb = getSelectFromSQL(PreauthKeyPair.class, false);
+ String sql = sb.toString();
+ log.debug("KeyPairList: " + sql);
+ PreparedStatement prep = conn.prepareStatement(sql);
+ return prep;
+ }
+ }
+
+ public class NodeGet implements EntityGet {
+ private UUID id;
+ private ContainerNode parent;
+ private String name;
+ private URI storageID;
+ private final boolean forUpdate;
+
+ public NodeGet(boolean forUpdate) {
+ this.forUpdate = forUpdate;
+ }
+
+ @Override
+ public void setID(UUID id) {
+ this.id = id;
+ }
+
+ public void setPath(ContainerNode parent, String name) {
+ this.parent = parent;
+ this.name = name;
+ }
+
+ public void setStorageID(URI storageID) {
+ this.storageID = storageID;
+ }
+
+ @Override
+ public Node execute(JdbcTemplate jdbc) {
+ return (Node) jdbc.query(this, new NodeExtractor(parent));
+ }
+
+ @Override
+ public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
+ StringBuilder sb = getSelectFromSQL(Node.class, false);
+ sb.append(" WHERE ");
+ if (id != null) {
+ String col = getKeyColumn(Node.class, true);
+ sb.append(col).append(" = ?");
+ } else if (storageID != null) {
+ String col = "storageID";
+ sb.append(col).append(" = ?");
+ } else if (parent != null && name != null) {
+ String pidCol = "parentID";
+ String nameCol = "name";
+ sb.append(pidCol).append(" = ? and ").append(nameCol).append(" = ?");
+ } else {
+ throw new IllegalStateException("primary key is null");
+ }
+ if (forUpdate) {
+ sb.append(" FOR UPDATE");
+ }
+ String sql = sb.toString();
+ log.debug("Node: " + sql);
+ PreparedStatement prep = conn.prepareStatement(sql);
+ if (id != null) {
+ prep.setObject(1, id);
+ } else if (storageID != null) {
+ prep.setObject(1, storageID.toASCIIString());
+ } else {
+ prep.setObject(1, parent.getID());
+ prep.setObject(2, name);
+ }
+
+ return prep;
+ }
+ }
+
+ public class NodeIteratorQuery implements EntityIteratorQuery {
+ private ContainerNode parent;
+ private String start;
+ private Integer limit;
+
+ public NodeIteratorQuery() {
+ }
+
+ public void setParent(ContainerNode parent) {
+ this.parent = parent;
+ }
+
+ public void setStart(String start) {
+ this.start = start;
+ }
+
+ public void setLimit(Integer limit) {
+ this.limit = limit;
+ }
+
+ @Override
+ public ResourceIterator query(DataSource ds) {
+ if (parent == null) {
+ throw new RuntimeException("BUG: cannot query for children with parent==null");
+ }
+
+ StringBuilder sb = getSelectFromSQL(Node.class, false);
+ sb.append(" WHERE parentID = ?");
+ if (start != null) {
+ sb.append(" AND ? <= name");
+ }
+ sb.append(" ORDER BY name ASC");
+ if (limit != null) {
+ sb.append(" LIMIT ?");
+ }
+
+ String sql = sb.toString();
+ log.debug("sql: " + sql);
+
+ try {
+ Connection con = ds.getConnection();
+ log.debug("NodeIteratorQuery: setAutoCommit(false)");
+ con.setAutoCommit(false);
+ // defaults for options: ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY
+ PreparedStatement ps = con.prepareStatement(sql);
+ ps.setFetchSize(1000);
+ ps.setFetchDirection(ResultSet.FETCH_FORWARD);
+ int col = 1;
+
+ ps.setObject(col++, parent.getID());
+ log.debug("parentID = " + parent.getID());
+ if (start != null) {
+ ps.setString(col++, start);
+ log.debug("start = " + start);
+ }
+ if (limit != null) {
+ ps.setInt(col++, limit);
+ log.debug("limit = " + limit);
+ }
+ ResultSet rs = ps.executeQuery();
+
+ return new NodeResultSetIterator(con, rs, parent);
+ } catch (SQLException ex) {
+ throw new RuntimeException("BUG: artifact iterator query failed", ex);
+ }
+
+ }
+ }
+
+ private void safeSetBoolean(PreparedStatement prep, int col, Boolean value) throws SQLException {
+ log.debug("safeSetBoolean: " + col + " " + value);
+ if (value != null) {
+ prep.setBoolean(col, value);
+ } else {
+ prep.setNull(col, Types.BOOLEAN);
+ }
+ }
+
+ private void safeSetString(PreparedStatement prep, int col, URI value) throws SQLException {
+ String v = null;
+ if (value != null) {
+ v = value.toASCIIString();
+ }
+ safeSetString(prep, col, v);
+ }
+
private void safeSetString(PreparedStatement prep, int col, String value) throws SQLException {
log.debug("safeSetString: " + col + " " + value);
if (value != null) {
@@ -786,6 +1167,24 @@ private void safeSetTimestamp(PreparedStatement prep, int col, Timestamp value,
}
}
+ private void safeSetArray(PreparedStatement prep, int col, Set values) throws SQLException {
+
+ if (values != null && !values.isEmpty()) {
+ log.debug("safeSetArray: " + col + " " + values.size());
+ String[] array1d = new String[values.size()];
+ int i = 0;
+ for (GroupURI u : values) {
+ array1d[i] = u.getURI().toASCIIString();
+ i++;
+ }
+ java.sql.Array arr = prep.getConnection().createArrayOf("text", array1d);
+ prep.setObject(col, arr);
+ } else {
+ log.debug("safeSetArray: " + col + " null");
+ prep.setNull(col, Types.ARRAY);
+ }
+ }
+
private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws SQLException {
if (value != null) {
@@ -798,6 +1197,25 @@ private void safeSetArray(PreparedStatement prep, int col, UUID[] value) throws
}
}
+ private void safeSetProps(PreparedStatement prep, int col, Set values) throws SQLException {
+
+ if (values != null && !values.isEmpty()) {
+ log.debug("safeSetProps: " + col + " " + values.size());
+ String[][] array2d = new String[values.size()][2]; // TODO: w-h or h-w??
+ int i = 0;
+ for (NodeProperty np : values) {
+ array2d[i][0] = np.getKey().toASCIIString();
+ array2d[i][1] = np.getValue();
+ i++;
+ }
+ java.sql.Array arr = prep.getConnection().createArrayOf("text", array2d);
+ prep.setObject(col, arr);
+ } else {
+ log.debug("safeSetProps: " + col + " = null");
+ prep.setNull(col, Types.ARRAY);
+ }
+ }
+
private class ArtifactPut implements EntityPut {
private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
private final boolean update;
@@ -852,8 +1270,8 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
safeSetString(prep, col++, value.storageLocation.getStorageID().toASCIIString());
safeSetString(prep, col++, value.storageLocation.storageBucket);
} else {
- safeSetString(prep, col++, null); // storageLocation.storageID
- safeSetString(prep, col++, null); // storageLocation.storageBucket
+ safeSetString(prep, col++, (URI) null); // storageLocation.storageID
+ safeSetString(prep, col++, (URI) null); // storageLocation.storageBucket
}
safeSetTimestamp(prep, col++, new Timestamp(value.getLastModified().getTime()), utc);
@@ -948,7 +1366,7 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
if (value.getLocation().storageBucket != null) {
safeSetString(prep, col++, value.getLocation().storageBucket);
} else {
- safeSetString(prep, col++, null);
+ safeSetString(prep, col++, (String) null);
}
prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc);
@@ -1004,6 +1422,11 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
} else {
prep.setNull(col++, Types.OTHER);
}
+ if (value.instanceID != null) {
+ prep.setObject(col++, value.instanceID);
+ } else {
+ prep.setNull(col++, Types.OTHER);
+ }
prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc);
prep.setString(col++, value.getMetaChecksum().toASCIIString());
@@ -1014,6 +1437,149 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
}
+ private class KeyPairPut implements EntityPut {
+ private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
+ private final boolean update;
+ private PreauthKeyPair value;
+
+ KeyPairPut(boolean update) {
+ this.update = update;
+ }
+
+ @Override
+ public void setValue(PreauthKeyPair value) {
+ this.value = value;
+ }
+
+ @Override
+ public void execute(JdbcTemplate jdbc) {
+ jdbc.update(this);
+ }
+
+ @Override
+ public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
+ String sql = null;
+ if (update) {
+ sql = getUpdateSQL(PreauthKeyPair.class);
+
+ } else {
+ sql = getInsertSQL(PreauthKeyPair.class);
+ }
+ log.debug("KeyPairPut: " + sql);
+ PreparedStatement prep = conn.prepareStatement(sql);
+ int col = 1;
+
+ prep.setString(col++, value.getName());
+ prep.setBytes(col++, value.getPublicKey());
+ prep.setBytes(col++, value.getPrivateKey());
+
+ prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc);
+ prep.setString(col++, value.getMetaChecksum().toASCIIString());
+ prep.setObject(col++, value.getID());
+
+ return prep;
+ }
+ }
+
+ private class NodePut implements EntityPut {
+ private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
+ private final boolean update;
+ private Node value;
+
+ NodePut(boolean update) {
+ this.update = update;
+ }
+
+ @Override
+ public void setValue(Node value) {
+ this.value = value;
+ }
+
+ @Override
+ public void execute(JdbcTemplate jdbc) {
+ jdbc.update(this);
+ }
+
+ @Override
+ public PreparedStatement createPreparedStatement(Connection conn) throws SQLException {
+ String sql = null;
+ if (update) {
+ sql = getUpdateSQL(Node.class);
+
+ } else {
+ sql = getInsertSQL(Node.class);
+ }
+ log.debug("NodePut: " + sql);
+ PreparedStatement prep = conn.prepareStatement(sql);
+ int col = 1;
+
+ if (value.parentID == null) {
+ throw new RuntimeException("BUG: cannot put Node without a parentID: " + value);
+ }
+ prep.setObject(col++, value.parentID);
+ prep.setString(col++, value.getName());
+ prep.setString(col++, value.getClass().getSimpleName().substring(0, 1)); // HACK
+ if (value.ownerID == null) {
+ throw new RuntimeException("BUG: cannot put Node without an ownerID: " + value);
+ }
+ prep.setString(col++, value.ownerID.toString());
+ safeSetBoolean(prep, col++, value.isPublic);
+ safeSetBoolean(prep, col++, value.isLocked);
+ safeSetArray(prep, col++, value.getReadOnlyGroup());
+ safeSetArray(prep, col++, value.getReadWriteGroup());
+ safeSetProps(prep, col++, value.getProperties());
+
+ // ContainerNode-specific fields
+ if (value instanceof ContainerNode) {
+ ContainerNode cn = (ContainerNode) value;
+ safeSetBoolean(prep, col++, cn.inheritPermissions);
+ } else {
+ safeSetBoolean(prep, col++, null);
+ }
+
+ // bytesUsed is in between CN and DN specific columns
+ if (value instanceof ContainerNode) {
+ ContainerNode cn = (ContainerNode) value;
+ safeSetLong(prep, col++, cn.bytesUsed);
+ } else if (value instanceof DataNode) {
+ DataNode dn = (DataNode) value;
+ safeSetLong(prep, col++, dn.bytesUsed);
+ } else {
+ safeSetLong(prep, col++, null);
+ }
+
+ // DataNode specific fields
+ if (value instanceof DataNode) {
+ DataNode dn = (DataNode) value;
+ if (dn.storageID == null) {
+ throw new RuntimeException("BUG: cannot put DataNode without a storageID: " + value);
+ }
+ safeSetBoolean(prep, col++, dn.busy);
+ safeSetString(prep, col++, dn.storageID);
+ safeSetString(prep, col++, InventoryUtil.computeBucket(dn.storageID, 5)); // same as Artifact
+ } else {
+ safeSetBoolean(prep, col++, null);
+ safeSetString(prep, col++, (URI) null);
+ safeSetString(prep, col++, (URI) null);
+ }
+
+ // LinkNode-specific fields
+ if (value instanceof LinkNode) {
+ LinkNode ln = (LinkNode) value;
+ prep.setString(col++, ln.getTarget().toASCIIString());
+ } else {
+ safeSetString(prep, col++, (URI) null);
+ }
+
+ // Entity fields
+ prep.setTimestamp(col++, new Timestamp(value.getLastModified().getTime()), utc);
+ prep.setString(col++, value.getMetaChecksum().toASCIIString());
+ prep.setObject(col++, value.getID());
+
+ return prep;
+ }
+ }
+
private class EntityEventPut implements EntityPut {
private final Calendar utc = Calendar.getInstance(DateUtil.UTC);
private final boolean update;
@@ -1083,11 +1649,13 @@ public PreparedStatement createPreparedStatement(Connection conn) throws SQLExce
}
private StringBuilder getSelectFromSQL(Class c, boolean entityCols) {
- String tab = tableMap.get(c);
- String[] cols = columnMap.get(c);
+ String tab = getTable(c);
+ Class targetClass = c;
if (entityCols) {
- cols = columnMap.get(Entity.class);
+ targetClass = Entity.class;
}
+ String[] cols = getColumns(targetClass);
+
if (tab == null || cols == null) {
throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName());
}
@@ -1106,21 +1674,13 @@ private StringBuilder getSelectFromSQL(Class c, boolean entityCols) {
return sb;
}
- private String getLockSQL(Class c) {
- StringBuilder sb = new StringBuilder();
- String pk = getKeyColumn(c, true);
- sb.append("UPDATE ");
- sb.append(tableMap.get(c));
- sb.append(" SET ").append(pk).append(" = ? WHERE ").append(pk).append(" = ?");
- return sb.toString();
- }
-
private String getUpdateSQL(Class c) {
StringBuilder sb = new StringBuilder();
+ String tab = getTable(c);
sb.append("UPDATE ");
- sb.append(tableMap.get(c));
+ sb.append(tab);
sb.append(" SET ");
- String[] cols = columnMap.get(c);
+ String[] cols = getColumns(c);
for (int i = 0; i < cols.length - 1; i++) { // PK is last
if (i > 0) {
sb.append(",");
@@ -1137,10 +1697,11 @@ private String getUpdateSQL(Class c) {
private String getInsertSQL(Class c) {
StringBuilder sb = new StringBuilder();
+ String tab = getTable(c);
sb.append("INSERT INTO ");
- sb.append(tableMap.get(c));
+ sb.append(tab);
sb.append(" (");
- String[] cols = columnMap.get(c);
+ String[] cols = getColumns(c);
for (int i = 0; i < cols.length; i++) {
if (i > 0) {
sb.append(",");
@@ -1161,14 +1722,15 @@ private String getInsertSQL(Class c) {
private String getDeleteSQL(Class c) {
StringBuilder sb = new StringBuilder();
+ String tab = getTable(c);
sb.append("DELETE FROM ");
- sb.append(tableMap.get(c));
+ sb.append(tab);
sb.append(" WHERE id = ?");
return sb.toString();
}
private String getKeyColumn(Class c, boolean pk) {
- String[] cols = columnMap.get(c);
+ String[] cols = getColumns(c);
if (cols == null) {
throw new IllegalArgumentException("BUG: no table/columns for class " + c.getName());
}
@@ -1226,6 +1788,7 @@ private class ArtifactResultSetIterator implements ResourceIterator {
private final Connection con;
private final ResultSet rs;
boolean hasRow;
+ boolean closeWhenDone = true; // return to pool | assume close suppressed for static connections
ArtifactResultSetIterator(Connection con, ResultSet rs) throws SQLException {
this.con = con;
@@ -1234,7 +1797,14 @@ private class ArtifactResultSetIterator implements ResourceIterator {
log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor " + hasRow);
if (!hasRow) {
log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)");
- con.setAutoCommit(true);
+ try {
+ con.setAutoCommit(true); // commit txn
+ if (closeWhenDone) {
+ con.close(); // return to pool
+ }
+ } catch (SQLException unexpected) {
+ log.error("Connection.setAutoCommit(true) & close() failed", unexpected);
+ }
}
}
@@ -1243,10 +1813,13 @@ public void close() throws IOException {
if (hasRow) {
log.debug("ArtifactResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)");
try {
- con.setAutoCommit(true);
- hasRow = false;
- } catch (SQLException ex) {
- throw new RuntimeException("BUG: artifact list query failed during close()", ex);
+ con.setAutoCommit(true); // commit txn
+ if (closeWhenDone) {
+ con.close(); // return to pool
+ }
+ hasRow = false;
+ } catch (SQLException unexpected) {
+ log.error("Connection.setAutoCommit(true) & close() failed", unexpected);
}
}
}
@@ -1263,17 +1836,27 @@ public Artifact next() {
hasRow = rs.next();
if (!hasRow) {
log.debug("ArtifactResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)");
- con.setAutoCommit(true);
+ try {
+ con.setAutoCommit(true); // commit txn
+ if (closeWhenDone) {
+ con.close(); // return to pool
+ }
+ } catch (SQLException unexpected) {
+ log.error("Connection.setAutoCommit(true) & close() failed", unexpected);
+ }
}
return ret;
} catch (Exception ex) {
if (hasRow) {
log.debug("ArtifactResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)");
try {
- close();
+ con.setAutoCommit(true); // commit txn
+ if (closeWhenDone) {
+ con.close(); // return to pool
+ }
hasRow = false;
- } catch (IOException unexpected) {
- log.debug("BUG: unexpected IOException from close", unexpected);
+ } catch (SQLException unexpected) {
+ log.error("Connection.setAutoCommit(true) & close() failed", unexpected);
}
}
throw new RuntimeException("BUG: artifact list query failed while iterating", ex);
@@ -1281,6 +1864,83 @@ public Artifact next() {
}
}
+ private class NodeResultSetIterator implements ResourceIterator {
+ final Calendar utc = Calendar.getInstance(DateUtil.UTC);
+ private final Connection con;
+ private final ResultSet rs;
+ boolean hasRow;
+
+ ContainerNode parent;
+
+ public NodeResultSetIterator(Connection con, ResultSet rs, ContainerNode parent) throws SQLException {
+ this.con = con;
+ this.rs = rs;
+ this.parent = parent;
+ hasRow = rs.next();
+ log.debug("NodeResultSetIterator: " + super.toString() + " ctor " + hasRow);
+ if (!hasRow) {
+ log.debug("NodeResultSetIterator: " + super.toString() + " ctor - setAutoCommit(true)");
+
+ try {
+ con.setAutoCommit(true); // commit txn
+ con.close(); // return to pool
+ } catch (SQLException ignore) {
+ log.error("Connection.setAutoCommit(true) & close() failed", ignore);
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (hasRow) {
+ log.debug("NodeResultSetIterator: " + super.toString() + " close - setAutoCommit(true)");
+ try {
+ con.setAutoCommit(true); // commit txn
+ con.close(); // return to pool
+ hasRow = false;
+ } catch (SQLException ignore) {
+ log.error("Connection.setAutoCommit(true) & close() failed", ignore);
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return hasRow;
+ }
+
+ @Override
+ public Node next() {
+ try {
+ Node ret = mapRowToNode(rs, utc, parent);
+ hasRow = rs.next();
+ if (!hasRow) {
+ log.debug("NodeResultSetIterator: " + super.toString() + " DONE - setAutoCommit(true)");
+ try {
+ con.setAutoCommit(true); // commit txn
+ con.close(); // return to pool
+ hasRow = false;
+ } catch (SQLException ignore) {
+ log.error("Connection.setAutoCommit(true) & close() failed", ignore);
+ }
+ }
+ return ret;
+ } catch (Exception ex) {
+ if (hasRow) {
+ log.debug("NodeResultSetIterator: " + super.toString() + " ResultSet.next() FAILED - setAutoCommit(true)");
+ try {
+ con.setAutoCommit(true); // commit txn
+ con.close(); // return to pool
+ hasRow = false;
+ } catch (SQLException ignore) {
+ log.error("Connection.setAutoCommit(true) & close() failed", ignore);
+ }
+ }
+ throw new RuntimeException("BUG: node list query failed while iterating", ex);
+ }
+ }
+ }
+
private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLException {
int col = 1;
final URI uri = Util.getURI(rs, col++);
@@ -1314,12 +1974,70 @@ private Artifact mapRowToArtifact(ResultSet rs, Calendar utc) throws SQLExceptio
return a;
}
- private class ObsoleteStorageLocationExtractor implements ResultSetExtractor {
+ private Node mapRowToNode(ResultSet rs, Calendar utc, ContainerNode parent) throws SQLException {
+ int col = 1;
+ final UUID parentID = Util.getUUID(rs, col++);
+ final String name = rs.getString(col++);
+ final String nodeType = rs.getString(col++);
+ final String ownerID = rs.getString(col++);
+ final Boolean isPublic = Util.getBoolean(rs, col++);
+ final Boolean isLocked = Util.getBoolean(rs, col++);
+ final String rawROG = rs.getString(col++);
+ final String rawRWG = rs.getString(col++);
+ final String rawProps = rs.getString(col++);
+ final Boolean inheritPermissions = Util.getBoolean(rs, col++);
+ final Long bytesUsed = Util.getLong(rs, col++);
+ final Boolean busy = Util.getBoolean(rs, col++);
+ final URI storageID = Util.getURI(rs, col++);
+ final String storageBucket = rs.getString(col++);
+ // TODO: return this somehow or just use in DataNode iterator?
+ final URI linkTarget = Util.getURI(rs, col++);
+ final Date lastModified = Util.getDate(rs, col++, utc);
+ final URI metaChecksum = Util.getURI(rs, col++);
+ final UUID id = Util.getUUID(rs, col++);
+
+ Node ret;
+ if (nodeType.equals("C")) {
+ ContainerNode cn = new ContainerNode(id, name);
+ cn.inheritPermissions = inheritPermissions;
+ cn.bytesUsed = bytesUsed;
+ ret = cn;
+ } else if (nodeType.equals("D")) {
+ DataNode dn = new DataNode(id, name, storageID);
+ dn.bytesUsed = bytesUsed;
+ ret = dn;
+ } else if (nodeType.equals("L")) {
+ ret = new LinkNode(id, name, linkTarget);
+ } else {
+ throw new RuntimeException("BUG: unexpected node type code: " + nodeType);
+ }
+ ret.parentID = parentID;
+ ret.ownerID = ownerID;
+ ret.isPublic = isPublic;
+ ret.isLocked = isLocked;
+
+ if (rawROG != null) {
+ Util.parseArrayGroupURI(rawROG, ret.getReadOnlyGroup());
+ }
+ if (rawRWG != null) {
+ Util.parseArrayGroupURI(rawRWG, ret.getReadWriteGroup());
+ }
+ if (rawProps != null) {
+ Util.parseArrayProps(rawProps, ret.getProperties());
+ }
+
+ InventoryUtil.assignLastModified(ret, lastModified);
+ InventoryUtil.assignMetaChecksum(ret, metaChecksum);
+
+ return ret;
+ }
+
+ private class ObsoleteStorageLocationExtractor implements ResultSetExtractor {
final Calendar utc = Calendar.getInstance(DateUtil.UTC);
@Override
- public Object extractData(ResultSet rs) throws SQLException, DataAccessException {
+ public ObsoleteStorageLocation extractData(ResultSet rs) throws SQLException, DataAccessException {
if (!rs.next()) {
return null;
}
@@ -1332,7 +2050,7 @@ public Object extractData(ResultSet rs) throws SQLException, DataAccessException
StorageLocation s = new StorageLocation(storLoc);
s.storageBucket = storBucket;
- Entity ret = new ObsoleteStorageLocation(id, s);
+ ObsoleteStorageLocation ret = new ObsoleteStorageLocation(id, s);
InventoryUtil.assignLastModified(ret, lastModified);
InventoryUtil.assignMetaChecksum(ret, metaChecksum);
return ret;
@@ -1352,6 +2070,7 @@ public HarvestState extractData(ResultSet rs) throws SQLException, DataAccessExc
final URI resourecID = Util.getURI(rs, col++);
final Date curLastModified = Util.getDate(rs, col++, utc);
final UUID curID = Util.getUUID(rs, col++);
+ final UUID instanceID = Util.getUUID(rs, col++);
final Date lastModified = Util.getDate(rs, col++, utc);
final URI metaChecksum = Util.getURI(rs, col++);
@@ -1360,6 +2079,7 @@ public HarvestState extractData(ResultSet rs) throws SQLException, DataAccessExc
HarvestState ret = new HarvestState(id, name, resourecID);
ret.curLastModified = curLastModified;
ret.curID = curID;
+ ret.instanceID = instanceID;
InventoryUtil.assignLastModified(ret, lastModified);
InventoryUtil.assignMetaChecksum(ret, metaChecksum);
return ret;
@@ -1401,6 +2121,40 @@ public StorageSite extractData(ResultSet rs) throws SQLException, DataAccessExce
}
}
+ private class KeyPairRowMapper implements RowMapper {
+ Calendar utc = Calendar.getInstance(DateUtil.UTC);
+
+ @Override
+ public PreauthKeyPair mapRow(ResultSet rs, int i) throws SQLException {
+ int col = 1;
+ final String name = rs.getString(col++);
+ final byte[] pub = rs.getBytes(col++);
+ final byte[] priv = rs.getBytes(col++);
+
+ final Date lastModified = Util.getDate(rs, col++, utc);
+ final URI metaChecksum = Util.getURI(rs, col++);
+ final UUID id = Util.getUUID(rs, col++);
+
+ PreauthKeyPair s = new PreauthKeyPair(id, name, pub, priv);
+ InventoryUtil.assignLastModified(s, lastModified);
+ InventoryUtil.assignMetaChecksum(s, metaChecksum);
+ return s;
+ }
+ }
+
+ private class KeyPairExtractor implements ResultSetExtractor {
+ final Calendar utc = Calendar.getInstance(DateUtil.UTC);
+
+ @Override
+ public PreauthKeyPair extractData(ResultSet rs) throws SQLException, DataAccessException {
+ if (!rs.next()) {
+ return null;
+ }
+ KeyPairRowMapper m = new KeyPairRowMapper();
+ return m.mapRow(rs, 1);
+ }
+ }
+
private class DeletedArtifactEventExtractor implements ResultSetExtractor {
final Calendar utc = Calendar.getInstance(DateUtil.UTC);
@@ -1465,4 +2219,22 @@ public StorageLocationEvent extractData(ResultSet rs) throws SQLException, DataA
return ret;
}
}
+
+ private class NodeExtractor implements ResultSetExtractor {
+ private ContainerNode parent;
+ final Calendar utc = Calendar.getInstance(DateUtil.UTC);
+
+ public NodeExtractor(ContainerNode parent) {
+ this.parent = parent; // optional
+ }
+
+ @Override
+ public Node extractData(ResultSet rs) throws SQLException, DataAccessException {
+ if (!rs.next()) {
+ return null;
+ }
+
+ return mapRowToNode(rs, utc, parent);
+ }
+ }
}
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java
index 404841ff8..782cb8a6f 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/Util.java
@@ -77,8 +77,12 @@
import java.sql.SQLException;
import java.util.Calendar;
import java.util.Date;
+import java.util.Set;
+import java.util.StringTokenizer;
import java.util.UUID;
import org.apache.log4j.Logger;
+import org.opencadc.gms.GroupURI;
+import org.opencadc.vospace.NodeProperty;
/**
*
@@ -378,39 +382,116 @@ public static byte[] getByteArray(ResultSet rs, int col)
throw new UnsupportedOperationException("converting " + o.getClass().getName() + " " + o + " to byte[]");
}
- /*
- * public static int[] getIntArray(ResultSet rs, int col)
- * throws SQLException
- * {
- * Object o = rs.getObject(col);
- * return toIntArray(o);
- * }
- *
- * static int[] toIntArray(Object o)
- * throws SQLException
- * {
- * if (o == null)
- * return null;
- * if (o instanceof Array)
- * {
- * Array a = (Array) o;
- * o = a.getArray();
- * }
- * if (o instanceof int[])
- * return (int[]) o;
- * if (o instanceof byte[])
- * return CaomUtil.decodeIntArray((byte[]) o);
- * if (o instanceof Integer[])
- * {
- * Integer[] ia = (Integer[]) o;
- * int[] ret = new int[ia.length];
- * for (int i=0; i dest) {
+ // postgresql 1D array: {a,"b,c"}
+ if (val == null || val.isEmpty()) {
+ return;
+ }
+ // GroupURI names can contain alphanumeric,comma,dash,dot,underscore,~
+ // PG quotes them if comma is present (eg in the group name)
+ char delim = '"';
+ int i = 0;
+ int j = val.indexOf(delim);
+ while (j != -1) {
+ String token = val.substring(i, j);
+ //log.warn("token: " + i + "," + j + " " + token);
+ i = j + 1;
+ j = val.indexOf(delim, i);
+
+ handleToken(token, dest);
+ }
+ String token = val.substring(i);
+ //log.warn("token: " + i + " " + token);
+ handleToken(token, dest);
+ }
+ private static void handleToken(String token, Set dest) {
+ if (token.startsWith("ivo://")) {
+ dest.add(new GroupURI(URI.create(token)));
+ } else {
+ StringTokenizer st = new StringTokenizer(token, "{,}");
+ while (st.hasMoreTokens()) {
+ String s = st.nextToken();
+ dest.add(new GroupURI(URI.create(s)));
+ }
+ }
+ }
+
+ // fills the dest set
+ public static void parseArrayProps(String val, Set dest) {
+ // postgresql 2D array: {{a,b},{c,d}}
+ if (val == null || val.isEmpty()) {
+ return;
+ }
+ char open = '{';
+ char close = '}';
+ char quote = '"';
+ int i = val.indexOf(open);
+ int j = val.lastIndexOf(close);
+ if (j > i) {
+ val = val.substring(i + 1, j);
+ }
+ i = val.indexOf(open);
+ j = val.indexOf(close, i + 1);
+ int k = 0;
+ while (i != -1 && j != -1 && k++ < 20) {
+ String t1 = val.substring(i + 1, j);
+ //log.warn("\tt1: " + i + "," + j + " " + t1);
+ handleProp(t1, dest);
+
+ if (i != -1 && j > 0) {
+ i = val.indexOf(open, j);
+ j = val.indexOf(close, i + 1);
+ // look ahead for quotes
+ int q = val.indexOf(quote, i);
+ //log.warn("i=" + i + " j=" + j + " q=" + q);
+ if (q != -1 && q < j) {
+ int cq = val.indexOf(quote, q + 1);
+ j = val.indexOf(close, cq);
+ //log.warn("\tcq=" + cq + " j=" + j);
+ }
+ }
+ }
+ }
+
+ private static void handleProp(String token, Set dest) {
+ int q = token.indexOf('"');
+ int cq = -1;
+ if (q == -1) {
+ q = Integer.MAX_VALUE;
+ } else {
+ cq = token.indexOf('"', q + 1);
+ }
+ int c = token.indexOf(',');
+
+ String key;
+ int split = c;
+ if (c < q) {
+ // key
+ key = token.substring(0, c);
+ } else {
+ // "key"
+ key = token.substring(q + 1, cq);
+ split = cq + 1;
+ }
+ //log.warn("\tkey: " + key);
+
+ q = token.indexOf('"', split + 1);
+ cq = -1;
+ if (q == -1) {
+ q = Integer.MAX_VALUE;
+ } else {
+ cq = token.indexOf('"', q + 1);
+ }
+ String val;
+ if (token.length() < q) {
+ val = token.substring(split + 1);
+ } else {
+ val = token.substring(q + 1, cq);
+ }
+ //log.warn("\tval: " + val);
+
+ dest.add(new NodeProperty(URI.create(key), val));
+ }
}
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java
similarity index 88%
rename from cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java
rename to cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java
index 6b299106a..f59d7d767 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabase.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/InitDatabaseSI.java
@@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
-* (c) 2020. (c) 2020.
+* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -67,6 +67,7 @@
package org.opencadc.inventory.db.version;
+import ca.nrc.cadc.db.version.InitDatabase;
import java.net.URL;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
@@ -75,32 +76,34 @@
*
* @author pdowler
*/
-public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase {
- private static final Logger log = Logger.getLogger(InitDatabase.class);
+public class InitDatabaseSI extends InitDatabase {
+ private static final Logger log = Logger.getLogger(InitDatabaseSI.class);
public static final String MODEL_NAME = "storage-inventory";
- public static final String MODEL_VERSION = "0.14";
- public static final String PREV_MODEL_VERSION = "0.10";
+ public static final String MODEL_VERSION = "1.0.0";
+ public static final String PREV_MODEL_VERSION = "0.14";
//public static final String PREV_MODEL_VERSION = "DO-NOT_UPGRADE-BY-ACCIDENT";
static String[] CREATE_SQL = new String[] {
- "inventory.ModelVersion.sql",
+ "generic.ModelVersion.sql",
"inventory.Artifact.sql",
"inventory.StorageSite.sql",
"inventory.ObsoleteStorageLocation.sql",
"inventory.DeletedArtifactEvent.sql",
"inventory.DeletedStorageLocationEvent.sql",
"inventory.StorageLocationEvent.sql",
- "inventory.HarvestState.sql",
- "inventory.permissions.sql"
+ "generic.HarvestState.sql",
+ "generic.PreauthKeyPair.sql",
+ "generic.permissions.sql"
};
static String[] UPGRADE_SQL = new String[] {
- "inventory.StorageLocationEvent.sql",
- "inventory.permissions.sql"
+ "inventory.upgrade-1.0.0.sql",
+ "generic.PreauthKeyPair.sql",
+ "generic.permissions.sql"
};
- public InitDatabase(DataSource ds, String database, String schema) {
+ public InitDatabaseSI(DataSource ds, String database, String schema) {
super(ds, database, schema, MODEL_NAME, MODEL_VERSION, PREV_MODEL_VERSION);
for (String s : CREATE_SQL) {
createSQL.add(s);
@@ -113,6 +116,6 @@ public InitDatabase(DataSource ds, String database, String schema) {
@Override
protected URL findSQL(String fname) {
// SQL files are stored inside the jar file
- return InitDatabase.class.getClassLoader().getResource(fname);
+ return InitDatabaseSI.class.getClassLoader().getResource(fname);
}
}
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java
index 5eddc5360..4d18205f3 100644
--- a/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/inventory/db/version/Main.java
@@ -145,7 +145,7 @@ public void run() {
DataSource ds = DBUtil.getDataSource(cc);
log.info("target: " + server + " " + database + " " + schema);
- InitDatabase init = new InitDatabase(ds, database, schema);
+ InitDatabaseSI init = new InitDatabaseSI(ds, database, schema);
boolean result = init.doInit();
if (result) {
log.info("init: complete");
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java
new file mode 100644
index 000000000..5bfcb30e3
--- /dev/null
+++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/DataNodeSizeWorker.java
@@ -0,0 +1,216 @@
+/*
+************************************************************************
+******************* CANADIAN ASTRONOMY DATA CENTRE *******************
+************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
+*
+* (c) 2024. (c) 2024.
+* Government of Canada Gouvernement du Canada
+* National Research Council Conseil national de recherches
+* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
+* All rights reserved Tous droits réservés
+*
+* NRC disclaims any warranties, Le CNRC dénie toute garantie
+* expressed, implied, or énoncée, implicite ou légale,
+* statutory, of any kind with de quelque nature que ce
+* respect to the software, soit, concernant le logiciel,
+* including without limitation y compris sans restriction
+* any warranty of merchantability toute garantie de valeur
+* or fitness for a particular marchande ou de pertinence
+* purpose. NRC shall not be pour un usage particulier.
+* liable in any event for any Le CNRC ne pourra en aucun cas
+* damages, whether direct or être tenu responsable de tout
+* indirect, special or general, dommage, direct ou indirect,
+* consequential or incidental, particulier ou général,
+* arising from the use of the accessoire ou fortuit, résultant
+* software. Neither the name de l'utilisation du logiciel. Ni
+* of the National Research le nom du Conseil National de
+* Council of Canada nor the Recherches du Canada ni les noms
+* names of its contributors may de ses participants ne peuvent
+* be used to endorse or promote être utilisés pour approuver ou
+* products derived from this promouvoir les produits dérivés
+* software without specific prior de ce logiciel sans autorisation
+* written permission. préalable et particulière
+* par écrit.
+*
+* This file is part of the Ce fichier fait partie du projet
+* OpenCADC project. OpenCADC.
+*
+* OpenCADC is free software: OpenCADC est un logiciel libre ;
+* you can redistribute it and/or vous pouvez le redistribuer ou le
+* modify it under the terms of modifier suivant les termes de
+* the GNU Affero General Public la “GNU Affero General Public
+* License as published by the License” telle que publiée
+* Free Software Foundation, par la Free Software Foundation
+* either version 3 of the : soit la version 3 de cette
+* License, or (at your option) licence, soit (à votre gré)
+* any later version. toute version ultérieure.
+*
+* OpenCADC is distributed in the OpenCADC est distribué
+* hope that it will be useful, dans l’espoir qu’il vous
+* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE
+* without even the implied GARANTIE : sans même la garantie
+* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ
+* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF
+* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence
+* General Public License for Générale Publique GNU Affero
+* more details. pour plus de détails.
+*
+* You should have received Vous devriez avoir reçu une
+* a copy of the GNU Affero copie de la Licence Générale
+* General Public License along Publique GNU Affero avec
+* with OpenCADC. If not, see OpenCADC ; si ce n’est
+* . pas le cas, consultez :
+* .
+*
+************************************************************************
+ */
+
+package org.opencadc.vospace.db;
+
+import ca.nrc.cadc.date.DateUtil;
+import ca.nrc.cadc.db.TransactionManager;
+import ca.nrc.cadc.io.ResourceIterator;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.util.Date;
+import org.apache.log4j.Logger;
+import org.opencadc.inventory.Artifact;
+import org.opencadc.inventory.Namespace;
+import org.opencadc.inventory.db.ArtifactDAO;
+import org.opencadc.inventory.db.HarvestState;
+import org.opencadc.inventory.db.HarvestStateDAO;
+import org.opencadc.vospace.DataNode;
+
+/**
+ * This class performs the work of synchronizing the size of Data Nodes from
+ * inventory (Artifact) to vopsace (Node).
+ *
+ * @author adriand
+ */
+public class DataNodeSizeWorker implements Runnable {
+ private static final Logger log = Logger.getLogger(DataNodeSizeWorker.class);
+
+ // lookback when doing incremental harvest because head of sequence is
+ // not monotonic over short timescales (events arrive out of sequence)
+ private static final long LOOKBACK_TIME_MS = 60 * 1000L;
+
+ private final HarvestState harvestState;
+ private final NodeDAO nodeDAO;
+ private final ArtifactDAO artifactDAO;
+ private final HarvestStateDAO harvestStateDAO;
+ private final Namespace storageNamespace;
+
+ private long numArtifactsProcessed;
+
+ /**
+ * Worker constructor.
+ *
+ * @param harvestStateDAO DAO class to persist progress in the vospace database
+ * @param harvestState current HarvestState instance
+ * @param artifactDAO DAO class to query for artifacts
+ * @param namespace artifact namespace
+ */
+ public DataNodeSizeWorker(HarvestStateDAO harvestStateDAO, HarvestState harvestState,
+ ArtifactDAO artifactDAO, Namespace namespace) {
+ this.harvestState = harvestState;
+ this.harvestStateDAO = harvestStateDAO;
+ this.nodeDAO = new NodeDAO(harvestStateDAO);
+ this.artifactDAO = artifactDAO;
+ this.storageNamespace = namespace;
+ }
+
+ public long getNumArtifactsProcessed() {
+ return numArtifactsProcessed;
+ }
+
+ @Override
+ public void run() {
+ this.numArtifactsProcessed = 0L;
+ String opName = DataNodeSizeWorker.class.getSimpleName() + ".artifactQuery";
+ DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC);
+ if (harvestState.curLastModified != null) {
+ log.debug(opName + " source=" + harvestState.getResourceID()
+ + " instance=" + harvestState.instanceID
+ + " start=" + df.format(harvestState.curLastModified));
+ } else {
+ log.debug(opName + " source=" + harvestState.getResourceID()
+ + " instance=" + harvestState.instanceID
+ + " start=null");
+ }
+
+ final Date now = new Date();
+ final Date lookBack = new Date(now.getTime() - LOOKBACK_TIME_MS);
+ Date startTime = getQueryLowerBound(lookBack, harvestState.curLastModified);
+ if (lookBack != null && harvestState.curLastModified != null) {
+ log.debug("lookBack=" + df.format(lookBack) + " curLastModified=" + df.format(harvestState.curLastModified)
+ + " -> " + df.format(startTime));
+ }
+
+ String uriBucket = null; // process all artifacts in a single thread
+ try (final ResourceIterator iter = artifactDAO.iterator(storageNamespace, uriBucket, startTime, true)) {
+ TransactionManager tm = nodeDAO.getTransactionManager();
+ while (iter.hasNext()) {
+ Artifact artifact = iter.next();
+ DataNode node = nodeDAO.getDataNode(artifact.getURI());
+ log.debug(artifact.getURI() + " len=" + artifact.getContentLength() + " -> " + node.getName());
+ if (node != null && !artifact.getContentLength().equals(node.bytesUsed)) {
+ tm.startTransaction();
+ try {
+ node = (DataNode)nodeDAO.lock(node);
+ if (node == null) {
+ continue; // node gone - race condition
+ }
+ node.bytesUsed = artifact.getContentLength();
+ nodeDAO.put(node);
+ tm.commitTransaction();
+ log.debug("ArtifactSyncWorker.updateDataNode id=" + node.getID()
+ + " bytesUsed=" + node.bytesUsed + " artifact.lastModified=" + df.format(artifact.getLastModified()));
+ } catch (Exception ex) {
+ log.debug("Failed to update data node size for " + node.getName(), ex);
+ tm.rollbackTransaction();
+ throw ex;
+ } finally {
+ if (tm.isOpen()) {
+ log.error("BUG: transaction open in finally. Rolling back...");
+ tm.rollbackTransaction();
+ log.error("Rollback: OK");
+ throw new RuntimeException("BUG: transaction open in finally");
+ }
+ }
+ }
+ harvestState.curLastModified = artifact.getLastModified();
+ harvestState.curID = artifact.getID();
+ harvestStateDAO.put(harvestState);
+ numArtifactsProcessed++;
+ }
+ } catch (IOException ex) {
+ log.error("Error closing iterator", ex);
+ throw new RuntimeException("error while closing ResourceIterator", ex);
+ }
+ if (harvestState.curLastModified != null) {
+ log.debug(opName + " source=" + harvestState.getResourceID()
+ + " instance=" + harvestState.instanceID
+ + " end=" + df.format(harvestState.curLastModified));
+ } else {
+ log.debug(opName + " source=" + harvestState.getResourceID()
+ + " instance=" + harvestState.instanceID
+ + " end=null");
+ }
+ }
+
+ private Date getQueryLowerBound(Date lookBack, Date lastModified) {
+ if (lookBack == null) {
+ // feature not enabled
+ return lastModified;
+ }
+ if (lastModified == null) {
+ // first harvest
+ return null;
+ }
+ if (lookBack.before(lastModified)) {
+ return lookBack;
+ }
+ return lastModified;
+
+ }
+}
diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/server/InitDatabaseAction.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java
similarity index 74%
rename from cadc-inventory-server/src/main/java/org/opencadc/inventory/server/InitDatabaseAction.java
rename to cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java
index 6236091a3..40d7f29de 100644
--- a/cadc-inventory-server/src/main/java/org/opencadc/inventory/server/InitDatabaseAction.java
+++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/InitDatabaseVOS.java
@@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
-* (c) 2020. (c) 2020.
+* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -65,52 +65,50 @@
************************************************************************
*/
-package org.opencadc.inventory.server;
+package org.opencadc.vospace.db;
-import ca.nrc.cadc.db.DBUtil;
-import ca.nrc.cadc.rest.InitAction;
-import java.util.Map;
-import java.util.TreeMap;
+import java.net.URL;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
-import org.opencadc.inventory.db.version.InitDatabase;
+import org.opencadc.inventory.db.version.InitDatabaseSI;
/**
- * Base class for storage service database initialisation.
- *
+ *
* @author pdowler
*/
-public abstract class InitDatabaseAction extends InitAction {
- private static final Logger log = Logger.getLogger(InitDatabaseAction.class);
+public class InitDatabaseVOS extends ca.nrc.cadc.db.version.InitDatabase {
+ private static final Logger log = Logger.getLogger(InitDatabaseVOS.class);
- protected final Map daoConfig = new TreeMap<>();
+ public static final String MODEL_NAME = "storage-vospace";
+ public static final String MODEL_VERSION = "1.0.0";
+ public static final String PREV_MODEL_VERSION = "n/a";
- protected InitDatabaseAction() {
- }
-
- @Override
- public void doInit() {
- initDaoConfig();
- initDatabase();
- }
+ static String[] CREATE_SQL = new String[] {
+ "generic.ModelVersion.sql",
+ "vospace.Node.sql",
+ "vospace.DeletedNodeEvent.sql",
+ "generic.HarvestState.sql",
+ "generic.PreauthKeyPair.sql",
+ "generic.permissions.sql"
+ };
- /**
- * Add content to the (protected) daoConfig map.
- */
- protected abstract void initDaoConfig();
+ static String[] UPGRADE_SQL = new String[] {
+ "generic.permissions.sql"
+ };
- private void initDatabase() {
- log.info("initDatabase: START");
- try {
- String jndiDataSourceName = (String) daoConfig.get("jndiDataSourceName");
- String database = (String) daoConfig.get("database");
- String schema = (String) daoConfig.get("schema");
- DataSource ds = DBUtil.findJNDIDataSource(jndiDataSourceName);
- InitDatabase init = new InitDatabase(ds, database, schema);
- init.doInit();
- log.info("initDatabase: " + jndiDataSourceName + " " + schema + " OK");
- } catch (Exception ex) {
- throw new IllegalStateException("check/init database failed", ex);
+ public InitDatabaseVOS(DataSource ds, String database, String schema) {
+ super(ds, database, schema, MODEL_NAME, MODEL_VERSION, PREV_MODEL_VERSION);
+ for (String s : CREATE_SQL) {
+ createSQL.add(s);
}
+ for (String s : UPGRADE_SQL) {
+ upgradeSQL.add(s);
+ }
+ }
+
+ @Override
+ protected URL findSQL(String fname) {
+ // SQL files are stored inside the jar file
+ return InitDatabaseSI.class.getClassLoader().getResource(fname);
}
}
diff --git a/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java
new file mode 100644
index 000000000..8088c66b7
--- /dev/null
+++ b/cadc-inventory-db/src/main/java/org/opencadc/vospace/db/NodeDAO.java
@@ -0,0 +1,214 @@
+/*
+************************************************************************
+******************* CANADIAN ASTRONOMY DATA CENTRE *******************
+************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
+*
+* (c) 2023. (c) 2023.
+* Government of Canada Gouvernement du Canada
+* National Research Council Conseil national de recherches
+* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
+* All rights reserved Tous droits réservés
+*
+* NRC disclaims any warranties, Le CNRC dénie toute garantie
+* expressed, implied, or énoncée, implicite ou légale,
+* statutory, of any kind with de quelque nature que ce
+* respect to the software, soit, concernant le logiciel,
+* including without limitation y compris sans restriction
+* any warranty of merchantability toute garantie de valeur
+* or fitness for a particular marchande ou de pertinence
+* purpose. NRC shall not be pour un usage particulier.
+* liable in any event for any Le CNRC ne pourra en aucun cas
+* damages, whether direct or être tenu responsable de tout
+* indirect, special or general, dommage, direct ou indirect,
+* consequential or incidental, particulier ou général,
+* arising from the use of the accessoire ou fortuit, résultant
+* software. Neither the name de l'utilisation du logiciel. Ni
+* of the National Research le nom du Conseil National de
+* Council of Canada nor the Recherches du Canada ni les noms
+* names of its contributors may de ses participants ne peuvent
+* be used to endorse or promote être utilisés pour approuver ou
+* products derived from this promouvoir les produits dérivés
+* software without specific prior de ce logiciel sans autorisation
+* written permission. préalable et particulière
+* par écrit.
+*
+* This file is part of the Ce fichier fait partie du projet
+* OpenCADC project. OpenCADC.
+*
+* OpenCADC is free software: OpenCADC est un logiciel libre ;
+* you can redistribute it and/or vous pouvez le redistribuer ou le
+* modify it under the terms of modifier suivant les termes de
+* the GNU Affero General Public la “GNU Affero General Public
+* License as published by the License” telle que publiée
+* Free Software Foundation, par la Free Software Foundation
+* either version 3 of the : soit la version 3 de cette
+* License, or (at your option) licence, soit (à votre gré)
+* any later version. toute version ultérieure.
+*
+* OpenCADC is distributed in the OpenCADC est distribué
+* hope that it will be useful, dans l’espoir qu’il vous
+* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE
+* without even the implied GARANTIE : sans même la garantie
+* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ
+* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF
+* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence
+* General Public License for Générale Publique GNU Affero
+* more details. pour plus de détails.
+*
+* You should have received Vous devriez avoir reçu une
+* a copy of the GNU Affero copie de la Licence Générale
+* General Public License along Publique GNU Affero avec
+* with OpenCADC. If not, see OpenCADC ; si ce n’est
+* . pas le cas, consultez :
+* .
+*
+************************************************************************
+*/
+
+package org.opencadc.vospace.db;
+
+import ca.nrc.cadc.io.ResourceIterator;
+import java.net.URI;
+import java.util.UUID;
+import org.apache.log4j.Logger;
+import org.opencadc.inventory.db.AbstractDAO;
+import org.opencadc.inventory.db.HarvestStateDAO;
+import org.opencadc.inventory.db.SQLGenerator;
+import org.opencadc.vospace.ContainerNode;
+import org.opencadc.vospace.DataNode;
+import org.opencadc.vospace.Node;
+import org.springframework.jdbc.BadSqlGrammarException;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+/**
+ *
+ * @author pdowler
+ */
+public class NodeDAO extends AbstractDAO {
+ private static final Logger log = Logger.getLogger(NodeDAO.class);
+
+ public NodeDAO() {
+ super(true);
+ }
+
+ public NodeDAO(boolean origin) {
+ super(origin);
+ }
+
+ public NodeDAO(HarvestStateDAO harvestStateDAO) {
+ super(harvestStateDAO);
+ }
+
+ @Override
+ public void put(Node val) {
+ super.put(val);
+ }
+
+ @Override
+ public Node lock(Node n) {
+ if (n == null) {
+ throw new IllegalArgumentException("entity cannot be null");
+ }
+ // override because Node has subclasses: force base class here
+ return super.lock(Node.class, n.getID());
+ }
+
+ public Node get(UUID id) {
+ checkInit();
+ return super.get(Node.class, id);
+ }
+
+ public Node get(ContainerNode parent, String name) {
+ checkInit();
+ log.debug("GET: " + parent.getID() + " + " + name);
+ long t = System.currentTimeMillis();
+
+ try {
+ JdbcTemplate jdbc = new JdbcTemplate(dataSource);
+ SQLGenerator.NodeGet get = (SQLGenerator.NodeGet) gen.getEntityGet(Node.class);
+ get.setPath(parent, name);
+ return get.execute(jdbc);
+ } catch (BadSqlGrammarException ex) {
+ handleInternalFail(ex);
+ } finally {
+ long dt = System.currentTimeMillis() - t;
+ log.debug("GET: " + parent.getID() + " + " + name + " " + dt + "ms");
+ }
+ throw new RuntimeException("BUG: handleInternalFail did not throw");
+ }
+
+ public DataNode getDataNode(URI storageID) {
+ checkInit();
+ log.debug("GET: " + storageID);
+ long t = System.currentTimeMillis();
+
+ try {
+ JdbcTemplate jdbc = new JdbcTemplate(dataSource);
+ SQLGenerator.NodeGet get = (SQLGenerator.NodeGet) gen.getEntityGet(Node.class);
+ get.setStorageID(storageID);
+ return (DataNode) get.execute(jdbc);
+ } catch (BadSqlGrammarException ex) {
+ handleInternalFail(ex);
+ } finally {
+ long dt = System.currentTimeMillis() - t;
+ log.debug("GET: " + storageID + " " + dt + "ms");
+ }
+ throw new RuntimeException("BUG: handleInternalFail did not throw");
+ }
+
+ public boolean isEmpty(ContainerNode parent) {
+ checkInit();
+ log.debug("isEmpty: " + parent.getID());
+ long t = System.currentTimeMillis();
+
+ try {
+ JdbcTemplate jdbc = new JdbcTemplate(dataSource);
+ SQLGenerator.NodeCount count = (SQLGenerator.NodeCount) gen.getNodeCount();
+ count.setID(parent.getID());
+ int num = count.execute(jdbc);
+ return (num == 0);
+ } catch (BadSqlGrammarException ex) {
+ handleInternalFail(ex);
+ } finally {
+ long dt = System.currentTimeMillis() - t;
+ log.debug("isEmpty: " + parent.getID() + " " + dt + "ms");
+ }
+ throw new RuntimeException("BUG: handleInternalFail did not throw");
+ }
+
+ public void delete(UUID id) {
+ super.delete(Node.class, id);
+ }
+
+ /**
+ * Get iterator of child nodes.
+ *
+ * @param parent the container node to list
+ * @param limit max number of nodes to return, or null
+ * @param start list starting point, or null
+ * @return iterator of child nodes matching the arguments
+ */
+ public ResourceIterator iterator(ContainerNode parent, Integer limit, String start) {
+ if (parent == null) {
+ throw new IllegalArgumentException("childIterator: parent cannot be null");
+ }
+ log.debug("iterator: " + parent.getID());
+
+ checkInit();
+ long t = System.currentTimeMillis();
+
+ try {
+ SQLGenerator.NodeIteratorQuery iter = (SQLGenerator.NodeIteratorQuery) gen.getEntityIteratorQuery(Node.class);
+ iter.setParent(parent);
+ iter.setStart(start);
+ iter.setLimit(limit);
+ return iter.query(dataSource);
+ } catch (BadSqlGrammarException ex) {
+ handleInternalFail(ex);
+ } finally {
+ long dt = System.currentTimeMillis() - t;
+ log.debug("iterator: " + parent.getID() + " " + dt + "ms");
+ }
+ throw new RuntimeException("BUG: should be unreachable");
+ }
+}
diff --git a/cadc-inventory-db/src/main/resources/inventory.HarvestState.sql b/cadc-inventory-db/src/main/resources/generic.HarvestState.sql
similarity index 94%
rename from cadc-inventory-db/src/main/resources/inventory.HarvestState.sql
rename to cadc-inventory-db/src/main/resources/generic.HarvestState.sql
index bccc28f9f..4a52dfcda 100644
--- a/cadc-inventory-db/src/main/resources/inventory.HarvestState.sql
+++ b/cadc-inventory-db/src/main/resources/generic.HarvestState.sql
@@ -3,6 +3,7 @@ create table .HarvestState (
resourceID varchar(128),
curLastModified timestamp,
curID uuid,
+ instanceID uuid,
lastModified timestamp not null,
metaChecksum varchar(136) not null,
diff --git a/cadc-inventory-db/src/main/resources/inventory.ModelVersion.sql b/cadc-inventory-db/src/main/resources/generic.ModelVersion.sql
similarity index 100%
rename from cadc-inventory-db/src/main/resources/inventory.ModelVersion.sql
rename to cadc-inventory-db/src/main/resources/generic.ModelVersion.sql
diff --git a/cadc-inventory-db/src/main/resources/generic.PreauthKeyPair.sql b/cadc-inventory-db/src/main/resources/generic.PreauthKeyPair.sql
new file mode 100644
index 000000000..50fc0dcc2
--- /dev/null
+++ b/cadc-inventory-db/src/main/resources/generic.PreauthKeyPair.sql
@@ -0,0 +1,12 @@
+
+create table .PreauthKeyPair (
+ name varchar(32) not null,
+ publicKey bytea not null,
+ privateKey bytea not null,
+
+ id uuid not null primary key,
+ lastModified timestamp not null,
+ metaChecksum varchar(136) not null
+);
+
+create unique index kp_name_index on .PreauthKeyPair(name);
diff --git a/cadc-inventory-db/src/main/resources/inventory.permissions.sql b/cadc-inventory-db/src/main/resources/generic.permissions.sql
similarity index 100%
rename from cadc-inventory-db/src/main/resources/inventory.permissions.sql
rename to cadc-inventory-db/src/main/resources/generic.permissions.sql
diff --git a/cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql b/cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql
new file mode 100644
index 000000000..e0ce85860
--- /dev/null
+++ b/cadc-inventory-db/src/main/resources/inventory.upgrade-1.0.0.sql
@@ -0,0 +1,5 @@
+
+-- changes for this version: incomplete
+
+alter table .HarvestState add column instanceID uuid;
+
diff --git a/cadc-inventory-db/src/main/resources/vospace.DeletedNodeEvent.sql b/cadc-inventory-db/src/main/resources/vospace.DeletedNodeEvent.sql
new file mode 100644
index 000000000..341a670e5
--- /dev/null
+++ b/cadc-inventory-db/src/main/resources/vospace.DeletedNodeEvent.sql
@@ -0,0 +1,17 @@
+
+create table .DeletedNodeEvent (
+ -- type is immutable
+ nodeType char(1) not null,
+
+ -- support cleanup of obsolete artifacts
+ storageID varchar(512),
+
+ lastModified timestamp not null,
+ metaChecksum varchar(136) not null,
+ id uuid not null primary key
+);
+
+
+
+create index dne_lastmodified on .DeletedNodeEvent(lastModified);
+
diff --git a/cadc-inventory-db/src/main/resources/vospace.Node.sql b/cadc-inventory-db/src/main/resources/vospace.Node.sql
new file mode 100644
index 000000000..51e5d6467
--- /dev/null
+++ b/cadc-inventory-db/src/main/resources/vospace.Node.sql
@@ -0,0 +1,43 @@
+
+create table .Node (
+ -- require a special root ID value but prevent bugs
+ parentID uuid not null,
+ name varchar(512) not null,
+ nodeType char(1) not null,
+
+ ownerID varchar(256) not null,
+ isPublic boolean,
+ isLocked boolean,
+ readOnlyGroups text[],
+ readWriteGroups text[],
+
+ -- store misc props in a 2D array
+ properties text[][],
+
+ -- ContainerNode
+ inheritPermissions boolean,
+
+ -- DataNode
+ busy boolean,
+ bytesUsed bigint,
+ -- Artifact.uri and Artifact.uriBucket
+ storageID varchar(512),
+ storageBucket varchar(5),
+
+ -- LinkNode
+ target text,
+
+ lastModified timestamp not null,
+ metaChecksum varchar(136) not null,
+ id uuid not null primary key
+);
+
+-- usage: vault path navigation
+create unique index node_parent_child on .Node(parentID,name);
+
+-- usage: Node metadata-sync
+create index node_lastmodified on .Node(lastModified);
+
+-- usage: vault incremental Artifact to Node for bytesUsed
+-- usage: vault Node vs Artifact validation
+create unique index node_storageID on .Node(storageID);
\ No newline at end of file
diff --git a/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java
new file mode 100644
index 000000000..c26cf3f26
--- /dev/null
+++ b/cadc-inventory-db/src/test/java/org/opencadc/inventory/db/UtilTest.java
@@ -0,0 +1,159 @@
+/*
+************************************************************************
+******************* CANADIAN ASTRONOMY DATA CENTRE *******************
+************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
+*
+* (c) 2023. (c) 2023.
+* Government of Canada Gouvernement du Canada
+* National Research Council Conseil national de recherches
+* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
+* All rights reserved Tous droits réservés
+*
+* NRC disclaims any warranties, Le CNRC dénie toute garantie
+* expressed, implied, or énoncée, implicite ou légale,
+* statutory, of any kind with de quelque nature que ce
+* respect to the software, soit, concernant le logiciel,
+* including without limitation y compris sans restriction
+* any warranty of merchantability toute garantie de valeur
+* or fitness for a particular marchande ou de pertinence
+* purpose. NRC shall not be pour un usage particulier.
+* liable in any event for any Le CNRC ne pourra en aucun cas
+* damages, whether direct or être tenu responsable de tout
+* indirect, special or general, dommage, direct ou indirect,
+* consequential or incidental, particulier ou général,
+* arising from the use of the accessoire ou fortuit, résultant
+* software. Neither the name de l'utilisation du logiciel. Ni
+* of the National Research le nom du Conseil National de
+* Council of Canada nor the Recherches du Canada ni les noms
+* names of its contributors may de ses participants ne peuvent
+* be used to endorse or promote être utilisés pour approuver ou
+* products derived from this promouvoir les produits dérivés
+* software without specific prior de ce logiciel sans autorisation
+* written permission. préalable et particulière
+* par écrit.
+*
+* This file is part of the Ce fichier fait partie du projet
+* OpenCADC project. OpenCADC.
+*
+* OpenCADC is free software: OpenCADC est un logiciel libre ;
+* you can redistribute it and/or vous pouvez le redistribuer ou le
+* modify it under the terms of modifier suivant les termes de
+* the GNU Affero General Public la “GNU Affero General Public
+* License as published by the License” telle que publiée
+* Free Software Foundation, par la Free Software Foundation
+* either version 3 of the : soit la version 3 de cette
+* License, or (at your option) licence, soit (à votre gré)
+* any later version. toute version ultérieure.
+*
+* OpenCADC is distributed in the OpenCADC est distribué
+* hope that it will be useful, dans l’espoir qu’il vous
+* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE
+* without even the implied GARANTIE : sans même la garantie
+* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ
+* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF
+* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence
+* General Public License for Générale Publique GNU Affero
+* more details. pour plus de détails.
+*
+* You should have received Vous devriez avoir reçu une
+* a copy of the GNU Affero copie de la Licence Générale
+* General Public License along Publique GNU Affero avec
+* with OpenCADC. If not, see OpenCADC ; si ce n’est
+* . pas le cas, consultez :
+* .
+*
+************************************************************************
+*/
+
+package org.opencadc.inventory.db;
+
+import ca.nrc.cadc.util.Log4jInit;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+import org.opencadc.gms.GroupURI;
+import org.opencadc.vospace.NodeProperty;
+
+/**
+ *
+ * @author pdowler
+ */
+public class UtilTest {
+ private static final Logger log = Logger.getLogger(UtilTest.class);
+
+ static {
+ Log4jInit.setLevel("org.opencadc.inventory.db", Level.INFO);
+ }
+
+ public UtilTest() {
+ }
+
+ @Test
+ public void testParseArrayGroupURI() throws Exception {
+
+ String str = "{ivo://opencadc.org/gms?g3,"
+ + "ivo://opencadc.org/gms?g6-g7,"
+ + "ivo://opencadc.org/gms?g6.g7,"
+ + "ivo://opencadc.org/gms?g6_g7,"
+ + "ivo://opencadc.org/gms?g6~g7}";
+
+ Set dest = new TreeSet<>();
+ Util.parseArrayGroupURI(str, dest);
+ for (GroupURI u : dest) {
+ log.info("uri: " + u.getURI());
+ }
+ }
+
+ @Test
+ public void testParseArrayNodeProperty() throws Exception {
+
+ String str = "";
+ Set dest = new TreeSet<>();
+
+ log.info("raw:\n" + str + "\n");
+ Util.parseArrayProps(str, dest);
+ for (NodeProperty p : dest) {
+ log.info("prop: " + p.getKey() + " = " + p.getValue());
+ }
+
+ str = "{{ivo://ivoa.net/vospace/core#description,stuff}}";
+ dest.clear();
+ log.info("raw:\n" + str + "\n");
+ Util.parseArrayProps(str, dest);
+ for (NodeProperty p : dest) {
+ log.info("prop: " + p.getKey() + " = " + p.getValue());
+ }
+
+ str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"}}";
+ dest.clear();
+ log.info("raw:\n" + str + "\n");
+ Util.parseArrayProps(str, dest);
+ for (NodeProperty p : dest) {
+ log.info("prop: " + p.getKey() + " = " + p.getValue());
+ }
+
+ str = "{{ivo://ivoa.net/vospace/core#description,\"this is the good stuff(tm)\"},"
+ + "{ivo://ivoa.net/vospace/core#type,text/plain}}";
+ dest.clear();
+ log.info("raw:\n" + str + "\n");
+ Util.parseArrayProps(str, dest);
+ for (NodeProperty p : dest) {
+ log.info("prop: " + p.getKey() + " = " + p.getValue());
+ }
+
+ str = "{{custom:prop,\"spaces in value\"},"
+ + "{ivo://ivoa.net/vospace/core#length,123},"
+ + "{ivo://ivoa.net/vospace/core#type,text/plain},"
+ + "{\"sketchy:a,b\",comma-in-uri},"
+ + "{sketchy:funny,\"value,with,{delims}\"}}";
+
+ dest.clear();
+ log.info("raw:\n" + str + "\n");
+ Util.parseArrayProps(str, dest);
+ for (NodeProperty p : dest) {
+ log.info("prop: " + p.getKey() + " = " + p.getValue());
+ }
+ }
+}
diff --git a/cadc-inventory-server/build.gradle b/cadc-inventory-server/build.gradle
index 2c6e23ba6..814e1f972 100644
--- a/cadc-inventory-server/build.gradle
+++ b/cadc-inventory-server/build.gradle
@@ -12,20 +12,22 @@ repositories {
sourceCompatibility = 1.8
group = 'org.opencadc'
-version = '0.2.2'
+version = '0.3.0'
description = 'OpenCADC Storage Inventory server utility library'
def git_url = 'https://github.com/opencadc/storage-inventory'
dependencies {
compile 'org.opencadc:cadc-inventory:[0.7,1.0)'
- compile 'org.opencadc:cadc-inventory-db:[0.9,)'
+ compile 'org.opencadc:cadc-inventory-db:[0.15,)'
compile 'org.opencadc:cadc-util:[1.9,2.0)'
compile 'org.opencadc:cadc-rest:[1.3.14,)'
compile 'org.opencadc:cadc-gms:[1.0.4,)'
compile 'org.opencadc:cadc-cdp:[1.3,2.0)'
compile 'org.opencadc:cadc-permissions:[0.2,)'
compile 'org.opencadc:cadc-permissions-client:[0.3,)'
+ compile 'org.opencadc:cadc-vos:[2.0,3.0)'
+ compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)'
testCompile 'junit:junit:[4.0,)'
}
diff --git a/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java
new file mode 100644
index 000000000..d42c13355
--- /dev/null
+++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/GetKeyAction.java
@@ -0,0 +1,125 @@
+/*
+************************************************************************
+******************* CANADIAN ASTRONOMY DATA CENTRE *******************
+************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
+*
+* (c) 2023. (c) 2023.
+* Government of Canada Gouvernement du Canada
+* National Research Council Conseil national de recherches
+* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
+* All rights reserved Tous droits réservés
+*
+* NRC disclaims any warranties, Le CNRC dénie toute garantie
+* expressed, implied, or énoncée, implicite ou légale,
+* statutory, of any kind with de quelque nature que ce
+* respect to the software, soit, concernant le logiciel,
+* including without limitation y compris sans restriction
+* any warranty of merchantability toute garantie de valeur
+* or fitness for a particular marchande ou de pertinence
+* purpose. NRC shall not be pour un usage particulier.
+* liable in any event for any Le CNRC ne pourra en aucun cas
+* damages, whether direct or être tenu responsable de tout
+* indirect, special or general, dommage, direct ou indirect,
+* consequential or incidental, particulier ou général,
+* arising from the use of the accessoire ou fortuit, résultant
+* software. Neither the name de l'utilisation du logiciel. Ni
+* of the National Research le nom du Conseil National de
+* Council of Canada nor the Recherches du Canada ni les noms
+* names of its contributors may de ses participants ne peuvent
+* be used to endorse or promote être utilisés pour approuver ou
+* products derived from this promouvoir les produits dérivés
+* software without specific prior de ce logiciel sans autorisation
+* written permission. préalable et particulière
+* par écrit.
+*
+* This file is part of the Ce fichier fait partie du projet
+* OpenCADC project. OpenCADC.
+*
+* OpenCADC is free software: OpenCADC est un logiciel libre ;
+* you can redistribute it and/or vous pouvez le redistribuer ou le
+* modify it under the terms of modifier suivant les termes de
+* the GNU Affero General Public la “GNU Affero General Public
+* License as published by the License” telle que publiée
+* Free Software Foundation, par la Free Software Foundation
+* either version 3 of the : soit la version 3 de cette
+* License, or (at your option) licence, soit (à votre gré)
+* any later version. toute version ultérieure.
+*
+* OpenCADC is distributed in the OpenCADC est distribué
+* hope that it will be useful, dans l’espoir qu’il vous
+* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE
+* without even the implied GARANTIE : sans même la garantie
+* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ
+* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF
+* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence
+* General Public License for Générale Publique GNU Affero
+* more details. pour plus de détails.
+*
+* You should have received Vous devriez avoir reçu une
+* a copy of the GNU Affero copie de la Licence Générale
+* General Public License along Publique GNU Affero avec
+* with OpenCADC. If not, see OpenCADC ; si ce n’est
+* . pas le cas, consultez :
+* .
+*
+************************************************************************
+*/
+
+package org.opencadc.inventory.transfer;
+
+import ca.nrc.cadc.rest.InlineContentHandler;
+import ca.nrc.cadc.rest.RestAction;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import javax.naming.Context;
+import javax.naming.InitialContext;
+import javax.naming.NamingException;
+import org.apache.log4j.Logger;
+import org.opencadc.inventory.PreauthKeyPair;
+
+/**
+ * Simple GET action that finds a PreauthKeyPair via JNDI and writes
+ * the binary public key to the output.
+ *
+ * @author pdowler
+ */
+public class GetKeyAction extends RestAction {
+ private static final Logger log = Logger.getLogger(GetKeyAction.class);
+
+ public GetKeyAction() {
+ super();
+ }
+
+ @Override
+ protected InlineContentHandler getInlineContentHandler() {
+ return null;
+ }
+
+ @Override
+ public void doAction() throws Exception {
+ String jndiPreauthKeys = appName + "-" + PreauthKeyPair.class.getName();
+ Context ctx = new InitialContext();
+ try {
+ log.debug("lookup: " + jndiPreauthKeys);
+ PreauthKeyPair keys = (PreauthKeyPair) ctx.lookup(jndiPreauthKeys);
+ log.debug("found: " + keys);
+ byte[] pub = keys.getPublicKey();
+ syncOutput.setHeader("content-length", pub.length);
+ syncOutput.setHeader("content-type", "application/octet-stream");
+ syncOutput.setCode(200);
+ try (OutputStream ostream = syncOutput.getOutputStream()) {
+ ostream.write(pub);
+ ostream.flush();
+ }
+ } catch (NamingException ex) {
+ syncOutput.setHeader("content-type", "test/plain");
+ syncOutput.setCode(404);
+ try (OutputStream ostream = syncOutput.getOutputStream()) {
+ PrintWriter w = new PrintWriter(ostream);
+ w.println("not found: key signing disabled");
+ w.flush();
+ w.close();
+ }
+ }
+ }
+}
diff --git a/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java
similarity index 69%
rename from raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java
rename to cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java
index 2c4aa7e98..d8fca2f08 100644
--- a/raven/src/main/java/org/opencadc/raven/ProtocolsGenerator.java
+++ b/cadc-inventory-server/src/main/java/org/opencadc/inventory/transfer/ProtocolsGenerator.java
@@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
-* (c) 2023. (c) 2023.
+* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -65,7 +65,7 @@
************************************************************************
*/
-package org.opencadc.raven;
+package org.opencadc.inventory.transfer;
import ca.nrc.cadc.cred.client.CredUtil;
import ca.nrc.cadc.net.HttpGet;
@@ -76,10 +76,6 @@
import ca.nrc.cadc.reg.Interface;
import ca.nrc.cadc.reg.Standards;
import ca.nrc.cadc.reg.client.RegistryClient;
-import ca.nrc.cadc.vos.Direction;
-import ca.nrc.cadc.vos.Protocol;
-import ca.nrc.cadc.vos.Transfer;
-import ca.nrc.cadc.vos.VOS;
import ca.nrc.cadc.vosi.Availability;
import java.io.File;
import java.io.IOException;
@@ -109,6 +105,10 @@
import org.opencadc.permissions.ReadGrant;
import org.opencadc.permissions.TokenTool;
import org.opencadc.permissions.WriteGrant;
+import org.opencadc.vospace.VOS;
+import org.opencadc.vospace.transfer.Direction;
+import org.opencadc.vospace.transfer.Protocol;
+import org.opencadc.vospace.transfer.Transfer;
/**
* Class for generating protocol lists corresponding to transfer requests.
@@ -119,112 +119,88 @@ public class ProtocolsGenerator {
private static final Logger log = Logger.getLogger(ProtocolsGenerator.class);
+ public static final URI SECURITY_EMBEDDED_TOKEN = URI.create("https://www.opencadc.org/std/storage#embedded-token");
+
public static final String ARTIFACT_ID_HDR = "x-artifact-id"; // matches minoc.HeadAction.ARTIFACT_ID_HDR
private final ArtifactDAO artifactDAO;
private final DeletedArtifactEventDAO deletedArtifactEventDAO;
- private final String user;
- private final File publicKeyFile;
- private final File privateKeyFile;
+
private final Map siteAvailabilities;
private final Map siteRules;
- private final StorageResolver storageResolver;
- private final boolean preventNotFound;
- // for use by FilesAction subclasses
+ /**
+ * Optional StorageResolver to resolve Artifact.uri to an external data provider.
+ */
+ public StorageResolver storageResolver;
+
+ /**
+ * Optional flag to enable prevention of 404 NotFound failure due to eventual
+ * consistency. Setting this to true will cause the code to make HTTP HEAD
+ * requests to all known storage sites looking for an artifact that is not
+ * in the local database.
+ */
+ public boolean preventNotFound = false;
+
+ /**
+ * Optional user value to put into generated preauth token.
+ */
+ public String user;
+
+ /**
+ * Optional TokenTool to generate and inject preauth tokens into otherwise anon URL.
+ */
+ public TokenTool tokenGen;
+
+ /**
+ * Optional restriction so that all anon URLs must have a preauth token.
+ */
+ public boolean requirePreauthAnon = false;
+
+ // for use by FilesAction subclasses to enhance logging
boolean storageResolverAdded = false;
-
- public ProtocolsGenerator(ArtifactDAO artifactDAO, File publicKeyFile, File privateKeyFile, String user,
- Map siteAvailabilities, Map siteRules,
- boolean preventNotFound, StorageResolver storageResolver) {
+ public ProtocolsGenerator(ArtifactDAO artifactDAO, Map siteAvailabilities, Map siteRules) {
this.artifactDAO = artifactDAO;
this.deletedArtifactEventDAO = new DeletedArtifactEventDAO(this.artifactDAO);
- this.user = user;
- this.publicKeyFile = publicKeyFile;
- this.privateKeyFile = privateKeyFile;
this.siteAvailabilities = siteAvailabilities;
this.siteRules = siteRules;
- this.preventNotFound = preventNotFound;
- this.storageResolver = storageResolver;
+ }
+
+ public boolean getStorageResolverAdded() {
+ return storageResolverAdded;
}
- List getProtocols(Transfer transfer) throws ResourceNotFoundException, IOException {
+ public List getProtocols(Transfer transfer) throws ResourceNotFoundException, IOException {
+ return getProtocols(transfer, null);
+ }
+
+ public List getProtocols(Transfer transfer, String filenameOverride) throws ResourceNotFoundException, IOException {
String authToken = null;
URI artifactURI = transfer.getTargets().get(0); // see PostAction line ~127
- if (publicKeyFile != null && privateKeyFile != null) {
+ if (tokenGen != null) {
// create an auth token
- TokenTool tk = new TokenTool(publicKeyFile, privateKeyFile);
if (transfer.getDirection().equals(Direction.pullFromVoSpace)) {
- authToken = tk.generateToken(artifactURI, ReadGrant.class, user);
+ authToken = tokenGen.generateToken(artifactURI, ReadGrant.class, user);
} else {
- authToken = tk.generateToken(artifactURI, WriteGrant.class, user);
+ authToken = tokenGen.generateToken(artifactURI, WriteGrant.class, user);
}
}
List protos = null;
if (Direction.pullFromVoSpace.equals(transfer.getDirection())) {
- protos = doPullFrom(artifactURI, transfer, authToken);
- } else {
+ // filename override only on GET
+ protos = doPullFrom(artifactURI, transfer, authToken, filenameOverride);
+ } else if (Direction.pushToVoSpace.equals(transfer.getDirection())) {
protos = doPushTo(artifactURI, transfer, authToken);
+ } else {
+ throw new UnsupportedOperationException("unexpected transfer direction: " + transfer.getDirection().getValue());
+
}
return protos;
}
- static void prioritizePullFromSites(List storageSites) {
- // contains the algorithm for prioritizing storage sites to pull from.
-
- // was: prefer read/write sites to put less load on a read-only "seeder" site during migration
- //storageSites.sort((site1, site2) -> Boolean.compare(!site1.getAllowWrite(), !site2.getAllowWrite()));
-
- // random
- Collections.shuffle(storageSites);
- }
-
- Artifact getRemoteArtifact(URL location, URI artifactURI) {
- try {
- HttpGet head = new HttpGet(location, true);
- head.setHeadOnly(true);
- head.setReadTimeout(10000);
- head.run();
- if (head.getResponseCode() != 200) {
- // caught at the end of the method
- throw new RuntimeException("Unsuccessful HEAD request: " + head.getResponseCode());
- }
- UUID id = UUID.fromString(head.getResponseHeader(ARTIFACT_ID_HDR));
- Artifact result = new
- Artifact(id, artifactURI, head.getDigest(), head.getLastModified(), head.getContentLength());
- result.contentType = head.getContentType();
- result.contentEncoding = head.getContentEncoding();
- return result;
- } catch (Throwable t) {
- log.debug("Could not retrieve artifact " + artifactURI.toASCIIString() + " from " + location, t);
- return null;
- }
- }
-
- private Capability getFilesCapability(StorageSite storageSite) {
- if (!isAvailable(storageSite.getResourceID())) {
- log.warn("storage site is offline: " + storageSite.getResourceID());
- return null;
- }
- Capability filesCap = null;
- try {
- RegistryClient regClient = new RegistryClient();
- Capabilities caps = regClient.getCapabilities(storageSite.getResourceID());
- filesCap = caps.findCapability(Standards.SI_FILES);
- if (filesCap == null) {
- log.warn("service: " + storageSite.getResourceID() + " does not provide " + Standards.SI_FILES);
- }
- } catch (ResourceNotFoundException ex) {
- log.warn("storage site not found: " + storageSite.getResourceID());
- } catch (Exception ex) {
- log.warn("storage site not responding (capabilities): " + storageSite.getResourceID(), ex);
- }
- return filesCap;
- }
-
- Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set storageSites, String authToken) {
+ public Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set storageSites, String authToken) {
Artifact result = null;
for (StorageSite storageSite : storageSites) {
// check if site is currently offline
@@ -296,86 +272,166 @@ Artifact getUnsyncedArtifact(URI artifactURI, Transfer transfer, Set