Merge pull request #546 from andamian/CADC-12946

`raven` generates and persists pre-auth keys (CADC 12946)

raven/README.md

@@ -41,16 +41,9 @@ overhead for the genuine not-found cases.
 The following optional keys configure raven to use external service(s) to obtain grant information in order
 to perform authorization checks:
 ```
-# keys to generate pre-auth URLs to minoc
-org.opencadc.raven.publicKeyFile={public key file name}
-org.opencadc.raven.privateKeyFile={private key file name}
-
 org.opencadc.raven.readGrantProvider={resourceID of a permission granting service}
 org.opencadc.raven.writeGrantProvider={resourceID of a permission granting service}
 ```
-The optional _privateKeyFile_ is used to sign pre-auth URLs (one-time token included in URL) so that a `minoc` service does not
-have to repeat permission checks. The _publicKeyFile_ is not currently used but may be required in future (either exported via URL
-or used to check if `minoc` services have the right key before generating pre-auth URLs: TBD).
 
 The optional _readGrantProvider_ and _writeGrantProvider_ keys configure minoc to call other services to get grants (permissions) for 
 operations. Multiple values of the permission granting service resourceID(s) may be provided by including multiple property 

raven/build.gradle

@@ -26,17 +26,17 @@ war {
 }
 
 dependencies {
-    compile 'org.opencadc:cadc-util:[1.10.2,2.0)'
+    compile 'org.opencadc:cadc-util:[1.10.3,2.0)'
     compile 'org.opencadc:cadc-log:[1.1.6,2.0)'
     compile 'org.opencadc:cadc-registry:[1.7,)'
     compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)'
     compile 'org.opencadc:cadc-rest:[1.3.14,)'
     compile 'org.opencadc:cadc-cdp:[1.0,)'
     compile 'org.opencadc:cadc-gms:[1.0.4,)'
     compile 'org.opencadc:cadc-inventory:[0.9.4,2.0)'
-    compile 'org.opencadc:cadc-inventory-db:[0.14.5,0.15)'
-    compile 'org.opencadc:cadc-inventory-server:[0.3,)'
-    compile 'org.opencadc:cadc-permissions:[0.3.1,)'
+    compile 'org.opencadc:cadc-inventory-db:[0.15.0,)'
+    compile 'org.opencadc:cadc-inventory-server:[0.3.0,)'
+    compile 'org.opencadc:cadc-permissions:[0.3.5,)'
     compile 'org.opencadc:cadc-permissions-client:[0.3,)'
     compile 'org.opencadc:cadc-vos:[1.2,2.0)'
 

raven/src/intTest/README.md

@@ -46,5 +46,5 @@ org.opencadc.raven.putPreference=@SITE3
 org.opencadc.raven.consistency.preventNotFound=true
 
 # external resolvers
-ca.nrc.cadc.net.StorageResolver=mast ca.nrc.cadc.caom2.artifact.resolvers.MastResolver
+ca.nrc.cadc.net.StorageResolver=ca.nrc.cadc.caom2.artifact.resolvers.MastResolver
 ```

raven/src/intTest/java/org/opencadc/raven/NegotiationTest.java

@@ -101,6 +101,8 @@
 import org.opencadc.inventory.db.StorageSiteDAO;
 import org.opencadc.inventory.db.version.InitDatabase;
 import org.opencadc.inventory.transfer.ProtocolsGenerator;
+import org.opencadc.permissions.TokenTool;
+import org.opencadc.permissions.WriteGrant;
 import org.opencadc.vospace.VOS;
 import org.opencadc.vospace.transfer.Direction;
 import org.opencadc.vospace.transfer.Protocol;
@@ -835,4 +837,59 @@ public Object run() throws Exception {
             Assert.fail("unexpected exception: " + e);
         }
     }
+
+    @Test
+    public void testPreauthURL() throws Exception {
+
+        StorageSite site = new StorageSite(CONSIST_RESOURCE_ID, "site1", true, true);
+        try {
+            // get raven pub key
+            URL pubKeyURL = anonURL.toURI().resolve("./pubkey").toURL();
+            log.debug("raven pub key URL: " + pubKeyURL);
+            HttpGet getPubKey = new HttpGet(pubKeyURL, true);
+            getPubKey.run();
+            Assert.assertEquals(200, getPubKey.getResponseCode());
+            Assert.assertNull(getPubKey.getThrowable());
+            final byte[] buffer = new byte[64 * 1024];
+            final InputStream inputStream = getPubKey.getInputStream();
+            int bytesRead = inputStream.read(buffer);
+            if (bytesRead == buffer.length) {
+                // might be incomplete
+                throw new RuntimeException("BUG - pubkey input buffer is too small");
+            }
+            byte[] pubKey = new byte[bytesRead];
+            System.arraycopy(buffer, 0, pubKey, 0, bytesRead);
+
+            TokenTool tokenTool = new TokenTool(pubKey);
+            siteDAO.put(site);
+            Subject.doAs(userSubject, new PrivilegedExceptionAction<Object>() {
+                public Object run() throws Exception {
+                    URI artifactURI = URI.create("cadc:TEST/" + UUID.randomUUID() + ".fits");
+                    Protocol p = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
+                    p.setSecurityMethod(Standards.SECURITY_METHOD_ANON);
+                    Transfer transfer = new Transfer(artifactURI, Direction.pushToVoSpace);
+                    transfer.getProtocols().add(p);
+                    transfer.version = VOS.VOSPACE_21;
+                    Transfer negotiated = negotiate(transfer);
+                    List<String> endPoints = negotiated.getAllEndpoints(VOS.PROTOCOL_HTTPS_PUT.toASCIIString());
+                    Assert.assertEquals(1, endPoints.size());
+                    URI endPoint = URI.create(endPoints.get(0));
+                    String path = endPoint.getPath();
+                    int columnIndex = path.indexOf(":");
+                    Assert.assertTrue(columnIndex>0);
+                    String tmp = path.substring(0, columnIndex); // ignore artifact URI slashes
+                    String[] pathComp = tmp.split("/");
+                    String token = pathComp[pathComp.length - 2];
+                    URI resArtifactURI = URI.create(pathComp[pathComp.length - 1] + path.substring(columnIndex));
+                    log.debug("Result artifact URI: " + resArtifactURI);
+                    Assert.assertEquals(artifactURI, resArtifactURI);
+                    log.debug("token: " + token);
+                    tokenTool.validateToken(token, artifactURI, WriteGrant.class);
+                    return negotiated;
+                }
+            });
+        } finally {
+            siteDAO.delete(site.getID());
+        }
+    }
 }

raven/src/main/java/org/opencadc/raven/ArtifactAction.java

@@ -73,10 +73,11 @@
 import ca.nrc.cadc.rest.RestAction;
 import ca.nrc.cadc.rest.Version;
 import ca.nrc.cadc.util.MultiValuedProperties;
+import ca.nrc.cadc.util.RsaSignatureGenerator;
 import ca.nrc.cadc.vosi.Availability;
-import java.io.File;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.security.KeyPair;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -85,7 +86,9 @@
 import javax.naming.InitialContext;
 import javax.naming.NamingException;
 import org.apache.log4j.Logger;
+import org.opencadc.inventory.PreauthKeyPair;
 import org.opencadc.inventory.db.ArtifactDAO;
+import org.opencadc.inventory.db.PreauthKeyPairDAO;
 import org.opencadc.inventory.transfer.StorageSiteRule;
 import org.opencadc.permissions.ReadGrant;
 import org.opencadc.permissions.TokenTool;
@@ -111,7 +114,9 @@ public abstract class ArtifactAction extends RestAction {
 
     // immutable state set in constructor
     protected final ArtifactDAO artifactDAO;
+    protected final PreauthKeyPairDAO preauthKeyPairDAO;
     protected final TokenTool tokenGen;
+    protected final byte[] publicKey;
     protected final List<URI> readGrantServices = new ArrayList<>();
     protected final List<URI> writeGrantServices = new ArrayList<>();
     protected StorageResolver storageResolver;
@@ -128,8 +133,10 @@ public abstract class ArtifactAction extends RestAction {
         this.authenticateOnly = false;
         this.tokenGen = null;
         this.artifactDAO = null;
+        this.preauthKeyPairDAO = null;
         this.preventNotFound = false;
         this.storageResolver = null;
+        this.publicKey = null;
     }
 
     protected ArtifactAction() {
@@ -174,26 +181,27 @@ protected ArtifactAction() {
 
         initResolver();
 
-        // technically, raven only needs the private key to generate pre-auth tokens
-        // but both are specified here for clarity
-        // - in principle, raven could export it's public key and minoc(s) could retrieve it
-        // - for now, minoc(s) need to be configured with the public key to validate pre-auth
-
-        String pubkeyFileName = props.getFirstPropertyValue(RavenInitAction.PUBKEYFILE_KEY);
-        String privkeyFileName = props.getFirstPropertyValue(RavenInitAction.PRIVKEYFILE_KEY);
-        if (pubkeyFileName == null && privkeyFileName == null) {
-            log.debug("public/private key preauth not enabled by config");
-            this.tokenGen = null;
-        } else {
-            File publicKeyFile = new File(System.getProperty("user.home") + "/config/" + pubkeyFileName);
-            File privateKeyFile = new File(System.getProperty("user.home") + "/config/" + privkeyFileName);
-            if (!publicKeyFile.exists() || !privateKeyFile.exists()) {
-                throw new IllegalStateException("invalid config: missing public/private key pair files -- " + publicKeyFile + " | " + privateKeyFile);
+        Map<String, Object> config = RavenInitAction.getDaoConfig(props);
+        this.preauthKeyPairDAO = new PreauthKeyPairDAO();
+        preauthKeyPairDAO.setConfig(config); // connectivity tested
+        PreauthKeyPair preauthKP = preauthKeyPairDAO.get(RavenInitAction.PREAUTH_KEYPAIR);
+        if (preauthKP == null) {
+            log.info("Generate the pre-auth key pair");
+            KeyPair keyPair = RsaSignatureGenerator.getKeyPair(4096); //TODO size?
+            preauthKP = new PreauthKeyPair(RavenInitAction.PREAUTH_KEYPAIR, keyPair.getPublic().getEncoded(), keyPair.getPrivate().getEncoded());
+            try {
+                preauthKeyPairDAO.put(preauthKP);
+            } catch (Exception ex) {
+                throw new IllegalStateException("Could not persist the pre-auth key pair.", ex);
             }
-            this.tokenGen = new TokenTool(publicKeyFile, privateKeyFile);
+            this.tokenGen = new TokenTool(keyPair.getPublic().getEncoded(), keyPair.getPrivate().getEncoded());
+            this.publicKey = keyPair.getPublic().getEncoded();
+        } else {
+            log.debug("Use existing pre-auth key pair");
+            this.tokenGen = new TokenTool(preauthKP.getPublicKey(), preauthKP.getPrivateKey());
+            this.publicKey = preauthKP.getPublicKey();
         }
 
-        Map<String, Object> config = RavenInitAction.getDaoConfig(props);
         this.artifactDAO = new ArtifactDAO();
         artifactDAO.setConfig(config); // connectivity tested
 
@@ -283,7 +291,6 @@ void init() throws Exception {
     protected String getServerImpl() {
         // no null version checking because fail to build correctly can't get past basic testing
         Version v = getVersionFromResource();
-        String ret = "storage-inventory/raven-" + v.getMajorMinor();
-        return ret;
+        return "storage-inventory/raven-" + v.getMajorMinor();
     }
 }

raven/src/main/java/org/opencadc/raven/GetFilesAction.java

@@ -140,9 +140,11 @@ URI getFirstURL() throws ResourceNotFoundException, IOException {
         proto.setSecurityMethod(Standards.SECURITY_METHOD_ANON);
         transfer.getProtocols().add(proto);
 
-        ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.tokenGen,
-                                                       this.user, this.siteAvailabilities, this.siteRules,
-                                                       this.preventNotFound, this.storageResolver);
+        ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.siteAvailabilities, this.siteRules);
+        pg.tokenGen = this.tokenGen;
+        pg.user = this.user;
+        pg.preventNotFound = this.preventNotFound;
+        pg.storageResolver = this.storageResolver;
         List<Protocol> protos = pg.getProtocols(transfer);
         if (protos.isEmpty()) {
             throw new ResourceNotFoundException("not available: " + artifactURI);

raven/src/main/java/org/opencadc/raven/HeadFilesAction.java

@@ -78,7 +78,6 @@
 import org.opencadc.inventory.db.StorageSiteDAO;
 import org.opencadc.inventory.transfer.ProtocolsGenerator;
 import org.opencadc.permissions.ReadGrant;
-import org.opencadc.permissions.TokenTool;
 import org.opencadc.vospace.VOS;
 import org.opencadc.vospace.transfer.Direction;
 import org.opencadc.vospace.transfer.Protocol;
@@ -112,8 +111,12 @@ public void doAction() throws Exception {
         if (artifact == null) {
             if (this.preventNotFound) {
                 // check known storage sites
-                ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.tokenGen,
-                        this.user, this.siteAvailabilities, this.siteRules, this.preventNotFound, this.storageResolver);
+                ProtocolsGenerator pg = new ProtocolsGenerator(
+                        this.artifactDAO, this.siteAvailabilities, this.siteRules);
+                pg.tokenGen = this.tokenGen;
+                pg.user = this.user;
+                pg.preventNotFound = this.preventNotFound;
+                pg.storageResolver = this.storageResolver;
                 StorageSiteDAO storageSiteDAO = new StorageSiteDAO(artifactDAO);
                 Transfer transfer = new Transfer(artifactURI, Direction.pullFromVoSpace);
                 Protocol proto = new Protocol(VOS.PROTOCOL_HTTPS_GET);

raven/src/main/java/org/opencadc/raven/PostAction.java

@@ -160,9 +160,11 @@ public InlineContentHandler.Content accept(String name, String contentType, Inpu
     public void doAction() throws Exception {
         initAndAuthorize();
 
-        ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.tokenGen,
-                                                       this.user, this.siteAvailabilities, this.siteRules,
-                                                       this.preventNotFound, this.storageResolver);
+        ProtocolsGenerator pg = new ProtocolsGenerator(this.artifactDAO, this.siteAvailabilities, this.siteRules);
+        pg.tokenGen = this.tokenGen;
+        pg.user = this.user;
+        pg.preventNotFound = this.preventNotFound;
+        pg.storageResolver = this.storageResolver;
         Transfer ret = new Transfer(artifactURI, transfer.getDirection());
         // TODO: change from pg.getProtocols(transfer) to pg.getResolvedTransfer(transfer)??
         ret.getProtocols().addAll(pg.getProtocols(transfer));

raven/src/main/java/org/opencadc/raven/RavenInitAction.java

@@ -71,7 +71,6 @@
 import ca.nrc.cadc.util.MultiValuedProperties;
 import ca.nrc.cadc.util.PropertiesReader;
 import ca.nrc.cadc.util.StringUtil;
-import java.io.File;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
@@ -105,8 +104,7 @@ public class RavenInitAction extends InitAction {
 
     static final String SCHEMA_KEY = RAVEN_KEY + ".inventory.schema";
 
-    static final String PUBKEYFILE_KEY = RAVEN_KEY + ".publicKeyFile";
-    static final String PRIVKEYFILE_KEY = RAVEN_KEY + ".privateKeyFile";
+    static final String PREAUTH_KEYPAIR = RAVEN_KEY + ".keyPair";
     static final String READ_GRANTS_KEY = RAVEN_KEY + ".readGrantProvider";
     static final String WRITE_GRANTS_KEY = RAVEN_KEY + ".writeGrantProvider";
 
@@ -129,7 +127,6 @@ public void doInit() {
         initConfig();
         initDAO();
         initGrantProviders();
-        initKeys();
         initStorageSiteRules();
         initAvailabilityCheck();
     }
@@ -180,22 +177,6 @@ void initGrantProviders() {
         }
         log.info("initGrantProviders: OK");
     }
-
-    void initKeys() {
-        log.info("initKeys: START");
-        String pubkeyFileName = props.getFirstPropertyValue(RavenInitAction.PUBKEYFILE_KEY);
-        String privkeyFileName = props.getFirstPropertyValue(RavenInitAction.PRIVKEYFILE_KEY);
-        if (pubkeyFileName == null && privkeyFileName == null) {
-            log.info("initKeys: disabled OK");
-            return;
-        }
-        File publicKeyFile = new File(System.getProperty("user.home") + "/config/" + pubkeyFileName);
-        File privateKeyFile = new File(System.getProperty("user.home") + "/config/" + privkeyFileName);
-        if (!publicKeyFile.exists() || !privateKeyFile.exists()) {
-            throw new IllegalStateException("invalid config: missing public/private key pair files -- " + publicKeyFile + " | " + privateKeyFile);
-        }
-        log.info("initKeys: OK");
-    }
 
     void initStorageSiteRules() {
         log.info("initStorageSiteRules: START");
@@ -267,24 +248,6 @@ static MultiValuedProperties getConfig() {
             sb.append("OK");
         }
 
-        // optional
-        String pub = mvp.getFirstPropertyValue(RavenInitAction.PUBKEYFILE_KEY);
-        sb.append("\n\t").append(RavenInitAction.PUBKEYFILE_KEY).append(": ");
-        if (pub == null) {
-            sb.append("MISSING");
-        } else {
-            sb.append("OK");
-        }
-
-        String priv = mvp.getFirstPropertyValue(RavenInitAction.PRIVKEYFILE_KEY);
-        sb.append("\n\t").append(RavenInitAction.PRIVKEYFILE_KEY).append(": ");
-        if (priv == null) {
-            sb.append("MISSING");
-        } else {
-            sb.append("OK");
-        }
-
-
         if (!ok) {
             throw new IllegalStateException(sb.toString());
         }

raven/src/main/webapp/WEB-INF/web.xml

@@ -47,6 +47,16 @@
         <load-on-startup>2</load-on-startup>
     </servlet>
 
+  <servlet>
+    <servlet-name>PubKeyServlet</servlet-name>
+    <servlet-class>ca.nrc.cadc.rest.RestServlet</servlet-class>
+    <init-param>
+      <param-name>get</param-name>
+      <param-value>org.opencadc.raven.GetPubKeyAction</param-value>
+    </init-param>
+    <load-on-startup>4</load-on-startup>
+  </servlet>
+
   <servlet>
     <servlet-name>FilesServlet</servlet-name>
     <servlet-class>ca.nrc.cadc.rest.RestServlet</servlet-class>
@@ -106,6 +116,11 @@
     <servlet-name>FilesServlet</servlet-name>
     <url-pattern>/files/*</url-pattern>
   </servlet-mapping>
+
+  <servlet-mapping>
+    <servlet-name>PubKeyServlet</servlet-name>
+    <url-pattern>/pubkey</url-pattern>
+  </servlet-mapping>
 
   <!-- Availability servlet endpoint -->
   <servlet-mapping>

raven/src/test/java/org/opencadc/raven/RavenInitActionTest.java

@@ -124,8 +124,6 @@ public void doInit() {
             String message = e.getMessage();
             log.debug(message);
             Assert.assertTrue(message.contains(String.format("%s: MISSING", RavenInitAction.SCHEMA_KEY)));
-            Assert.assertTrue(message.contains(String.format("%s: MISSING", RavenInitAction.PUBKEYFILE_KEY)));
-            Assert.assertTrue(message.contains(String.format("%s: MISSING", RavenInitAction.PRIVKEYFILE_KEY)));
         } finally {
             System.setProperty("user.home", USER_HOME);
         }