Minor bufixes partner poller and proxy host (#276)

* Fix bad merge

* Updated documentation for rest API

* Make error message more useful

* Support setting the valid days and start date as ENV vars

* Fix missing closing $ in parameter string.

* Merge upstream

* Remove unused imports

* Make all modules require supporting the getModuleAction method to
facilitate testing.

* Support getModuleAction method.

* Fix cache not being reset when partnerships changed.

* Rework the test so that it is less brittle.

* Remove redundant pollerConfig attribute on partnership element.

* Add helper method for finding modules.

* Start the partnership pollers after a partnership file refresh.

* Documentation for new release

RELEASE-NOTES.md

@@ -1,27 +1,31 @@
 #              OpenAS2 Server
-#              Version 3.0.0
+#              Version 3.0.1
 #              RELEASE NOTES
 -----
-The OpenAS2 project is pleased to announce the release of OpenAS2 3.0.0
+The OpenAS2 project is pleased to announce the release of OpenAS2 3.0.1
 
-The release download file is: OpenAS2Server-3.0.0.zip
+The release download file is: OpenAS2Server-3.0.1.zip
 
 The zip file contains a PDF document (OpenAS2HowTo.pdf) providing information on installing and using the application.
 ## NOTE: Testing covers Java 8 to 17. The application should work for older versions down to Java 7 but they are not tested as part of the CI/CD pipeline.
 
-Version 3.0.0 - 2022-04-10
-This is a major release:
+Version 3.0.1 - 2022-05-30
+This is a  and minor enhancement release:
        **IMPORTANT NOTE**: Please review upgrade notes below if you are upgrading
 
 
-  1. The directory polling module is no longer configured in the config.xml file. By default, any partnership will get a polling module unless explicitly disabled using the appropriate attribute. See documentation for details.
-  2. Support for expired certificates. Some partners seem to continue to use expired certificates when sending messages. The expiration date can be ignored and still validate the signature by adding an attribute to the partnership to ignore the expiration. See documentation for details.
+  1. Fix partnership directory poller cache not refreshing when partnerships file is modified on a live system.
+  2. Fix the support for bypassing proxy hosts.
+  3. Fix access via the web application. See the README here for using it: https://github.com/OpenAS2/OpenAs2App/blob/master/WebUI/README.md
+  4. Support setting the valid days count and start date for certificate generation in the gen_p12_key_par.sh script.
+  5. Provide a more useful message when parsing dynamic strings throws an error.
 
 
 ##Upgrade Notes
  See the openAS2HowTo appendix for the general process on upgrading OpenAS2.
 
-### For this upgrade, the old config will work without change but it is strongly recommended that you follow these steps to convert your existing configuration to the new format:
+### Upgrading to 3.xx from 2.x (or older) version:
+## NOTE: The old config will work without change but it is strongly recommended that you follow these steps to convert your existing configuration to the new format as it provides a cleaner and less complicated setup and the old config will eventually be discontinued
       1. Follow the instructions for specific versions between your current version and this version as defined below before executing the commands below to convert your existing config.xml and partnerships.xml files to use the enhanced poller configuration.
       2. Open a terminal window (command window in Windows)
       3. Change to the <install>/config directory of the new version.

Remote/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.0.0</version>
+        <version>3.0.1</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

Server/pom.xml

@@ -7,7 +7,7 @@
         <!-- DO NOT CHANGE THIS "groupId" WITHOUT CHANGING XMLSession.getManifestAttributes.MANIFEST_VENDOR_ID_ATTRIB -->
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.0.0</version>
+        <version>3.0.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

Server/src/bin/gen_p12_key_par.sh

@@ -1,20 +1,24 @@
 #!/bin/sh
 
-CertValidDays=3650
 x=`basename $0`
 if test $# -ne 4; then
   echo "Generate a certificate to a PKCS12 key store."
   echo "You must supply a target key store without the extension (extension will be added as .p12) and an alias for generated certificate."
-  echo "usage: ${x} <target keystore> <cert alias> <sigalg> <distinguished name>"
+  echo "usage: ${x} <target keystore> <cert alias> <sigalg> <distinguished name> <start date>"
   echo "            WHERE"
   echo "               target keystore = name of the target keystore file without .p12 extension"
   echo "               cert alias = alias name used to store the created digital certificate in the keystore"
   echo "               sigalg = signing algorithm for the digital certificate ... SHA256, SHA512 etc"
   echo "               distinguished name = a string in the format:"
   echo "                                       CN=<cName>, OU=<orgUnit>, O=<org>, L=<city>, S=<state>, C=<countryCode>"
+  echo "            The start date and number of valid daysi for the certificate can be passed in as environment variables:"
+  echo "               CERT_START_DATE = date the certificate should be valid from in format \"yyyy/MM/dd [HH:mm:ss]\""
+  echo "               CERT_VALID_DAYS = number of days the certificate should be valid for. defaults to 730 days (~2 years)"
 
   echo ""
-  echo "       eg. $0 as2_certs partnera SHA256 \"CN=as2.partnerb.com, OU=QA, O=PartnerA, L=New York, S=New York, C=US\""
+  echo "eg.  >export CERT_START_DATE=2022/11/31"
+  echo "     >export CERT_VALID_DAYS=365"
+  echo "     >$0 as2_certs partnera SHA256 \"CN=as2.partnerb.com, OU=QA, O=PartnerA, L=New York, S=New York, C=US\""
   echo "     Expected OUTPUT: as2_certs.p12 -  keystore containing both public and private key"
   echo "                     partnera.cer - public key certificate file ."
   exit 1
@@ -25,6 +29,17 @@ certAlias=$2
 sigAlg="$3withRSA"
 dName=$4
 
+if [ -z $CERT_VALID_DAYS ]; then
+  CertValidDays=730
+else
+  CertValidDays=$CERT_VALID_DAYS
+fi
+AdditionalGenArgs=""
+if [ -n "$CERT_START_DATE" ]; then
+  AdditionalGenArgs="-startdate $CERT_START_DATE "
+  PRE_GEN_MSG_ADDITIONAL=" with a start date of $CERT_START_DATE"
+fi
+
 if [ -z $JAVA_HOME ]; then
   OS=$(uname -s)
 
@@ -46,21 +61,22 @@ if [ -z $JAVA_HOME ]; then
 fi
 
 echo "Generate a certificate to a PKCS12 key store."
-echo "Generating certificate:  using alias $certAlias to ${tgtStore}.p12"
+echo "Generating certificate:  using alias $certAlias to ${tgtStore}.p12 $PRE_GEN_MSG_ADDITIONAL"
 read -p "Do you wish to execute this request? [Y/N]" Response
 if [  $Response != "Y" -a $Response != "y"  ] ; then
   exit 1
 fi
 
 read -p "Enter password for keystore:" ksPwd
-$JAVA_HOME/bin/keytool -genkeypair -alias $certAlias -validity $CertValidDays  -keyalg RSA -sigalg $sigAlg -keystore ${tgtStore}.p12 -storepass $ksPwd -storetype pkcs12 -dname "$dName"
+$JAVA_HOME/bin/keytool -genkeypair -alias $certAlias -validity $CertValidDays  -keyalg RSA -sigalg $sigAlg -keystore ${tgtStore}.p12 -storepass $ksPwd -storetype pkcs12 $AdditionalGenArgs -dname "$dName"
 if [ "$?" != 0 ]; then
 	echo ""
     echo "Failed to create a keystore. See errors above to correct the problem."
     exit 1
 fi
 
-$JAVA_HOME/bin/keytool -selfcert -alias $certAlias -validity $CertValidDays  -sigalg $sigAlg -keystore ${tgtStore}.p12 -storepass $ksPwd -storetype pkcs12
+#$JAVA_HOME/bin/keytool -selfcert -alias $certAlias -validity $CertValidDays  -sigalg $sigAlg -keystore ${tgtStore}.p12 -storepass $ksPwd -storetype pkcs12
+$JAVA_HOME/bin/keytool -selfcert -alias $certAlias $AdditionalGenArgs -validity $CertValidDays  -sigalg $sigAlg -keystore ${tgtStore}.p12 -storepass $ksPwd -storetype pkcs12
 if [ "$?" != 0 ]; then
 	echo ""
     echo "Failed to self certifiy the certificates in the keystore. See errors above to correct the problem."

Server/src/bin/start-openas2.bat

@@ -42,8 +42,9 @@ for /R %OPENAS2_BASE_DIR%/lib %%a in (*.jar) do (
 )
 set LIB_JARS=".!LIB_JARS!"
 setLocal disableDelayedExpansion
-rem    
-rem echo Running: "%JAVA%" %EXTRA_PARMS%  -cp .;%LIB_JARS% org.openas2.app.OpenAS2Server "%OPENAS2_BASE_DIR%/config/config.xml"
+rem  Include the bin dir so that commons-logging.properties is found
+CLASSPATH=.;%LIB_JARS%;%OPENAS2_BASE_DIR%/bin
+rem echo Running: "%JAVA%" %EXTRA_PARMS%  -cp %CLASSPATH% org.openas2.app.OpenAS2Server "%OPENAS2_BASE_DIR%/config/config.xml"
 "%JAVA%" %EXTRA_PARMS%  -cp .;%LIB_JARS% org.openas2.app.OpenAS2Server "%OPENAS2_BASE_DIR%/config/config.xml"
 
 :warn

Server/src/bin/start-openas2.sh

@@ -65,6 +65,8 @@ if [ -z $JAVA_HOME ]; then
 fi
 # Expand the classpath instead of using file globbing expansion in the java command as it seems to mess with Mailcap loading
 CLASSPATH=$(echo "${binDir}/../lib/"*".jar" | tr ' ' ':')
+# Include the bin dir so that icommons-logging.properties is always found
+CLASSPATH=${CLASSPATH}:${binDir}
 CMD=$(echo "${JAVA_HOME}/bin/java ${PWD_OVERRIDE} ${EXTRA_PARMS} -cp .:${CLASSPATH} org.openas2.app.OpenAS2Server")
 echo
 echo Running ${CMD}
@@ -81,4 +83,4 @@ else
   ${CMD}
   RETVAL="$?"
 fi
-exit $RETVAL
+exit $RETVAL

Server/src/config/partnerships.xml

@@ -14,7 +14,7 @@
              x509_alias="partnerb"
              email="as2msgs@partnerb.com"/>
 
-    <partnership name="MyCompany-to-PartnerA" pollerEnabled="false">
+    <partnership name="MyCompany-to-PartnerA">
         <sender name="MyCompany"/>
         <receiver name="PartnerA"/>
         <!--  The pollerConfig node defines whether this partnership will receive files to send to the "receiver" partner via a directory poller.

Server/src/main/java/org/openas2/BaseSession.java

@@ -14,7 +14,10 @@
 
 import javax.activation.CommandMap;
 import javax.activation.MailcapCommandMap;
+
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 
@@ -39,10 +42,12 @@ public BaseSession() throws OpenAS2Exception {
     @Override
     public void start() throws OpenAS2Exception {
         getProcessor().startActiveModules();
+        startPartnershipPollers();
     }
 
     @Override
     public void stop() throws Exception {
+        destroyPartnershipPollers();
         for (Component component : components.values()) {
             component.destroy();
         }
@@ -132,35 +137,70 @@ private void trackPollerModule(String pollerDir, String partnershipName, String
         polledDirectories.put(pollerDir, meta);
     }
 
+    public void startPartnershipPollers() throws OpenAS2Exception {
+        for (Map.Entry<String, Map<String, Object>> entry : polledDirectories.entrySet()) {
+            Map<String, Object> meta = entry.getValue();
+            DirectoryPollingModule poller = (DirectoryPollingModule) meta.get("pollerInstance");
+            LOGGER.trace("Starting directory poller:" + meta);
+            poller.start();
+        }
+    }
+
     public void destroyPartnershipPollers() {
+        LOGGER.trace("Destroying partnership pollers...");
+        List<String> stoppedPollerKeys = new ArrayList<String>();
         for (Map.Entry<String, Map<String, Object>> entry : polledDirectories.entrySet()) {
             Map<String, Object> meta = entry.getValue();
-            ProcessorModule poller = (ProcessorModule) meta.get("pollerInstance");
+            DirectoryPollingModule poller = (DirectoryPollingModule) meta.get("pollerInstance");
             try {
-                poller.destroy();
+                LOGGER.trace("Destroying poller:" + meta);
+                if (poller.isRunning()) {
+                        poller.stop();
+                        poller = null;
+                }
+                // track the removed pollers keys to update the map after iteration to avoid concurrent modification error
+                stoppedPollerKeys.add(entry.getKey());
             } catch (Exception e) {
-                // something went wrong stoppint it - report and keep going
+                // something went wrong stopping it - report and keep going but make sure the key is still removed
                 LOGGER.error("Failed to stop a partnership poller for directory " + entry.getKey() + ": " + meta, e);
+                stoppedPollerKeys.add(entry.getKey());
+            } 
+        }
+        for (String pollerKey : stoppedPollerKeys) {
+            // Remove the poller entry in the map now that we have killed the active poller
+            polledDirectories.remove(pollerKey);
+            LOGGER.trace("Removed poller from cache map:" + pollerKey);
+        }
+    }
+
+    public DirectoryPollingModule getPartnershipPoller(String partnershipName) {
+        for (Map.Entry<String, Map<String, Object>> entry : polledDirectories.entrySet()) {
+            Map<String, Object> meta = entry.getValue();
+            if (partnershipName.equals(meta.get("partnershipName"))) {
+                return (DirectoryPollingModule) meta.get("pollerInstance");
             }
-
         }
+        return null;
     }
 
     public void loadPartnershipPoller(Node moduleNode, String partnershipName, String configSource) throws OpenAS2Exception {
-        ProcessorModule procmod = (ProcessorModule) XMLUtil.getComponent(moduleNode, this);
+        DirectoryPollingModule procmod = (DirectoryPollingModule) XMLUtil.getComponent(moduleNode, this);
         String pollerDir = procmod.getParameters().get(DirectoryPollingModule.PARAM_OUTBOX_DIRECTORY);
         try {
             checkPollerModuleConfig(pollerDir);
         } catch (OpenAS2Exception oae) {
             try {
-                procmod.destroy();
+                procmod.stop();
+                procmod = null;
             } catch (Exception e) {
-                throw new OpenAS2Exception("Failed to destroy a partnershipthat has config errors", e);
+                throw new OpenAS2Exception("Failed to destroy a partnership poller that has config errors", e);
             }
-            throw new OpenAS2Exception("Partnership cannot be loaded because there is a configuration error: " + partnershipName, oae);
+            throw new OpenAS2Exception("Partnership poller cannot be loaded because there is a configuration error: " + partnershipName, oae);
         }
+        /*
         Processor proc = (Processor)getComponent(Processor.COMPID_PROCESSOR);
         proc.getModules().add(procmod);
+        */
         trackPollerModule(pollerDir, partnershipName, configSource, procmod);
     }
 

Server/src/main/java/org/openas2/Session.java

@@ -96,6 +96,7 @@ public interface Session {
      */
     public void loadPartnershipPoller(Node moduleNode, String partnershipName, String configSource) throws OpenAS2Exception;
 
+    public void startPartnershipPollers() throws OpenAS2Exception;
     public void destroyPartnershipPollers();
 
     String getBaseDirectory();

Server/src/main/java/org/openas2/params/ParameterParser.java

@@ -127,7 +127,7 @@ public String format(String format) throws InvalidParameterException {
             prev = next + 1;
             next = format.indexOf('$', prev);
             if (next == -1) {
-                throw new InvalidParameterException("Invalid key (missing closing $)");
+                throw new InvalidParameterException("Invalid key (missing closing $) parsing this string: " + format);
             }
 
             // If we have just $$ then output $, else we have $xxx$, lookup xxx

Server/src/main/java/org/openas2/partner/XMLPartnershipFactory.java

@@ -219,6 +219,9 @@ public void loadPartnership(Map<String, Object> partners, List<Partnership> part
             String[] requiredPollerAttributes = {"enabled"};
             Map<String, String> partnershipPollerCfgAttributes = XMLUtil.mapAttributes(pollerCfgNode, requiredPollerAttributes);
             if ("true".equalsIgnoreCase(partnershipPollerCfgAttributes.get("enabled"))) {
+                if (logger.isTraceEnabled()) {
+                        logger.trace("Found partnership poller for partnership: " + name);
+                }
                 // Create a copy of the base config node
                 Node basePollerConfigNode = ((XMLSession)getSession()).getBasePartnershipPollerConfig();
                 if (basePollerConfigNode == null) {
@@ -291,7 +294,9 @@ public void schedule(ScheduledExecutorService executor) throws OpenAS2Exception
         new FileMonitorAdapter() {
             @Override
             public void onConfigFileChanged() throws OpenAS2Exception {
+                logger.info("Partnerships file change detected. Starting refresh...");
                 refresh();
+                getSession().startPartnershipPollers();
                 logger.info("Partnerships file change detected - Partnerships Reloaded");
             }
         }.scheduleIfNeed(executor, new File(getFilename()), getRefreshInterval(), TimeUnit.SECONDS);

Server/src/main/java/org/openas2/processor/BaseActiveModule.java

@@ -21,6 +21,7 @@ public boolean isRunning() {
     public abstract boolean healthcheck(List<String> failures);
 
     public boolean canHandle(String action, Message msg, Map<String, Object> options) {
+        // Active modules that are continuously doing their job do not handle stuff on demand
         return false;
     }
 

Server/src/main/java/org/openas2/processor/BaseProcessorModule.java

@@ -1,8 +1,34 @@
 package org.openas2.processor;
 
+import java.util.Map;
+
 import org.openas2.BaseComponent;
+import org.openas2.OpenAS2Exception;
+import org.openas2.message.Message;
+import org.openas2.params.InvalidParameterException;
+
+
+public class BaseProcessorModule extends BaseComponent implements ProcessorModule {
+
+    @Override
+    public String getModuleAction() {
+        try {
+            return getParameter(MODULE_ACTION_ATTRIB, false);
+        } catch (InvalidParameterException e) {
+            // For now return null but will change so that all modules must set the parameter
+            return null;
+        }
+    }
 
+    @Override
+    public boolean canHandle(String action, Message msg, Map<String, Object> options) {
+        return action.equalsIgnoreCase(getModuleAction());
+    }
 
-public abstract class BaseProcessorModule extends BaseComponent implements ProcessorModule {
+    @Override
+    public void handle(String action, Message msg, Map<String, Object> options) throws OpenAS2Exception {
+        throw new OpenAS2Exception("Module must implement the handle() method: " + this.getClass().getName());
+
+    }
 
 }

Server/src/main/java/org/openas2/processor/DefaultProcessor.java

@@ -74,6 +74,24 @@ public void handle(String action, Message msg, Map<String, Object> options) thro
         }
     }
 
+    public List<ProcessorModule> getModulesSupportingAction(String action) {
+        List<ProcessorModule> modules = new ArrayList<ProcessorModule>();
+        Iterator<ProcessorModule> moduleIt = getModules().iterator();
+        ProcessorModule module;
+
+        if (logger.isTraceEnabled()) {
+            logger.trace("Processor searching for module handlers for action: " + action);
+        }
+
+        while (moduleIt.hasNext()) {
+            module = moduleIt.next();
+            if (action.equals(module.getModuleAction())) {
+                modules.add(module);
+            }
+        }
+        return modules;
+    }
+
     public void startActiveModules() throws OpenAS2Exception {
 
         List<ActiveModule> activeModules = getActiveModules();

Server/src/main/java/org/openas2/processor/Processor.java

@@ -23,4 +23,6 @@ public interface Processor extends Component {
     List<ActiveModule> getActiveModulesByClass(Class<?> clazz);
 
     boolean checkActiveModules(List<String> failures);
+
+    public List<ProcessorModule> getModulesSupportingAction(String action);
 }

Server/src/main/java/org/openas2/processor/ProcessorModule.java

@@ -8,8 +8,11 @@
 
 
 public interface ProcessorModule extends Component {
+    static final String MODULE_ACTION_ATTRIB = "module_action"; 
 
     boolean canHandle(String action, Message msg, Map<String, Object> options);
 
+    String getModuleAction();
+
     void handle(String action, Message msg, Map<String, Object> options) throws OpenAS2Exception;
 }