Fix async mdn (#278)

* Make the default example use a signed MDN and synchronous MDN

* Fix logging.

* Fix the Content-Type not geting set correctly due to a bug in the
MimeBodyPart lib.
Rework the code for creating an MDN response.
Make it easier to understand.

* Fix logging and change check for Async MDN

* Allow canHandle response specifically for Resender modules.

* Improve the error message when a parameter cannot be evaluated.

* REmove unnecessary dependency on Sendermodule

* Prevent setting Content-Length to fix ASYNC MDN sending.

* Upfgrade packages for security patches

* Support embedded properties dynamic variables within propterties

* Fix warnings

* ERwork creating the MDN to fix issues wit hthe content type

* Fix extracting the MimeBodyPart from the HTTP response.

* Parametrise the Content-Type header to make it easier to find in code

* Fix getting the report from the received MDN response.

* Add a helper method to get directory pollers by sender and receiver ID

* Fix replacing embedded properties

* Make it easier to configure the Asyn MDN receiver correctly.

* Use a base folder for storage as a property to make tests easier.

* Use the async MDN URL property to minimise fixes when changed.

* Do Sync and Async MDN testing.

* Additional release notes on use of properties.

* Support setting executable permissions in packaging zip file.

* New release version

* Change history and release notes.

* Fix spelling error.

* Move control parameters into base class for better usability. Support
NOT replacing param if not found.

* Prevent replacing missing parser tokens when loading Properties.

RELEASE-NOTES.md

@@ -1,24 +1,21 @@
 #              OpenAS2 Server
-#              Version 3.0.1
+#              Version 3.1.0
 #              RELEASE NOTES
 -----
-The OpenAS2 project is pleased to announce the release of OpenAS2 3.0.1
+The OpenAS2 project is pleased to announce the release of OpenAS2 3.1.0
 
-The release download file is: OpenAS2Server-3.0.1.zip
+The release download file is: OpenAS2Server-3.1.0.zip
 
 The zip file contains a PDF document (OpenAS2HowTo.pdf) providing information on installing and using the application.
 ## NOTE: Testing covers Java 8 to 17. The application should work for older versions down to Java 7 but they are not tested as part of the CI/CD pipeline.
 
-Version 3.0.1 - 2022-05-30
-This is a  and minor enhancement release:
+Version 3.1.0 - 2022-06-04
+This is a minor enhancement and bugfix release:
        **IMPORTANT NOTE**: Please review upgrade notes below if you are upgrading
 
-
-  1. Fix partnership directory poller cache not refreshing when partnerships file is modified on a live system.
-  2. Fix the support for bypassing proxy hosts.
-  3. Fix access via the web application. See the README here for using it: https://github.com/OpenAS2/OpenAs2App/blob/master/WebUI/README.md
-  4. Support setting the valid days count and start date for certificate generation in the gen_p12_key_par.sh script.
-  5. Provide a more useful message when parsing dynamic strings throws an error.
+  1. Support embedded $properties.xxx$ within property element values.
+  2. Support setting unix executables in ZIP file when packaging new release in Maven.
+  3. Fix asynchronous MDN sending and receiving.
 
 
 ##Upgrade Notes

Remote/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.0.1</version>
+        <version>3.1.0</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

Server/pom.xml

@@ -7,7 +7,7 @@
         <!-- DO NOT CHANGE THIS "groupId" WITHOUT CHANGING XMLSession.getManifestAttributes.MANIFEST_VENDOR_ID_ATTRIB -->
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.0.1</version>
+        <version>3.1.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -28,6 +28,7 @@
         <help.file.src>${project.basedir}/../docs/OpenAS2HowTo.pdf</help.file.src>
         <release.notes>${project.basedir}/../RELEASE-NOTES.md</release.notes>
         <release.history>${project.basedir}/../changes.txt</release.history>
+        <package.assembly.dir>${project.build.directory}/dist</package.assembly.dir>
     </properties>
 
     <distributionManagement>
@@ -65,39 +66,51 @@
                         <phase>package</phase>
                         <configuration>
                             <target>
-                                <copy todir="${project.build.directory}/dist/bin"
+                                <copy todir="${package.assembly.dir}/bin"
                                       verbose="true">
                                     <fileset dir="${project.basedir}/src/bin"/>
                                 </copy>
-                                <!--Copy config directory to assembly directory -->
-                                <copy todir="${project.build.directory}/dist/config"
+                                <echo message="OS Name: ${os.name}"/>
+								<exec executable="sh" osfamily="unix">
+								    <arg value="-c"/> 
+								    <arg line="'chmod uga+x ${package.assembly.dir}/bin/*.sh'"/>
+								</exec>
+								<!--Copy config directory to assembly directory -->
+                                <copy todir="${package.assembly.dir}/config"
                                       verbose="true">
                                     <fileset dir="${project.basedir}/src/config"/>
                                 </copy>
                                 <!--Copy resources directory to assembly directory -->
-                                <copy todir="${project.build.directory}/dist/resources"
+                                <copy todir="${package.assembly.dir}/resources"
                                       verbose="true">
                                     <fileset dir="${project.basedir}/src/resources"/>
                                 </copy>
-                                <!--Copy artifact to build lib dicetory -->
+                                <!--Copy artifact to build lib directory -->
                                 <copy
                                     file="${project.build.directory}/${project.build.finalName}.jar"
-                                    todir="${project.build.directory}/dist/lib" verbose="true"/>
+                                    todir="${package.assembly.dir}/lib" verbose="true"/>
                                 <!-- Add the help PDF version to the distro -->
                                 <copy file="${help.file.src}"
-                                      todir="${project.build.directory}/dist" verbose="true"/>
+                                      todir="${package.assembly.dir}" verbose="true"/>
                                 <!-- Add the release notes files -->
                                 <copy file="${release.notes}"
-                                      todir="${project.build.directory}/dist" verbose="true"/>
+                                      todir="${package.assembly.dir}" verbose="true"/>
                                 <copy file="${release.history}"
-                                      todir="${project.build.directory}/dist" verbose="true"/>
+                                      todir="${package.assembly.dir}" verbose="true"/>
                                 <!-- Add the remote jar for connecting to the server command interface -->
                                 <copy file="${remote.connect.jar}"
-                                      todir="${project.build.directory}/dist/bin/remote" verbose="true"/>
+                                      todir="${package.assembly.dir}/bin/remote" verbose="true"/>
                                 <!-- create distribution package -->
                                 <zip
                                     destfile="${project.basedir}/dist/${project.dist.package.name}"
-                                    basedir="${project.build.directory}/dist"/>
+                                    basedir="${package.assembly.dir}"
+                                    update="true">
+                                        <zipfileset dir="${package.assembly.dir}" encoding="UTF-8"
+                                                 includes="**/*.sh" filemode="755"/>
+                                        <zipfileset dir="${package.assembly.dir}" encoding="UTF-8"
+                                                 excludes="**/*.sh"/>
+
+                                </zip>
                             </target>
                         </configuration>
                     </execution>

Server/src/config/config.xml

@@ -4,7 +4,8 @@
                log_date_format="yyyy-MM-dd HH:mm:ss.SSS"
                sql_timestamp_format="yyyy-MM-dd HH:mm:ss.SSS"
                as2_message_id_format="OPENAS2-$date.ddMMyyyyHHmmssZ$-$rand.UUID$@$msg.sender.as2_id$_$msg.receiver.as2_id$"
-               as2_async_mdn_url="http://localhost:10081"
+               async_mdn_receiver_port="10081"
+               as2_async_mdn_url="http://localhost:$properties.async_mdn_receiver_port$"
                as2_receive_message_filename_fallback="$rand.shortUUID$"
                console.logger.enabled="true"
                file.logger.enabled="true"
@@ -163,7 +164,7 @@
               errorformat="sender.as2_id, receiver.as2_id, headers.message-id"/>
       <module enabled="$properties.module.AS2MDNReceiverModule.http.enabled$"
               classname="org.openas2.processor.receiver.AS2MDNReceiverModule"
-              port="10081"/>
+              port="$properties.async_mdn_receiver_port$"/>
       <module enabled="$properties.module.AS2MDNReceiverModule.https.enabled$"
               classname="org.openas2.processor.receiver.AS2MDNReceiverModule"
               port="10444"

Server/src/config/partnerships.xml

@@ -28,14 +28,14 @@
         <attribute name="subject" value="File $attributes.filename$ sent from $sender.name$ to $receiver.name$"/>
         <attribute name="as2_url" value="http://localhost:10080"/>
         <attribute name="as2_mdn_to" value="edi@myCompany.com"/>
-        <attribute name="as2_mdn_options" value="none"/>
+        <attribute name="as2_mdn_options" value="signed-receipt-protocol=optional, pkcs7-signature; signed-receipt-micalg=optional, $attribute.sign$"/>
         <!--
-                   value="signed-receipt-protocol=optional, pkcs7-signature; signed-receipt-micalg=optional, $attribute.sign$"/>
+            For an unsigned MDN use this value for as2_mdn_options:       value="none"
         -->
         <!--  For enabling ASYNC MDN uncomment this attribute and set this to the URL partner must send MDN back to.
               Example below uses a property from the config.xml to facilitate centralised management of the URL
-         -->
             <attribute name="as2_receipt_option" value="$properties.as2_async_mdn_url$"/> 
+         -->
         <attribute name="encrypt" value="3DES"/>
         <attribute name="sign" value="SHA256"/>
         <attribute name="resend_max_retries" value="3"/>

Server/src/main/java/org/openas2/BaseSession.java

@@ -183,6 +183,20 @@ public DirectoryPollingModule getPartnershipPoller(String partnershipName) {
         return null;
     }
 
+    public DirectoryPollingModule getPartnershipPoller(String senderAs2Id, String receiverAs2Id) {
+        // search by the defaults since the partnershipName used as the key is not consistent due to config.xml defined pollers issue
+        // so iterate over all and search by the AS2 ID's in the defaults element
+        for (Map.Entry<String, Map<String, Object>> entry : polledDirectories.entrySet()) {
+            Map<String, Object> meta = entry.getValue();
+            DirectoryPollingModule pollerModule = (DirectoryPollingModule)meta.get("pollerInstance");
+            String defaults = pollerModule.getParameters().get("defaults");
+            if (defaults != null && defaults.contains("receiver.as2_id=" + receiverAs2Id) && defaults.contains("sender.as2_id=" + senderAs2Id)) {
+                    return pollerModule;
+            }
+        }
+        return null;
+    }
+
     public void loadPartnershipPoller(Node moduleNode, String partnershipName, String configSource) throws OpenAS2Exception {
         DirectoryPollingModule procmod = (DirectoryPollingModule) XMLUtil.getComponent(moduleNode, this);
         String pollerDir = procmod.getParameters().get(DirectoryPollingModule.PARAM_OUTBOX_DIRECTORY);

Server/src/main/java/org/openas2/XMLSession.java

@@ -8,6 +8,9 @@
 import org.openas2.lib.xml.PropertyReplacementFilter;
 import org.openas2.logging.LogManager;
 import org.openas2.logging.Logger;
+import org.openas2.params.CompositeParameters;
+import org.openas2.params.InvalidParameterException;
+import org.openas2.params.ParameterParser;
 import org.openas2.partner.PartnershipFactory;
 import org.openas2.processor.Processor;
 import org.openas2.processor.ProcessorModule;
@@ -133,8 +136,9 @@ protected void load(InputStream in) throws Exception {
      * Finally checks if an additional property file was provided and loads those.
      * 
      * @param propNode - the "properties" element of the configuration file containing property values
+     * @throws InvalidParameterException 
      */
-    private void loadProperties(Node propNode) {
+    private void loadProperties(Node propNode) throws InvalidParameterException {
         LOGGER.info("Loading properties...");
 
         Map<String, String> properties = XMLUtil.mapAttributes(propNode, false);
@@ -174,6 +178,23 @@ private void loadProperties(Node propNode) {
                 }
             }
         }
+        /* Process all loaded values in case they reference other properties in the value
+           Use the properties object instead of Properties so we only parse the properties that were in the config.xml
+           so that we can use system property values to replace config.xml properties.
+         */
+        // Pass "true" to ignore unmatched parse ID's in case the properties contain dynamic parameters needed for JIT evaluation
+        CompositeParameters parser = new CompositeParameters(true);
+        parser.setReturnParamStringForMissingParsers(true);
+        for (Map.Entry<String, String> entry : properties.entrySet()) {
+            String val = entry.getValue();
+            String parsedVal = ParameterParser.parse(val, parser);
+            // Parser will return empty string if there is an unmatched parser ID in the string
+            if (parsedVal.length() > 0 && !val.equals(parsedVal)) {
+                String key = entry.getKey();
+                // Put the changed value into the Properties set
+                Properties.setProperty(key, parsedVal);
+            }
+        }
     }
 
     private void loadCertificates(Node rootNode) throws OpenAS2Exception {

Server/src/main/java/org/openas2/app/OpenAS2WindowsService.java

@@ -3,6 +3,7 @@
 import java.util.Arrays;
 
 public class OpenAS2WindowsService {
+    @SuppressWarnings("unused")
     private static OpenAS2Server server;
 
     public static boolean stop = false;

Server/src/main/java/org/openas2/app/partner/AddPartnershipCommand.java

@@ -11,16 +11,10 @@
 import org.openas2.util.XMLUtil;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
-import org.w3c.dom.Node;
 
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.FactoryConfigurationError;
-import javax.xml.parsers.ParserConfigurationException;
-
 /**
  * adds a new partnership entry in partneship store
  *

Server/src/main/java/org/openas2/app/partner/ViewPartnerCommand.java

@@ -38,6 +38,7 @@ protected CommandResult execute(PartnershipFactory partFx, Object[] params) thro
             while (parts.hasNext()) {
                 String partName = parts.next();
                 if (partName.equals(name)) {
+                    @SuppressWarnings("unchecked")
                     Map<Object, Object> partDefs = (Map<Object, Object>) partFx.getPartners().get(name);
                     //String out = name + "\n" + partDefs.toString();
                     return new CommandResult(CommandResult.TYPE_OK, partDefs);

Server/src/main/java/org/openas2/cmd/processor/RestCommandProcessor.java

@@ -21,7 +21,6 @@
 import org.openas2.cmd.CommandResult;
 import org.openas2.cmd.processor.restapi.ApiResource;
 import org.openas2.cmd.processor.restapi.AuthenticationRequestFilter;
-import org.openas2.cmd.processor.restapi.CORSFilter;
 import org.openas2.cmd.processor.restapi.LoggerRequestFilter;
 
 import java.io.IOException;

Server/src/main/java/org/openas2/cmd/processor/restapi/AuthenticationRequestFilter.java

@@ -12,10 +12,6 @@
 import org.bouncycastle.util.encoders.Base64;
 import org.openas2.cmd.CommandResult;
 
-import jakarta.annotation.security.DenyAll;
-import jakarta.annotation.security.PermitAll;
-import jakarta.annotation.security.RolesAllowed;
-
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.container.ResourceInfo;
 import jakarta.ws.rs.container.ContainerRequestFilter;
@@ -25,9 +21,7 @@
 import jakarta.ws.rs.core.MultivaluedMap;
 import jakarta.ws.rs.core.Response;
 import jakarta.ws.rs.ext.Provider;
-import java.lang.reflect.Method;
 import java.security.Principal;
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -48,7 +42,7 @@ public class AuthenticationRequestFilter implements ContainerRequestFilter {
     private static final String AUTHORIZATION_PROPERTY = "Authorization";
     private static final String AUTHENTICATION_SCHEME = "Basic";
     private static final CommandResult ERROR_ACCESS_DENIED = new CommandResult(CommandResult.TYPE_ERROR, "You cannot access this resource");
-    private static final CommandResult ERROR_ACCESS_FORBIDDEN = new CommandResult(CommandResult.TYPE_ERROR, "Access blocked for all users !!");
+    //private static final CommandResult ERROR_ACCESS_FORBIDDEN = new CommandResult(CommandResult.TYPE_ERROR, "Access blocked for all users !!");
     private static String adminUsername;
     private static String adminPassword;
     private final Log logger = LogFactory.getLog(AuthenticationRequestFilter.class.getSimpleName());

Server/src/main/java/org/openas2/lib/helper/BCCryptoHelper.java

@@ -44,6 +44,7 @@
 import org.openas2.DispositionException;
 import org.openas2.OpenAS2Exception;
 import org.openas2.Session;
+import org.openas2.lib.util.MimeUtil;
 import org.openas2.message.AS2Message;
 import org.openas2.message.Message;
 import org.openas2.processor.receiver.AS2ReceiverModule;
@@ -326,7 +327,7 @@ public AttributeTable getAttributes(@SuppressWarnings("rawtypes") Map parameters
         MimeBodyPart tmpBody = new MimeBodyPart();
         tmpBody.setContent(signedData);
         // Content-type header is required, unit tests fail badly on async MDNs if not set.
-        tmpBody.setHeader("Content-Type", signedData.getContentType());
+        tmpBody.setHeader(MimeUtil.MIME_CONTENT_TYPE_KEY, signedData.getContentType());
         return tmpBody;
     }
 
@@ -369,6 +370,7 @@ public MimeBodyPart verifySignature(MimeBodyPart part, Certificate cert) throws
             SignerInformation signer = it.next();
             if (logger.isTraceEnabled()) {
                 try { // Code block below does not do null-checks or other encoding error checking.
+                    @SuppressWarnings("unchecked")
                     Map<Object, Attribute> attrTbl = signer.getSignedAttributes().toHashtable();
                     StringBuilder strBuf = new StringBuilder();
                     for (Map.Entry<Object, Attribute> pair : attrTbl.entrySet()) {

Server/src/main/java/org/openas2/lib/util/GeneralUtil.java

@@ -25,6 +25,7 @@ public static String[] convertKeys(Map<?, Object> map) {
         return map.keySet().toArray(keys);
     }
 
+    @SuppressWarnings("unchecked")
     public static String convert(Map<?, ?> map, String valueDelimiter, String pairDelimiter) {
         StringBuffer strBuf = new StringBuffer();
         Iterator<?> it = map.entrySet().iterator();
@@ -40,6 +41,7 @@ public static String convert(Map<?, ?> map, String valueDelimiter, String pairDe
         return strBuf.toString();
     }
 
+    @SuppressWarnings("unchecked")
     public static Object getKey(Map<?, Object> map, Object value) {
         Iterator<?> it = map.entrySet().iterator();
         Map.Entry<Object, Object> entry;

Server/src/main/java/org/openas2/lib/util/MimeUtil.java

@@ -15,6 +15,10 @@
 
 
 public class MimeUtil {
+    public static final String MIME_CONTENT_TYPE_KEY = "Content-Type";
+    public static final String MIME_CONTENT_DISPOSITION_KEY = "Content-Type";
+    public static final String MIME_CONTENT_TRANSFER_ENCODING_KEY = "Content-Transfer-Encoding";
+
     private static final String HEADER_VALUE_SEPARATOR = ", ";
 
     public static int getContentLength(InternetHeaders headers) throws IOException {
@@ -56,16 +60,16 @@ public static MimeBodyPart createMimeBodyPart(byte[] data, String contentType, S
         // create a MimeBodyPart and set up it's content and content headers
         MimeBodyPart part = new MimeBodyPart();
         part.setDataHandler(new DataHandler(new ByteArrayDataSource(data, contentType, null)));
-        part.setHeader("Content-Type", contentType);
-        part.setHeader("Content-Transfer-Encoding", contentTransferEncoding);
+        part.setHeader(MIME_CONTENT_TYPE_KEY, contentType);
+        part.setHeader(MIME_CONTENT_TRANSFER_ENCODING_KEY, contentTransferEncoding);
 
         return part;
     }
 
     public static MimeBodyPart createMimeBodyPart(MimeMultipart multipart) throws MessagingException {
         MimeBodyPart part = new MimeBodyPart();
         part.setContent(multipart);
-        part.setHeader("Content-Type", multipart.getContentType());
+        part.setHeader(MIME_CONTENT_TYPE_KEY, multipart.getContentType());
 
         return part;
     }
@@ -93,7 +97,7 @@ public static MimeBodyPart readMimeBodyPart(InputStream source, InternetHeaders
         if (contentTransferEncoding == null) {
             contentTransferEncoding = Session.DEFAULT_CONTENT_TRANSFER_ENCODING;
         }
-        return createMimeBodyPart(data, getHeader(headers, "Content-Type"), contentTransferEncoding);
+        return createMimeBodyPart(data, getHeader(headers, MIME_CONTENT_TYPE_KEY), contentTransferEncoding);
     }
 
     public static String toString(MimeBodyPart mbp, boolean addDelimiterText) throws IOException, MessagingException {