Fix sentdir processing logic (#279)

* Adapt the HTTP executor method signature  to make it easier to support
non-chunked sending.

* REmove the incorrect handling of sentDir and make sure the files are
deleted when an error condition occurs.

* Support sending non-chunked transfers based on configuration.

* Rework the class in preparation for supporting a more efficient process
for very large files.

* Changes to support preventing chunked transfer

* Enhanced documentation for semnding without chunking.

* Version upgrades

* Release notes for 3.2.0

RELEASE-NOTES.md

@@ -1,21 +1,20 @@
 #              OpenAS2 Server
-#              Version 3.1.0
+#              Version 3.2.0
 #              RELEASE NOTES
 -----
-The OpenAS2 project is pleased to announce the release of OpenAS2 3.1.0
+The OpenAS2 project is pleased to announce the release of OpenAS2 3.2.0
 
-The release download file is: OpenAS2Server-3.1.0.zip
+The release download file is: OpenAS2Server-3.2.0.zip
 
 The zip file contains a PDF document (OpenAS2HowTo.pdf) providing information on installing and using the application.
 ## NOTE: Testing covers Java 8 to 17. The application should work for older versions down to Java 7 but they are not tested as part of the CI/CD pipeline.
 
-Version 3.1.0 - 2022-06-04
+Version 3.2.0 - 2022-06-21
 This is a minor enhancement and bugfix release:
        **IMPORTANT NOTE**: Please review upgrade notes below if you are upgrading
 
-  1. Support embedded $properties.xxx$ within property element values.
-  2. Support setting unix executables in ZIP file when packaging new release in Maven.
-  3. Fix asynchronous MDN sending and receiving.
+  1. Support "prevent_chunking" attribute on partnership to support older AS2 systems.
+  2. Fix copying the sent file to the sent folder when successfully sent.
 
 
 ##Upgrade Notes

Remote/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.1.0</version>
+        <version>3.2.0</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

Server/pom.xml

@@ -7,7 +7,7 @@
         <!-- DO NOT CHANGE THIS "groupId" WITHOUT CHANGING XMLSession.getManifestAttributes.MANIFEST_VENDOR_ID_ATTRIB -->
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.1.0</version>
+        <version>3.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

Server/src/main/java/CheckCertificate.java

@@ -226,7 +226,7 @@ private void checkUsingApacheHttp(String host, int port, String uri, String targ
             httpOptions.put(HTTPUtil.PARAM_HTTP_USER, auth_user);
             httpOptions.put(HTTPUtil.PARAM_HTTP_PWD, auth_pwd);
         }
-        ResponseWrapper resp = HTTPUtil.execRequest(HTTPUtil.Method.POST, "https://" + host + ":" + port + "/" + uri, null, null, new ByteArrayInputStream("Testing".getBytes()), httpOptions, 1000000000);
+        ResponseWrapper resp = HTTPUtil.execRequest(HTTPUtil.Method.POST, "https://" + host + ":" + port + "/" + uri, null, null, new ByteArrayInputStream("Testing".getBytes()), httpOptions, 1000000000, false);
         System.out.println("Got a response using Apache HTTP Client: " + resp.getStatusCode());
         System.out.println("\t\tHEADERS: " + resp.getHeaders());
         System.out.println("\t\tBODY: " + resp.getBody());

Server/src/main/java/org/openas2/partner/Partnership.java

@@ -55,7 +55,8 @@ public class Partnership implements Serializable {
     public static final String PA_CUSTOM_MIME_HEADER_NAMES_REGEX_ON_FILENAME = "custom_mime_header_names_regex_on_filename"; // Regex to split filename into values
     public static final String PAIB_NAMES_FROM_FILENAME = "attribute_names_from_filename"; // List of attribute names to be set from parsed filename
     public static final String PAIB_VALUES_REGEX_ON_FILENAME = "attribute_values_regex_on_filename"; // Regex to split filename into values
-    public static final String PA_HTTP_NO_CHUNKED_MAX_SIZE = "no_chunked_max_size"; // Disables chunked HTTP transfer when file size is set larger as 0
+    public static final String PA_HTTP_NO_CHUNKED_MAX_SIZE = "no_chunked_max_size"; // Disables chunked HTTP transfer when file size is set larger than the value in this param
+    public static final String PA_HTTP_PREVENT_CHUNKING = "prevent_chunking"; // Will try to force the send without using chunked HTTP transfer
     public static final String PA_STORE_RECEIVED_FILE_TO = "store_received_file_to"; // Allows overriding the MessageFileModule "filename" parameter per partnership
     // A hopefully temporary key to maintain backwards compatibility
     public static final String USE_NEW_CERTIFICATE_LOOKUP_MODE = "use_new_certificate_lookup_mode";
@@ -86,6 +87,10 @@ public void setAttribute(String id, String value) {
         getAttributes().put(id, value);
     }
 
+    /** Gets the value of the attribute for the provided key
+     * @param id
+     * @return Returns the value to which the specified key is mapped, or null if this map contains no mapping for the key.
+     */
     public String getAttribute(String id) {
         return getAttributes().get(id);
     }
@@ -254,10 +259,6 @@ public boolean isRemoveCmsAlgorithmProtectionAttr() {
         return "true".equalsIgnoreCase(getAttribute(Partnership.PA_REMOVE_PROTECTION_ATTRIB));
     }
 
-    public boolean isNoChunkedTransfer() {
-        return (getNoChunkedMaxSize() > 0L);
-    }
-
     public long getNoChunkedMaxSize() {
         long max = 0L;
         try {
@@ -267,4 +268,8 @@ public long getNoChunkedMaxSize() {
         return max;
     }
 
+    public boolean isPreventChunking(boolean defaultPreference) {
+        String preventChunking = getAttribute(Partnership.PA_HTTP_PREVENT_CHUNKING);
+        return preventChunking == null?defaultPreference:"true".equalsIgnoreCase(preventChunking);
+    }
 }

Server/src/main/java/org/openas2/processor/receiver/DirectoryPollingModule.java

@@ -9,7 +9,7 @@
 import org.openas2.util.IOUtil;
 
 import java.io.File;
-import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
@@ -238,20 +238,12 @@ protected void processFile(File file) throws OpenAS2Exception {
             logger.info("processing " + file.getAbsolutePath());
         }
 
-        try (FileInputStream in = new FileInputStream(file)) {
-            processDocument(in, file.getName());
-            try {
-                if(sentDir != null) {
-                    // Archive Sent file on disk
-                    IOUtil.handleArchive(file, sentDir, false);
-                }else{
-                    IOUtil.deleteFile(file);
-                }
-            } catch (IOException e) {
-                throw new OpenAS2Exception("Failed to archive/remove file handed off for processing:" + file.getAbsolutePath(), e);
-            }
-        } catch (IOException e) {
-            throw new OpenAS2Exception("Failed to process file:" + file.getAbsolutePath(), e);
+        try {
+            processDocument(file, file.getName());
+        } catch (FileNotFoundException e) {
+            // Try to move original file to error dir in case error handling has not done it  for us.
+            IOUtil.handleArchive(file, errorDir, false);
+            throw new OpenAS2Exception("Failed to initiate processing for file:" + file.getAbsolutePath(), e);
         }
     }
 

Server/src/main/java/org/openas2/processor/receiver/MessageBuilderModule.java

@@ -20,6 +20,7 @@
 import org.openas2.processor.resender.ResenderModule;
 import org.openas2.processor.sender.SenderModule;
 import org.openas2.util.AS2Util;
+import org.openas2.util.IOUtil;
 
 import javax.activation.DataHandler;
 import javax.activation.DataSource;
@@ -62,23 +63,49 @@ protected CompositeParameters createParser(Message msg) {
     }
 
 
-    protected Message processDocument(InputStream ip, String filename) throws OpenAS2Exception, FileNotFoundException {
+    /**
+     * Move the file into the processing folder then invoke the sending process.
+     * @param fileToSend
+     * @param filename
+     * @return
+     * @throws OpenAS2Exception
+     * @throws FileNotFoundException
+     */
+    protected Message processDocument(File fileToSend, String filename) throws OpenAS2Exception, FileNotFoundException {
         Message msg = buildMessageMetadata(filename);
+        File pendingFile = new File(msg.getAttribute(FileAttribute.MA_PENDINGFILE));
+        try {
+            IOUtil.moveFile(fileToSend, pendingFile, false);
+        } catch (IOException e) {
+            logger.error(": " + e.getMessage(), e);
+            throw new OpenAS2Exception("Failed to move the inbound file " + fileToSend.getPath() + " to the processing location " + pendingFile.getName());
+        }
+        return processDocument(pendingFile, msg); 
+    }
 
-        String pendingFile = msg.getAttribute(FileAttribute.MA_PENDINGFILE);
-        // Persist the file that has been passed in
-        File doc = new File(pendingFile);
+    /**
+     * Take the file input stream and write it to a file system file in the processing folder. 
+     * Use this method if the file is produced in real time through a stream.
+     * @param ip
+     * @param filename
+     * @return
+     * @throws OpenAS2Exception
+     * @throws FileNotFoundException
+     */
+    protected Message processDocument(InputStream ip, String filename) throws OpenAS2Exception, FileNotFoundException {
+        Message msg = buildMessageMetadata(filename);
+        File pendingFile = new File(msg.getAttribute(FileAttribute.MA_PENDINGFILE));
         FileOutputStream fo = null;
         try {
-            fo = new FileOutputStream(doc);
+            fo = new FileOutputStream(pendingFile);
         } catch (FileNotFoundException e1) {
-            throw new OpenAS2Exception("Could not create file in pending folder: " + pendingFile, e1);
+            throw new OpenAS2Exception("Could not create file in pending folder: " + pendingFile.getName(), e1);
         }
         try {
             IOUtils.copy(ip, fo);
         } catch (IOException e1) {
             fo = null;
-            throw new OpenAS2Exception("Could not write file to pending folder: " + pendingFile, e1);
+            throw new OpenAS2Exception("Could not write file to pending folder: " + pendingFile.getName(), e1);
         }
         try {
             ip.close();
@@ -94,19 +121,19 @@ protected Message processDocument(InputStream ip, String filename) throws OpenAS
             e1.printStackTrace();
         }
         fo = null;
+        return processDocument(pendingFile, msg);
+    }
 
-        try {
-            buildMessageData(msg, doc, null);
-        } finally {
-            doc = null;
-        }
+    protected Message processDocument(File pendingFile, Message msg) throws OpenAS2Exception, FileNotFoundException {
+        buildMessageData(msg, pendingFile, null);
         String customHeaderList = msg.getPartnership().getAttribute(Partnership.PA_CUSTOM_MIME_HEADER_NAMES_FROM_FILENAME);
         if (customHeaderList != null && customHeaderList.length() > 0) {
             String[] headerNames = customHeaderList.split("\\s*,\\s*");
             String delimiters = msg.getPartnership().getAttribute(Partnership.PA_CUSTOM_MIME_HEADER_NAME_DELIMITERS_IN_FILENAME);
             if (logger.isTraceEnabled()) {
                 logger.trace("Adding custom headers based on message file name to custom headers map. Delimeters: " + delimiters + msg.getLogMsgID());
             }
+            String filename = msg.getAttribute(FileAttribute.MA_FILENAME);
             if (delimiters != null) {
                 // Extract the values based on delimiters which means the mime header names are
                 // prefixed with a target
@@ -146,11 +173,11 @@ protected Message processDocument(InputStream ip, String filename) throws OpenAS
             }
         }
         if (logger.isInfoEnabled()) {
-            logger.info("File assigned to message: " + filename + msg.getLogMsgID());
+            logger.info("File assigned to message: " + pendingFile.getName() + msg.getLogMsgID());
         }
 
         if (msg.getData() == null) {
-            throw new InvalidMessageException("Failed to retrieve data for outbound AS2 message for file: " + filename);
+            throw new InvalidMessageException("Failed to retrieve data for outbound AS2 message for file: " + pendingFile.getName());
         }
         if (logger.isTraceEnabled()) {
             logger.trace("PARTNERSHIP parms: " + msg.getPartnership().getAttributes() + msg.getLogMsgID());
@@ -171,6 +198,9 @@ protected Message processDocument(InputStream ip, String filename) throws OpenAS
             msg.setStatus(Message.MSG_STATUS_MSG_SEND);
             // Transmit the message
             getSession().getProcessor().handle(SenderModule.DO_SEND, msg, options);
+            if (!msg.isConfiguredForAsynchMDN()) {
+                AS2Util.cleanupFiles(msg, false);
+            }
         } catch (Exception e) {
             msg.setLogMsg("Fatal error sending message: " + org.openas2.logging.Log.getExceptionMsg(e));
             logger.error(msg, e);

Server/src/main/java/org/openas2/processor/receiver/NetModule.java

@@ -104,7 +104,7 @@ public boolean healthcheck(List<String> failures) {
             }
             Map<String, String> options = new HashMap<String, String>();
             options.put(HTTPUtil.HTTP_PROP_OVERRIDE_SSL_CHECKS, "true");
-            ResponseWrapper rw = HTTPUtil.execRequest(HTTPUtil.Method.GET, urlString, null, null, null, options, 0L);
+            ResponseWrapper rw = HTTPUtil.execRequest(HTTPUtil.Method.GET, urlString, null, null, null, options, 0L, false);
             if (200 != rw.getStatusCode()) {
                 failures.add(this.getClass().getSimpleName() + " - Error making HTTP connection. Response code: " + rw.getStatusCode() + " " + rw.getStatusPhrase());
                 return false;

Server/src/main/java/org/openas2/processor/sender/AS2SenderModule.java

@@ -39,6 +39,7 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.ObjectOutputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -58,13 +59,13 @@ public class AS2SenderModule extends HttpSenderModule implements HasSchedule {
     /** TODO: Remove this when module config enforces setting the action so that the super method does all the work
     *
     */
-   public String getModuleAction() {
-       String action = super.getModuleAction();
-       if (action == null) {
-           return SenderModule.DO_SEND;
-       }
-       return action;
-   }
+    public String getModuleAction() {
+        String action = super.getModuleAction();
+        if (action == null) {
+            return SenderModule.DO_SEND;
+        }
+        return action;
+    }
 
     public boolean canHandle(String action, Message msg, Map<String, Object> options) {
         if (!super.canHandle(action, msg, options)) {
@@ -186,7 +187,8 @@ private void sendMessage(String url, Message msg, MimeBodyPart securedData) thro
         httpOptions.put(HTTPUtil.PARAM_HTTP_USER, msg.getPartnership().getAttribute(HTTPUtil.PARAM_HTTP_USER));
         httpOptions.put(HTTPUtil.PARAM_HTTP_PWD, msg.getPartnership().getAttribute(HTTPUtil.PARAM_HTTP_PWD));
         long maxSize = msg.getPartnership().getNoChunkedMaxSize();
-        ResponseWrapper resp = HTTPUtil.execRequest(HTTPUtil.Method.POST, url, ih.getAllHeaders(), null, securedData.getInputStream(), httpOptions, maxSize);
+        boolean preventChunking = msg.getPartnership().isPreventChunking(false);
+        ResponseWrapper resp = HTTPUtil.execRequest(HTTPUtil.Method.POST, url, ih, null, securedData.getInputStream(), httpOptions, maxSize, preventChunking);
         if (logger.isInfoEnabled()) {
             logger.info("Message sent and response received in " + resp.getTransferTimeMs() + "ms" + msg.getLogMsgID());
         }

Server/src/main/java/org/openas2/processor/sender/MDNSenderModule.java

@@ -2,6 +2,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.protocol.HTTP;
 import org.openas2.OpenAS2Exception;
 import org.openas2.WrappedException;
 import org.openas2.message.AS2Message;
@@ -156,9 +157,10 @@ private boolean sendAsyncMDN(MessageMDN mdn, String url, DispositionType disposi
                 throw new WrappedException(we);
             }
             byte[] data = dataOutputStream.toByteArray();
-            // make sure to set the content-length header
-            //mdn.setHeader("Content-Length", Integer.toString(data.length));
-            ResponseWrapper resp = HTTPUtil.execRequest(HTTPUtil.Method.POST, url, mdn.getHeaders().getAllHeaders(), null, new ByteArrayInputStream(data), httpOptions, maxSize);
+            // make sure to set the content-length header to avoid transferring as chunked which some AS2 software implementations do not support
+            mdn.setHeader(HTTP.CONTENT_LEN, Integer.toString(data.length));
+            boolean preventChunking = msg.getPartnership().isPreventChunking(false);
+            ResponseWrapper resp = HTTPUtil.execRequest(HTTPUtil.Method.POST, url, mdn.getHeaders(), null, new ByteArrayInputStream(data), httpOptions, maxSize, preventChunking);
 
             int respCode = resp.getStatusCode();
             // Check the HTTP Response code