Support file splitting and other DB's on WebGUI (#301)

* Fix exit code when failures occur

* Remove the zip before creating to avoid retaining old versions.

* Supported automated execution mode for chaining calls.

* Use generic delete task to pre-deete the zip package

* Make it possible to run the script without user input

* Grant permission to local network user for service control

* Fix issue with system properties containing $ sign:
#298

* Fix style errors

* Refactor to support splitting files

* Enhancements to support other database types for the UI queries.

* Remove unused import

* Support for splitting files for sending.

* Support for splitting files to be sent into smaller chunks.

* Change the name of the split files attribute to be more indicative of
function

* Update documentation for splitting files

* Release notes and version numbers

* Hide the file split attributes

* Add relase note

* Fix style errors

RELEASE-NOTES.md

@@ -1,25 +1,22 @@
 #              OpenAS2 Server
-#              Version 3.3.0
+#              Version 3.4.0
 #              RELEASE NOTES
 -----
-The OpenAS2 project is pleased to announce the release of OpenAS2 3.3.0
+The OpenAS2 project is pleased to announce the release of OpenAS2 3.4.0
 
-The release download file is: OpenAS2Server-3.3.0.zip
+The release download file is: OpenAS2Server-3.4.0.zip
 
 The zip file contains a PDF document (OpenAS2HowTo.pdf) providing information on installing and using the application.
 ## NOTE: Testing covers Java 8 to 17. The application should work for older versions down to Java 7 but they are not tested as part of the CI/CD pipeline.
 
-Version 3.3.0 - 2022-08-13
-This is a significant enhancement and minor bugfix release:
+Version 3.4.0 - 2022-10-04
+This is an enhancement and minor bugfix release:
        **IMPORTANT NOTE**: Please review upgrade notes below if you are upgrading
 
-  1. Support rejecting messages being sent that are unsigned. See section "Reject Unsigned Messages" in the documentation.
-  2. Support having 2 certificates for your partner definition. This allows switch over of your own certificate to be zero down time. See section "Overlapping Old And New Certificates When Changing" in the documentation.
-  3. Support overriding most of the config.xml attribute values using the external property file. This allows custom configuration to be restricted to a single properties file and makes version upgrades much simpler.
-  4. Enhance shell and bat scripts to prepare for install and upgrade scripts to simplify managing the install and upgrade of OpenAS2 in the future.
-  5. Enhance the properties parser to support periods in the property name.
-  6. Fix the erroneous error when cleaning up files due to a duplicate call to the file cleanup function.
-
+  1. Support for splitting line based files into multiple file. This is useful for very large files where encryption consumes too much memory.
+  2. Support other databases than H2 for the WebUI commands.
+  3. Catch exceptions in the strm command processor to avoid crashing the command processor.
+  4. Pre-enhance AS2 properties before adding system properties to cater for $ in system properties
 
 
 ##Upgrade Notes

Remote/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.3.0</version>
+        <version>3.4.0</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

Server/pom.xml

@@ -7,7 +7,7 @@
         <!-- DO NOT CHANGE THIS "groupId" WITHOUT CHANGING XMLSession.getManifestAttributes.MANIFEST_VENDOR_ID_ATTRIB -->
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>3.3.0</version>
+        <version>3.4.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -66,7 +66,7 @@
                         <phase>package</phase>
                         <configuration>
                             <target>
-                                <copy todir="${package.assembly.dir}/bin"
+                               <copy todir="${package.assembly.dir}/bin"
                                       verbose="true">
                                     <fileset dir="${project.basedir}/src/bin"/>
                                 </copy>
@@ -102,7 +102,11 @@
                                       todir="${package.assembly.dir}/bin/remote" verbose="true"/>
                                 <!-- create distribution package
                                     basedir="${package.assembly.dir}"
- -->
+                                -->
+                                <!--
+                                    Pre-delete the package otherwise it keeps old files in it
+                                -->
+ 							    <delete file="${project.basedir}/dist/${project.dist.package.name}"/>
                                 <zip
                                     destfile="${project.basedir}/dist/${project.dist.package.name}"
                                     update="true">

Server/src/bin/gen_p12_key_par.bat

@@ -9,30 +9,43 @@ set dName=%4%
 
 set CertValidDays=2900
 
+SET tmppath=%~dp0
+pushd %tmppath%
+cd ..
+set OPENAS2_BASE_DIR=%CD%
+popd
+
 rem Setup the Java Virtual Machine
 call "%OPENAS2_BASE_DIR%\bin\find_java.bat"
-if %ERRORLEVEL% NEQ 0 exit /B 1
+if %ERRORLEVEL% NEQ 0 EXIT /B 1
 
 echo        Generate a certificate to a PKCS12 key store.
 echo        Generating certificate:  using alias %certAlias% to %tgtStore%.p12"
 
-set /p ksPwd=Enter password for keystore:%=%
+setLocal EnableDelayedExpansion
+if /I "!IS_AUTOMATED_EXEC!" == "1" (
+  set ksPwd=$KEYSTORE_PASSWORD
+)
+else (
+  set /p ksPwd=Enter password for keystore:%=%
+)
+
 
 "%JAVA_HOME%\bin\keytool" -genkeypair -alias %certAlias% -validity %CertValidDays%  -keyalg RSA -sigalg %sigAlg% -keystore %tgtStore%.p12 -storepass %ksPwd% -storetype pkcs12 -dname %dName%
 if errorlevel 1 (
     echo Failed to generate keystore
-    goto END
+    EXIT /B 1
 )
 "%JAVA_HOME%\bin\keytool" -selfcert -alias %certAlias% -validity %CertValidDays%  -sigalg %sigAlg% -keystore %tgtStore%.p12 -storepass %ksPwd% -storetype pkcs12
 if errorlevel 1 (
     echo Failed to self certify certificate
-    goto END
+    EXIT /B 1
 )
 
 "%JAVA_HOME%\bin\keytool" -export -rfc -file %certAlias%.cer -alias %certAlias%  -keystore %tgtStore%.p12 -storepass %ksPwd% -storetype pkcs12
 if errorlevel 1 (
     echo Failed to export public key from keystore
-    goto END
+    EXIT /B 1
 )
 
 echo.
@@ -58,7 +71,7 @@ goto :END
   echo      Expected OUTPUT: as2_certs.p12 -  keystore containing both public and private key
   echo                      partnera.cer - public key certificate file .
 
-:warn
+  EXIT /B 1
 
 :END
 

Server/src/bin/gen_p12_key_par.sh

@@ -44,8 +44,7 @@ if [ -n "$CERT_START_DATE" ]; then
 fi
 
 if [ -z $JAVA_HOME ]; then
-  OS=$(uname -s)
-
+  OS=$(uname -s)  echo "Looking for JAVA_HOME on OS: ${OS}..."
   if [[ "${OS}" == *Darwin* ]]; then
     # Mac OS X platform
     JAVA_HOME=$(/usr/libexec/java_home)
@@ -57,12 +56,12 @@ if [ -z $JAVA_HOME ]; then
     echo "Windows not supported by this script"
   fi
 fi
-
 if [ -z $JAVA_HOME ]; then
   echo "ERROR: Cannot find JAVA_HOME"
-  exit
+  exit 1
 fi
 
+echo "Using JAVA_HOME: ${JAVA_HOME}"
 if [ "1" != "$IS_AUTOMATED_EXEC" ]; then
   echo "Generate a certificate to a PKCS12 key store."
   echo "Generating certificate:  using alias $certAlias to ${tgtStore}.p12 $PRE_GEN_MSG_ADDITIONAL"

Server/src/bin/import_alias_from_keystore.sh

@@ -45,17 +45,22 @@ fi
 
 if [ -z $JAVA_HOME ]; then
   echo "ERROR: Cannot find JAVA_HOME"
-  exit
+  exit 1
 fi
 
 echo "Executing action \"${action}\" on certificate from key \"${srcKeystore}\" using alias \"${tgtAlias}\" to: ${tgtKeystore}"
-read -p "Do you wish to execute this request? [Y/N]" Response
-if [  $Response != "Y" -a $Response != "y"  ] ; then
-  exit 1
+if [ "1" != "$IS_AUTOMATED_EXEC" ]; then
+  read -p "Do you wish to execute this request? [Y/N]" Response
+  if [  $Response != "Y" -a $Response != "y"  ] ; then
+    exit 1
+  fi
+  read -p "Enter password for source keystore:" srcksPwd
+  read -p "Enter password for destination keystore:" destksPwd
+else
+  srcksPwd=${KEYSTORE_PASSWORD}
+  destksPwd=${KEYSTORE_PASSWORD}
 fi
 
-read -p "Enter password for source keystore:" srcksPwd
-read -p "Enter password for destination keystore:" destksPwd
 
 if [ "${action}" = "replace" ]; then
     $JAVA_HOME/bin/keytool -delete -alias ${tgtAlias} -keystore ${tgtKeystore} -storepass $destksPwd -storetype pkcs12

Server/src/bin/import_public_cert.bat

@@ -10,7 +10,6 @@ set action=%4%
 rem Setup the Java Virtual Machine
 if not "%JAVA%" == "" goto :Check_JAVA_END
     if not "%JAVA_HOME%" == "" goto :TryJDKEnd
-        call :warn JAVA_HOME not set; results may vary
 :TryWOWJRE
     FOR /F "usebackq tokens=3*" %%A IN (`REG QUERY "HKLM\Software\WOW6432NODE\JavaSoft\Java Runtime Environment" /s /v CurrentVersion ^| find "CurrentVersion"`) DO (
        set JAVA_VERSION=%%A
@@ -46,12 +45,13 @@ if not "%JAVA%" == "" goto :Check_JAVA_END
        set JAVA_HOME=%%A %%B
     )
     if not exist "%JAVA_HOME%" (
-       call :warn Unable to retrieve JAVA_HOME from Registry
+       echo Unable to retrieve JAVA_HOME from Registry
+       EXIT /B 1
     )
 :TryJDKEnd
     if not exist "%JAVA_HOME%" (
-        call :warn JAVA_HOME is not valid: "%JAVA_HOME%"
-        goto END
+        echo JAVA_HOME is not valid: "%JAVA_HOME%"
+        EXIT /B 1
     )
     set JAVA=%JAVA_HOME%\bin\java
 :Check_JAVA_END
@@ -65,7 +65,7 @@ if "%action%" == "replace" (
     if errorlevel 1 (
     	echo 
         echo Failed to delete the certificate in the keystore for alias "%certAlias%". See errors above to correct the problem.
-        goto END
+        EXIT /B 1
     )
 )
 
@@ -74,7 +74,7 @@ if errorlevel 1 (
 	echo. 
     echo ***** Failed to import the certificate to the keystore. See errors above to correct the problem.
     echo       If the error shows the certifcate already eists then add the "replace" option to the command line.
-    goto END
+    EXIT /B 1
 )
 
 echo. 
@@ -100,6 +100,5 @@ goto END
   echo        eg. %~nx0 partnera.cer as2_certs.p12 partnera
   echo                 OR
   echo        eg. %~nx0 partnera.cer as2_certs.p12 partnera replace
-
-:warn
+  EXIT /B 1
 :END

Server/src/bin/import_public_cert.sh

@@ -43,16 +43,19 @@ fi
 
 if [ -z $JAVA_HOME ]; then
   echo "ERROR: Cannot find JAVA_HOME"
-  exit
-fi
-
-echo "Executing action \"${action}\" on certificate from file \"${srcFile}\" using alias \"${certAlias}\" to: ${tgtStore}"
-read -p "Do you wish to execute this request? [Y/N]" Response
-if [  $Response != "Y" -a $Response != "y"  ] ; then
   exit 1
 fi
 
-read -p "Enter password for keystore:" ksPwd
+if [ "1" != "$IS_AUTOMATED_EXEC" ]; then
+  echo "Executing action \"${action}\" on certificate from file \"${srcFile}\" using alias \"${certAlias}\" to: ${tgtStore}"
+  read -p "Do you wish to execute this request? [Y/N]" Response
+  if [  $Response != "Y" -a $Response != "y"  ] ; then
+    exit 1
+  fi
+  read -p "Enter password for keystore:" ksPwd
+else
+  ksPwd=$KEYSTORE_PASSWORD
+fi
 
 if [ "${action}" = "replace" ]; then
     $JAVA_HOME/bin/keytool -delete -alias ${certAlias} -keystore ${tgtStore} -storepass $ksPwd -storetype pkcs12

Server/src/bin/install_winsvc.bat

@@ -44,7 +44,9 @@ set PR_LOGLEVEL=Error
 REM Path to java installation
 REM If the auto mode does not work then you can explicitly set the path to the Java install DLL 
 set PR_JVM=auto
-if /I "!CUSTOM_JAVA_HOME!" == "" goto SkipCustomJava
+if /I "%CUSTOM_JAVA_HOME%" == "" goto SkipCustomJava
+rem remove any enclosing quotes
+set CUSTOM_JAVA_HOME=%CUSTOM_JAVA_HOME:"=%
 set PR_JVM=%CUSTOM_JAVA_HOME%\bin\server\jvm.dll
 :SkipCustomJava
 
@@ -71,13 +73,13 @@ set PR_STOPPARAMS=stop
 
 REM  Add the below line into the install command if using a specific JVM
 REM  --JavaHome="%JAVA_HOME%" ^
-if /I "!CUSTOM_JAVA_HOME!" == "" goto SkipCustomJavaHome
+if /I "!CUSTOM_JAVA_HOME!" == "" goto NoCustomJVM
 rem Add the property arg to JVM options
 echo Setting custom properties file for service startup: !OPENAS2_PROPERTIES_FILE!
 set CUSTOM_SERVICE_PARAMS=%CUSTOM_SERVICE_PARAMS% ++JavaHome="%CUSTOM_JAVA_HOME%"
-:SkipCustomJavaHome
-REM Make the folder accessible to the "Local Service" user running the servioce
-icacls "%OPENAS2_BASE_DIR%" /grant *S-1-5-19:(OI)(CI)(M)
+:NoCustomJVM
+REM Make the folder for service creation and control accessible to the "Local Service" user running the servioce
+icacls "%APACHE_COMMONS_DAEMON%" /grant *S-1-5-19:(OI)(CI)(M)
 
 REM Install service
 "%PR_INSTALL%" //IS/%SERVICE_NAME% ^

Server/src/config/config.xml

@@ -3,7 +3,7 @@
    <properties storageBaseDir="%home%/../data"
                log_date_format="yyyy-MM-dd HH:mm:ss.SSS"
                sql_timestamp_format="yyyy-MM-dd HH:mm:ss.SSS"
-               as2_message_id_format="OPENAS2-$date.ddMMyyyyHHmmssZ$-$rand.UUID$@$msg.sender.as2_id$_$msg.receiver.as2_id$"
+               as2_message_id_format="$date.yyyyMMddHHmmssZ$-$rand.123$-$msg.sender.as2_id$_$msg.receiver.as2_id$_$msg.attributes.filename$"
                as2_receive_message_filename_fallback="$rand.shortUUID$"
                console.logger.enabled="true"
                file.logger.enabled="true"
@@ -162,13 +162,13 @@
               tcp_server_password="$properties.msg_tracking.tcp_server_password$"/>
       <module enabled="$properties.module.MDNFileModule.enabled$"
               classname="org.openas2.processor.storage.MDNFileModule"
-              filename="$properties.storageBaseDir$/$mdn.msg.sender.as2_id$-$mdn.msg.receiver.as2_id$/mdn/$date.yyyy-MM-dd$/$mdn.msg.content-disposition.filename$-$mdn.msg.headers.message-id$"
+              filename="$properties.storageBaseDir$/$mdn.msg.sender.as2_id$-$mdn.msg.receiver.as2_id$/mdn/$date.yyyy-MM-dd$/$mdn.msg.headers.message-id$"
               protocol="as2"
               tempdir="$properties.storageBaseDir$/temp"/>
       <module enabled="$properties.module.MessageFileModule.enabled$"
               classname="org.openas2.processor.storage.MessageFileModule"
-              filename="$properties.storageBaseDir$/$msg.sender.as2_id$-$msg.receiver.as2_id$/inbox/$msg.content-disposition.filename$-$msg.headers.message-id$"
-              header="$properties.storageBaseDir$/$msg.sender.as2_id$-$msg.receiver.as2_id$/msgheaders/$date.yyyy-MM-dd$/$msg.content-disposition.filename$-$msg.headers.message-id$"
+              filename="$properties.storageBaseDir$/$msg.sender.as2_id$-$msg.receiver.as2_id$/inbox/$msg.headers.message-id$"
+              header="$properties.storageBaseDir$/$msg.sender.as2_id$-$msg.receiver.as2_id$/msgheaders/$date.yyyy-MM-dd$/$msg.headers.message-id$"
               protocol="as2"
               tempdir="$properties.storageBaseDir$/temp"/>
       <module enabled="$properties.module.AS2ReceiverModule.http.enabled$"

Server/src/config/partnerships.xml

@@ -60,6 +60,12 @@
         <attribute name="attribute_names_from_filename" value="P-DynamicParm1,P-DynamicParm2"/>
         <attribute name="attribute_values_regex_on_filename" value="([A-Za-z]*)([^.]*).edi"/>
 		-->
+		<!--
+		Example attributes to support splitting files into smaller chunks
+        <attribute name="split_file_threshold_size_in_bytes" value="1073741824"/>
+        <attribute name="split_file_contains_header_row" value="true"/>
+        <attribute name="split_file_name_prefix" value="SF"/>
+		-->
     </partnership>
     <partnership name="PartnerA-to-MyCompany">
         <sender name="PartnerA"/>

Server/src/main/java/org/openas2/XMLSession.java

@@ -146,9 +146,6 @@ private void loadProperties(Node propNode) throws InvalidParameterException {
         LOGGER.info("Loading properties...");
 
         Map<String, String> properties = XMLUtil.mapAttributes(propNode, false);
-        @SuppressWarnings({ "unchecked", "rawtypes" })
-        Map<String, String> sysProps = (Map)System.getProperties();
-        properties.putAll(sysProps);
         // Make key things accessible via static object for things that do not have
         // accesss to session object
         properties.put(Properties.APP_TITLE_PROP, getAppTitle());
@@ -202,6 +199,17 @@ private void loadProperties(Node propNode) throws InvalidParameterException {
                 Properties.setProperty(key, parsedVal);
             }
         }
+        /* Put system properties in afterwards to avoid parsing embedded properties that may have 
+           a $ sign in the value but only if the key does not exist.
+        */
+        @SuppressWarnings({ "unchecked", "rawtypes" })
+        Map<String, String> sysProps = (Map)System.getProperties();
+        for (Map.Entry<String, String> entry : sysProps.entrySet()) {
+            String key = entry.getKey();
+            if (Properties.getProperty(key, null) == null) {
+                Properties.setProperty(key, entry.getValue());
+            }
+        }
     }
 
     private void loadCertificates(Node rootNode) throws OpenAS2Exception {

Server/src/main/java/org/openas2/app/message/GetDataForCharts.java

@@ -6,10 +6,11 @@
 
 import org.openas2.message.MessageFactory;
 import org.openas2.processor.msgtracking.DbTrackingModule;
-
+import org.openas2.processor.msgtracking.TrackingModule;
+import org.openas2.processor.ProcessorModule;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Map;
+import java.util.List;
 
 /**
  * list messages entries
@@ -39,14 +40,14 @@ public CommandResult execute(MessageFactory messageFactory, Object[] params) thr
                 String value = object.toString().split("=")[1];
                 map.put(name, value);
             }
-            DbTrackingModule db = new DbTrackingModule();
-            HashMap<String, String> options = new HashMap<String, String>();
-
-            options.put("jdbc_connect_string", "jdbc:h2:tcp://localhost:9092/openas2");
-            options.put("db_user", "sa");
-            options.put("db_pwd", "OpenAS2");
+            List<ProcessorModule> mpl = getSession().getProcessor().getModulesSupportingAction(TrackingModule.DO_TRACK_MSG);
+            if (mpl == null || mpl.isEmpty()) {
+                CommandResult cmdRes = new CommandResult(CommandResult.TYPE_ERROR);
+                cmdRes.getResults().add("No DB tracking module available.");
+            }
+            // Assume we only load one DB tracking module - not sure it makes sense if more than 1 was loaded
+            DbTrackingModule db = (DbTrackingModule) mpl.get(0);
 
-            db.init(getSession(), options);
             ArrayList<HashMap<String, String>> data = db.getDataCharts(map);
 
             CommandResult cmdRes = new CommandResult(CommandResult.TYPE_OK);