Bugfix db tracking plus errordir/sentdir enhancements (#157)

* Fix the field lookup in update mode to use lowercase always

* Update sample config.xml to show new attrobutes and extended options for
errordir and sentdir

* Simple bash script to connect to H2 DB

* Support controlling name of file stored in "sentdir" and "errordir"

* Document stored_sentdir_filename and sotred_errordir_filename as well as
provide more examples for setting errordir and sentdir using extended
dynamic variables

* Update release notes

* Updated version number

* Make config file settable using environment variable

Bundle/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>2.9.1</version>
+        <version>2.9.2</version>
     </parent>
 
     <artifactId>openas2-osgi</artifactId>

RELEASE-NOTES.md

@@ -1,25 +1,22 @@
 #              OpenAS2 Server
-#              Version 2.9.1
+#              Version 2.9.2
 #              RELEASE NOTES
 -----
-The OpenAS2 project is pleased to announce the release of OpenAS2 2.9.1
+The OpenAS2 project is pleased to announce the release of OpenAS2 2.9.2
 
-The release download file is: OpenAS2Server-2.9.1.zip
+The release download file is: OpenAS2Server-2.9.2.zip
 
 The zip file contains a PDF document (OpenAS2HowTo.pdf) providing information on installing and using the application.
 
-Version 2.9.1 - 2019-08-03
+Version 2.9.2 - 2019-08-16
 This is a an enhancement and bugfix release:
        **IMPORTANT NOTE**: Please review upgrade notes if you are upgrading
 
-  1. Support additional filters on logging to support reduced email logger errors flagging interface errors due to exernal access attemopts to AS2 server..
-  2. Fixed error where DB tracking module was not logging fields to database.
-  3. Documented additional logging filters.
-  4. Support for dynamic variables in sentDir and errorDir attributes in the config.
-  5. Enhanced the remote command processor to support text output format.
-  6. Updarted remote command processor documentation.
-  7. Added a script to run the remote command interface from the command line.
-  8. Included the remote socket command processor app jar into the release package to allow use without a separate effort to set up.
+  1. Fixed error where DB tracking module was not logging fields to database.
+  2. Added a script to connect to the H2 database from the command line.
+  3. Simplified the default config.xml for changing ports by using properties.
+  4. Added support for location of the config.xml using an environment variable in the start-openas2.sh file
+  5. Document stored_sent_filename and stored_error_filename as well as provide more examples for setting errordir and sentdir using extended dynamic variables
 
 ##Upgrade Notes
  See the openAS2HowTo appendix for the general process on upgrading OpenAS2.

Remote/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>net.sf.openas2</groupId>
         <artifactId>OpenAS2</artifactId>
-        <version>2.9.1</version>
+        <version>2.9.2</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>

Server/pom.xml

@@ -7,7 +7,7 @@
         <!-- DO NOT CHANGE THIS "groupId" WITHOUT CHANGING XMLSession.getManifestAttributes.MANIFEST_VENDOR_ID_ATTRIB -->
 		<groupId>net.sf.openas2</groupId>
 		<artifactId>OpenAS2</artifactId>
-		<version>2.9.1</version>
+		<version>2.9.2</version>
 	</parent>
 
 	<artifactId>openas2-server</artifactId>

Server/src/bin/db_connect.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+set -e
+# purpose: runs the remote OpenAS2 connect application     
+x=`basename $0`
+
+function usage() {
+           echo "Connect to a running instance of OpenAS2."
+           echo "usage: ${x} [-u user ID] <-P password> [-h host] [-p port]"
+           echo "            WHERE"
+           echo "               user ID = the user ID configured for the DB Defaults to 'sa'"
+           echo "               password = the password configured for the DB"
+           echo "                           Can be set as OPENAS2_DB_PWD environment variable"
+           echo "               host = hostname or IP address of OpenAS2 server. Defaults to \"localhost\" if not provided."
+           echo "               port = port that the OpenAS2 DB is running on. Defaults to 9092 if not provided."
+           echo ""
+           echo "       eg. $0 -u MyuserId -p MySecret"
+           echo "           $0 -u MyuserId -p MySecret -h as2.mydomain.com"
+           echo "           $0 -u MyuserId -p MySecret -h as2.mydomain.com -c SSL_DH_anon_WITH_RC4_128_MD5"
+           exit 1
+}
+
+if test $# -lt 1; then
+  usage
+fi
+
+OPENAS2_DB_UID=sa
+HOST_NAME=localhost
+HOST_PORT=9092
+
+while getopts "u:p:h:P:" opt; do
+  case ${opt} in
+    u ) OPENAS2_DB_UID=$OPTARG
+      ;;
+    P ) OPENAS2_DB_PWD=$OPTARG
+      ;;
+    h ) HOST_NAME=$OPTARG
+      ;;
+    p ) HOST_PORT=$OPTARG
+      ;;
+    \? )   usage
+      ;;
+  esac
+done
+
+binDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Backwards compatibility: use value from pid_file if pid_file has a value and openas2_pid has no value.
+#
+if [ -z $JAVA_HOME ]; then
+  OS=$(uname -s)
+
+  if [[ "${OS}" == *Darwin* ]]; then
+    # Mac OS X platform
+    JAVA_HOME=$(/usr/libexec/java_home)
+  elif [[ "${OS}" == *Linux* ]]; then
+    # Linux platform
+    JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
+  elif [[ "${OS}" == *MINGW* ]]; then
+    # Windows NT platform
+    echo "Windows not supported by this script"
+  fi
+fi
+
+if [ -z $JAVA_HOME ]; then
+  echo "ERROR: Cannot find JAVA_HOME"
+  exit 1
+fi
+
+CMD=`echo "${JAVA_HOME}/bin/java -cp .:${binDir}/../lib/h2* org.h2.tools.Shell -user ${OPENAS2_DB_UID} -password ${OPENAS2_DB_PWD} -url jdbc:h2:tcp://${HOST_NAME}:${HOST_PORT}/openas2"`
+echo
+echo Running ${CMD}
+echo
+  ${CMD}
+  RETVAL="$?"
+exit $RETVAL

Server/src/bin/start-openas2.sh

@@ -23,7 +23,10 @@ fi
 EXTRA_PARMS="-Xms32m -Xmx384m -Dorg.apache.commons.logging.Log=org.openas2.logging.Log"
 
 # Set the config file location
-EXTRA_PARMS="$EXTRA_PARMS -Dopenas2.config.file=${binDir}/../config/config.xml"
+if [ -z $OPENAS2_CONFIG_FILE ]; then
+  OPENAS2_CONFIG_FILE=${binDir}/../config/config.xml
+fi
+EXTRA_PARMS="$EXTRA_PARMS -Dopenas2.config.file=${OPENAS2_CONFIG_FILE}"
 
 # For versions of Java that prevent restricted HTTP headers (see documentation for discussion on this)
 #EXTRA_PARMS="$EXTRA_PARMS -Dsun.net.http.allowRestrictedHeaders=true"

Server/src/config/config.xml

@@ -68,7 +68,7 @@
 		     The name of the file sent to the partner will be "OrderID-745634.edi" -->
 		<module classname="org.openas2.processor.receiver.AS2DirectoryPollingModule"
 			outboxdir="$properties.storageBaseDir$/toAny"
-			errordir="$properties.storageBaseDir$/toAny/error"
+			errordir="$properties.storageBaseDir$/toAny/error/$mdn.msg.sender.as2_id$-$mdn.msg.receiver.as2_id"
 			interval="5"
 			delimiters="-"
 			mergeextratokens="true"
@@ -79,7 +79,10 @@
 		<!-- This directory polling module will is dedicated to sending to partner PartnerA_OID -->
 		<module classname="org.openas2.processor.receiver.AS2DirectoryPollingModule"
 			outboxdir="$properties.storageBaseDir$/toPartnerA/"
-			errordir="$properties.storageBaseDir$/toPartnerA/error"
+			errordir="$properties.storageBaseDir$/toPartnerA/error/$date.YYYY$/$date.MM$"
+			stored_error_filename="$msg.attributes.filename$-$date.YYYY$-$date.MM$-$msg.headers.message-id$"
+			sentdir="$properties.storageBaseDir$/$mdn.msg.sender.as2_id$-$mdn.msg.receiver.as2_id/sent/$date.YYYY$/$date.MM$"
+			stored_sent_filename="$msg.attributes.filename$-$msg.headers.message-id$"
 			interval="5"
 			defaults="sender.as2_id=MyCompany_OID, receiver.as2_id=PartnerA_OID"
 			sendfilename="true"

Server/src/main/java/org/openas2/message/FileAttribute.java

@@ -8,5 +8,7 @@ public interface FileAttribute {
 	public static final String MA_PENDINGFILE = "pendingfilename";
 	public static final String MA_STATUS = "status";
 	public static final String MA_ERROR_DIR = "errordir";
+	public static final String MA_ERROR_FILENAME = "errorfilename";
 	public static final String MA_SENT_DIR = "sentdir";
+	public static final String MA_SENT_FILENAME = "sentfilename";
 }

Server/src/main/java/org/openas2/processor/msgtracking/DbTrackingModule.java

@@ -139,8 +139,10 @@ else if (colName.equalsIgnoreCase(FIELDS.UPDATE_DT))
 					else appendFieldForInsert(colName, DateUtil.getSqlTimestamp(), fieldStmt, valuesStmt, meta.getColumnType(i + 1));
 				else if (isUpdate)
 				{
-					// Only write unchanged field values
-					String mapVal = map.get(colName.toUpperCase());
+					/* Only write unchanged field values.
+					 * Map is field names in LOWER case so convert in case DB server returns column names in uppercase
+					 */
+					String mapVal = map.get(colName.toLowerCase());
 					if (mapVal == null)
 					{
 						continue;

Server/src/main/java/org/openas2/processor/receiver/DirectoryPollingModule.java

@@ -38,11 +38,6 @@ public void init(Session session, Map<String, String> options) throws OpenAS2Exc
 	try {
 	    outboxDir = getParameter(PARAM_OUTBOX_DIRECTORY, true);
 	    IOUtil.getDirectoryFile(outboxDir);
-	    errorDir = getParameter(PARAM_ERROR_DIRECTORY, true);
-	    IOUtil.getDirectoryFile(errorDir);
-	    sentDir = getParameter(PARAM_SENT_DIRECTORY, false);
-	    if (sentDir != null)
-		IOUtil.getDirectoryFile(sentDir);
 	    String pendingInfoFolder = getSession().getProcessor().getParameters().get("pendingmdninfo");
 	    IOUtil.getDirectoryFile(pendingInfoFolder);
 	    String pendingFolder = getSession().getProcessor().getParameters().get("pendingmdn");

Server/src/main/java/org/openas2/processor/receiver/MessageBuilderModule.java

@@ -38,7 +38,9 @@
 public abstract class MessageBuilderModule extends BaseReceiverModule {
 
     public static final String PARAM_ERROR_DIRECTORY = "errordir";
+    public static final String PARAM_ERROR_FILENAME = "stored_error_filename";
     public static final String PARAM_SENT_DIRECTORY = "sentdir";
+    public static final String PARAM_SENT_FILENAME = "stored_sent_filename";
 
     public static final String PARAM_FORMAT = "format";
     public static final String PARAM_DELIMITERS = "delimiters";
@@ -234,11 +236,14 @@ public Message buildMessageMetadata(String filename) throws OpenAS2Exception {
 	// receiver ID
 	String pendingFile = AS2Util.buildPendingFileName(msg, getSession().getProcessor(), "pendingmdn");
 	msg.setAttribute(FileAttribute.MA_PENDINGFILE, pendingFile);
-	CompositeParameters parser = createParser(msg);
+	CompositeParameters parser = new CompositeParameters(false).add("date", new DateParameters())
+		.add("msg", new MessageParameters(msg)).add("rand", new RandomParameters());
 	msg.setAttribute(FileAttribute.MA_ERROR_DIR, ParameterParser.parse(getParameter(PARAM_ERROR_DIRECTORY, true), parser));
-	if (getParameter(PARAM_SENT_DIRECTORY, false) != null)
+	msg.setAttribute(FileAttribute.MA_ERROR_FILENAME, getParameter(PARAM_ERROR_FILENAME, false));
+	if (getParameter(PARAM_SENT_DIRECTORY, false) != null) {
 	    msg.setAttribute(FileAttribute.MA_SENT_DIR, ParameterParser.parse(getParameter(PARAM_SENT_DIRECTORY, false), parser));
-
+	    msg.setAttribute(FileAttribute.MA_SENT_FILENAME, getParameter(PARAM_SENT_FILENAME, false));
+	}
 	return msg;
 
     }

Server/src/main/java/org/openas2/util/AS2Util.java

@@ -651,12 +651,12 @@ public static void getMetaData(AS2Message msg, File inFile) throws OpenAS2Except
 	    if (logger.isTraceEnabled())
 		logger.trace("Data retrieved from Pending info file:" + "\n        Original MIC: "
 			+ msg.getCalculatedMIC() + "\n        Retry Count: " + retries
-			+ "\n        Original file name : " + msg.getPayloadFilename()
-			+ "\n        Sent file name : " + msg.getAttribute(FileAttribute.MA_FILENAME)
-			+ "\n        Pending message file : " + msg.getAttribute(FileAttribute.MA_PENDINGFILE)
-			+ "\n        Error directory: " + msg.getAttribute(FileAttribute.MA_ERROR_DIR)
-			+ "\n        Sent directory: " + msg.getAttribute(FileAttribute.MA_SENT_DIR)
-			+ "\n        Attributes: " + msg.getAttributes() + msg.getLogMsgID());
+			+ "\n        Original file name : " + msg.getPayloadFilename() + "\n        Sent file name : "
+			+ msg.getAttribute(FileAttribute.MA_FILENAME) + "\n        Pending message file : "
+			+ msg.getAttribute(FileAttribute.MA_PENDINGFILE) + "\n        Error directory: "
+			+ msg.getAttribute(FileAttribute.MA_ERROR_DIR) + "\n        Sent directory: "
+			+ msg.getAttribute(FileAttribute.MA_SENT_DIR) + "\n        Attributes: " + msg.getAttributes()
+			+ msg.getLogMsgID());
 	} catch (IOException e) {
 	    throw new OpenAS2Exception("Failed to retrieve the pending MDN information from file: "
 		    + org.openas2.logging.Log.getExceptionMsg(e), e);
@@ -692,8 +692,7 @@ public static void cleanupFiles(Message msg, boolean isError) {
 		    msg.setLogMsg("File was successfully sent but info file not deleted: " + pendingInfoFileName);
 		    logger.warn(msg, e);
 		}
-	    }
-	    else {
+	    } else {
 		msg.setLogMsg("Cleanup could not find pendinginfo file: " + pendingInfoFileName);
 		logger.warn(msg);
 	    }
@@ -717,18 +716,27 @@ public static void cleanupFiles(Message msg, boolean isError) {
 	    try {
 		boolean isMoved = false;
 		String tgtDir = null;
+		String targetFilenameUnparsed = "";
 		if (isError) {
 		    tgtDir = msg.getAttribute(FileAttribute.MA_ERROR_DIR);
+		    targetFilenameUnparsed = msg.getAttribute(FileAttribute.MA_ERROR_FILENAME);
 		} else {
 		    // If the Sent Directory option is set, move the transmitted file to the sent
 		    // directory
 		    tgtDir = msg.getAttribute(FileAttribute.MA_SENT_DIR);
+		    targetFilenameUnparsed = msg.getAttribute(FileAttribute.MA_SENT_FILENAME);
 		}
 		if (tgtDir != null && tgtDir.length() > 0) {
 		    File tgtFile = null;
-
 		    try {
-			tgtFile = new File(tgtDir + "/" + fPendingFile.getName());
+			String tgtFileName = fPendingFile.getName();
+			if (targetFilenameUnparsed != null && targetFilenameUnparsed.length() > 0) {
+			    CompositeParameters parser = new CompositeParameters(false)
+				    .add("date", new DateParameters()).add("msg", new MessageParameters(msg))
+				    .add("rand", new RandomParameters());
+			    tgtFileName = ParameterParser.parse(targetFilenameUnparsed, parser);
+			}
+			tgtFile = new File(tgtDir + "/" + tgtFileName);
 			tgtFile = IOUtil.moveFile(fPendingFile, tgtFile, false);
 			isMoved = true;
 
@@ -739,7 +747,7 @@ public static void cleanupFiles(Message msg, boolean isError) {
 		    } catch (IOException iose) {
 			msg.setLogMsg("Error moving file to " + tgtDir + " : "
 				+ org.openas2.logging.Log.getExceptionMsg(iose));
-			logger.error(msg,iose);
+			logger.error(msg, iose);
 		    }
 		}
 

changes.txt

@@ -1,3 +1,13 @@
+Version 2.9.2 - 2019-08-16
+This is a an enhancement and bugfix release:
+       **IMPORTANT NOTE**: Please review upgrade notes in the RELEASE-NOTES.md if you are upgrading
+
+  1. Fixed error where DB tracking module was not logging fields to database.
+  2. Added a script to connect to the H2 database from the command line.
+  3. Simplified the default config.xml for changing ports by using properties.
+  4. Added support for location of the config.xml using an environment variable in the start-openas2.sh file
+  5. Document stored_sent_filename and stored_error_filename as well as provide more examples for setting errordir and sentdir using extended dynamic variables
+
 Version 2.9.1 - 2019-08-03
 This is a an enhancement and bugfix release:
        **IMPORTANT NOTE**: Please review upgrade notes in the RELEASE-NOTES.md if you are upgrading

pom.xml

@@ -5,7 +5,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>net.sf.openas2</groupId>
 	<artifactId>OpenAS2</artifactId>
-	<version>2.9.1</version>
+	<version>2.9.2</version>
 	<name>OpenAS2</name>
 	<packaging>pom</packaging>