Merge pull request #563 from Imvertor/FB-Documentor

Fb documentor

src/main/java/nl/imvertor/common/Configurator.java

@@ -250,6 +250,10 @@ public AnyFolder getBaseFolder() {
 	public AnyFolder getWorkFolder() {
 		return workFolder;
 	}
+
+	public AnyFolder getInputFolder() {
+		return inputFolder;
+	}
 
 	public AnyFolder getWorkFolder(String subfolderName) {
 		return new AnyFolder(workFolder,subfolderName);
@@ -1040,7 +1044,7 @@ private void loadFromPropertyFile(String filePath) throws Exception {
 			String optionName = e.nextElement().toString();
 			String value = properties.getProperty(optionName);
 			// process file properties in context of the current file
-			if (optionName.equals("umlfile") | optionName.equals("zipfile") | optionName.equals("hisfile")) {
+			if (optionName.equals("umlfile") | optionName.equals("zipfile") | optionName.equals("hisfile") | optionName.equals("documentorfile")) {
 				File parent = (new File(filePath)).getParentFile();
 				if (AnyFile.isAbsolutePath(value))
 					value = (new File(value)).getCanonicalPath();

src/main/java/nl/imvertor/common/file/HttpFile.java

@@ -5,13 +5,15 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.NameValuePair;
@@ -220,13 +222,7 @@ public int getStatus() {
 		return status;
 	}
 	public String getResponseBody(HttpResponse response) throws Exception, IOException {
-		BufferedReader in = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
-		String inputLine;
-		StringBuffer responseText = new StringBuffer();
-		while ((inputLine = in.readLine()) != null)
-			responseText.append(inputLine);
-		in.close();
-		return responseText.toString();
+		return IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
 	}
 
 }

src/main/java/nl/imvertor/common/file/WordFile.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright (C) 2016 Dienst voor het kadaster en de openbare registers
+ * 
+ * This file is part of Imvertor.
+ *
+ * Imvertor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Imvertor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Imvertor.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package nl.imvertor.common.file;
+
+import java.io.File;
+import java.net.URI;
+import java.util.Base64;
+import java.util.HashMap;
+
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHeaders;
+import org.apache.log4j.Logger;
+import org.json.JSONObject;
+
+import nl.imvertor.common.Configurator;
+import nl.imvertor.common.Runner;
+import nl.imvertor.common.Transformer;
+import nl.imvertor.common.helper.OsExecutor;
+import nl.imvertor.common.helper.OsExecutor.OsExecutorResultHandler;
+
+public class WordFile extends AnyFile {
+
+	public static final Logger logger = Logger.getLogger(ExecFile.class);
+	private static final long serialVersionUID = 2409879811971148189L;
+
+	public WordFile(String filepath) {
+		super(filepath);
+	}
+	public WordFile(File file) {
+		super(file.getAbsolutePath());
+	}
+	public WordFile(File file, String subpath) {
+		super(file, subpath);
+	}
+
+	/**
+	 * Transform the Doc file to XHTML structure.
+	 * This uses Pandoc.
+	 * 
+	 * @return XmlFile
+	 * @param filePath
+	 * @throws Exception 
+	 */
+	public boolean toXhtmlFile(XmlFile outFile) throws Exception {
+
+		Configurator configurator = Configurator.getInstance();
+		Runner runner = configurator.getRunner();
+		boolean debugging = runner.getDebug("DOCUMENTOR");
+
+		String pandocServerUrl = configurator.getServerProperty("pandoc.server", false);
+		if (pandocServerUrl != null) {
+			// gebruik de pandoc server: https://pandoc.org/pandoc-server.html
+
+			HttpFile localFile = new HttpFile(this);
+
+			HashMap<String,String> headerMap = new HashMap<String,String>();
+			//headerMap.put(HttpHeaders.AUTHORIZATION,"token " + OAUTH);
+			headerMap.put(HttpHeaders.ACCEPT, "application/octet-stream");
+			headerMap.put(HttpHeaders.CONTENT_TYPE, "application/json");
+			headerMap.put(HttpHeaders.CONTENT_ENCODING, getEncoding());
+
+			String payloadBase64 = Base64.getEncoder().encodeToString(this.getBinaryContent());
+			String payload = "{"
+					+ "\"from\": \"docx+styles\","
+					+ "\"to\": \"html\","
+					+ "\"indented-code-classes\": ["
+						+ "\"Programmacode\""
+					+ "],"
+					+ "\"section-divs\": true,"
+					+ "\"embed-resources\": true,"
+					//+ "\"ipynb-output\": \"all\","
+					+ "\"standalone\": true,"
+					+ "\"variables\": {"
+						+ "\"lang\": \"nl-NL\","
+						+ "\"title-meta\": \"NOTITLE\""
+					+ "},"
+					+ "\"text\": \"" + payloadBase64 + "\""
+					+ "}";
+			try {
+				String result = localFile.post(HttpFile.METHOD_POST_CONTENT, URI.create(pandocServerUrl), headerMap, null, new String[] {payload});
+				if (StringUtils.startsWith(result,"<")) {
+					outFile.setContent(result);
+					return true;
+				} else {
+					runner.error(logger, "Documentor processing error: \"" + result + "\"");
+					return false;
+				}
+			} catch (Exception e) {
+				runner.error(logger, "Documentor server error: \"" + e.getMessage() + "\"");
+			}
+			return false;
+		} else {
+			// Implementatie van Pandoc omzetting naar XHTML.
+
+			OsExecutor osExecutor = new OsExecutor();
+
+			String toolloc = (new AnyFile(configurator.getServerProperty("documentor.msword.transformer"))).getCanonicalPath(); // location of the tool
+			long osExecutorJobTimeout = Long.parseLong(configurator.getServerProperty("documentor.msword.transformer.timeout")); // location of the tool
+			boolean osExecutorInBackground = false;
+
+			OsExecutorResultHandler osExecutorResult = null;
+
+			/*
+			 * Dit batch file doet het volgende
+			 * - Bereid MsWord voor door roep o.a. pandoc aan en corrigeert allerlei 
+			 */
+			CommandLine commandLine = new CommandLine(toolloc + "\\documentor.bat"); // TODO: *nix
+			commandLine.addArgument(this.getName()); // the docx file
+			commandLine.addArgument(toolloc); // the tool folder
+			commandLine.addArgument(this.getParent()); // The work folder
+			commandLine.addArgument(debugging ? "true" : "false"); // debugging?
+
+			try {
+				osExecutorResult = osExecutor.osexec(commandLine, osExecutorJobTimeout, osExecutorInBackground);
+				osExecutorResult.waitFor();
+				// assume the msword file * is transformed to *.xhtml
+				configurator.setXParm("appinfo/documentor-transformation-result", outFile.getName(),false);
+				return true;
+
+			} catch (Exception e) {
+				if (osExecutorResult != null)
+					runner.error(logger, "Documentor exit value " + osExecutorResult.getExitValue() + ". " + osExecutorResult.getException().getMessage());
+				else 
+					runner.error(logger, e.getMessage());
+			}
+		}
+		return false;
+
+	}
+
+	/*
+	 * Uitlezen van msword gaat niet altijd goed; in preserveSpace secties worden leading blanks niet goed verwerkt. Vervang deze door harde spaties.
+	 */
+	public boolean correctCodeSpaces() throws Exception{
+		Configurator configurator = Configurator.getInstance();
+		try {
+			ZipFile thisFile = new ZipFile(this);
+			AnyFolder tempFolder = configurator.getWorkFolder("documentor/msword");
+			thisFile.decompress(tempFolder);
+			// run stylesheet om de spaties in code vast te zetten
+			XmlFile wordFile = new XmlFile(tempFolder,"/word/document.xml");
+			XmlFile outFile = new XmlFile(tempFolder,"/word/document.xml.transformed");
+			XslFile xslFile = new XslFile(configurator.getBaseFolder() + "/xsl/OfficeCompiler/Imvert2documentor-msword-fixes.xsl");
+			Transformer transformer = new Transformer();
+			transformer.transform(wordFile, outFile, xslFile, "filefix");
+			// zet het resultaat van de transformatie op de plek van het input document
+			outFile.copyFile(wordFile);
+			outFile.delete();
+			// Maak het MdWord bestand opnieuw aan door de tijdelijke folder weer te zippen.
+			thisFile.compress(tempFolder);
+			return true;
+		} catch (Exception e) {
+			configurator.getRunner().error(logger,"Cannot fix MsWord program code fragment(s)",e);
+			return false;
+		}
+	}
+}

src/main/java/nl/imvertor/common/file/ZipFile.java

@@ -21,19 +21,12 @@
 package nl.imvertor.common.file;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Vector;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import java.util.zip.ZipOutputStream;
 
 import javax.xml.transform.Transformer;
 import javax.xml.transform.TransformerFactory;
@@ -63,6 +56,7 @@ public ZipFile(String zipFilePath) throws IOException {
 	}
 	public ZipFile(File file) throws IOException {
 		super(file);
+		fileList = new ArrayList<String>();
 	}
 	public ZipFile(File folder, String file) throws IOException {
 		super(folder, file);
@@ -79,9 +73,7 @@ public ZipFile(File folder, String file) throws IOException {
 	 */
     public void compress(File folderToCompress) throws Exception {
     	if (folderToCompress.isDirectory()) {
-    	  	sourceFolder = folderToCompress.getAbsolutePath();
-           	generateFileList(folderToCompress);
-        	zipIt();
+    	  	ZipUtils.zipDirectory(folderToCompress, this); 
     	} else 
     		throw new Exception("Source folder for zip is not a folder: " + folderToCompress);
     }
@@ -94,113 +86,9 @@ public void compress(File folderToCompress) throws Exception {
      * @throws Exception 
      */
 	public void decompress(AnyFolder targetFolder) throws Exception {
-		unZipIt(this.getAbsolutePath(),targetFolder.getAbsolutePath(),null);
-	}
-	public void decompress(AnyFolder targetFolder,Pattern requestedFilePattern) throws Exception {
-		if (targetFolder.isFile())
-			throw new Exception("Cannot decompress to a file: " + targetFolder);
-		if (!targetFolder.exists() )
-			targetFolder.mkdirs();
-		unZipIt(this.getAbsolutePath(),targetFolder.getAbsolutePath(),requestedFilePattern);
+		ZipUtils.unzipFile(this, targetFolder);
 	}
-
-    /**
-     * Traverse a folder and get all files,
-     * and add the file into fileList  
-     * @param node file or folder
-     */
-    // adapted from http://www.mkyong.com/java/how-to-compress-files-in-zip-format/
-    private void generateFileList(File node){
-
-    	//add file only
-    	if(node.isFile()) {
-    		fileList.add(generateZipEntry(node.getAbsoluteFile().toString()));
-    	}
-    	if(node.isDirectory()){
-    		String[] subNote = node.list();
-    		for(String filename : subNote){
-    			generateFileList(new File(node, filename));
-    		}
-    	}
-    }
-
-    /**
-     * Format the file path for zip
-     * @param file file path
-     * @return Formatted file path
-     */
-    private String generateZipEntry(String file){
-    	return file.substring(sourceFolder.length()+1, file.length());
-    }
-
-    /**
-     * Zip it
-     * @throws IOException 
-     */
-	// adapted from http://www.mkyong.com/java/how-to-compress-files-in-zip-format/
-    private void zipIt() throws IOException {
-    	byte[] buffer = new byte[1024];
-		FileOutputStream fos = new FileOutputStream(this);
-		ZipOutputStream zos = new ZipOutputStream(fos);
-		for(String file : this.fileList){
-			ZipEntry ze = new ZipEntry(file);
-			zos.putNextEntry(ze);
-			FileInputStream in = new FileInputStream(sourceFolder + File.separator + file);
-			int len;
-			while ((len = in.read(buffer)) > 0) {
-				zos.write(buffer, 0, len);
-			}
-			in.close();
-		}
-		zos.closeEntry();
-		zos.close();
-    }
-
-    /**
-     * Unzip it
-     * @param zipFile input zip file
-     * @param outputFolder zip file output folder
-     * @param requestedFilePattern Pattern to match the file name.
-     * 
-     * @throws Exception 
-     */
-    private void unZipIt(String zipFile, String outputFolder, Pattern requestedFilePattern) throws Exception{
-
-    	byte[] buffer = new byte[1024];
-
-		//create output folder is not exists
-		AnyFolder folder = new AnyFolder(outputFolder);
-		folder.mkdir();
-
-		//get the zip file content
-		ZipInputStream zis = 
-				new ZipInputStream(new FileInputStream(zipFile));
-		//get the zipped file list entry
-		ZipEntry ze = zis.getNextEntry();
-
-		while(ze!=null){
-			if (!ze.isDirectory()) {
-				String fileName = ze.getName();
-				// if the pattern specified, use a matcher. Otherwise accept any file.
-				Matcher m = (requestedFilePattern != null) ? requestedFilePattern.matcher(fileName) : null;
-				if (requestedFilePattern == null || m.find()) {
-					File newFile = new File(outputFolder + File.separator + fileName);
-					//create all non exists folders
-					//else you will hit FileNotFoundException for compressed folder
-					new File(newFile.getParent()).mkdirs();
-					FileOutputStream fos = new FileOutputStream(newFile);             
-					int len;
-					while ((len = zis.read(buffer)) > 0) {
-						fos.write(buffer, 0, len);
-					}
-					fos.close();   
-				}
-			}
-			ze = zis.getNextEntry();
-		}
-		zis.closeEntry();
-		zis.close();
-    }
+
     /**
      * Create an XML serialization in a folder that will hold 
      * 1/ a single xml file content.xml