Use RFC4180 for delimiter reader

mccullen · Feb 2, 2021 · 9ae549e · 9ae549e
1 parent f441d36
commit 9ae549e
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/src/main/java/icapa/Runner.java b/src/main/java/icapa/Runner.java
@@ -27,7 +27,7 @@
 public class Runner implements Serializable {
     static private final Logger LOGGER = Logger.getLogger(Runner.class.getName());
     public static final String CONFIG_FILENAME = "config.properties";
-    public static final String MAIN_PIPER = "C:/root/vdt/icapa/nlp/custom-components/src/main/resources/sql-server.piper";
+    public static final String MAIN_PIPER = "C:/root/vdt/icapa/nlp/custom-components/src/main/resources/default-pipeline.piper";
     public static int n = 0;
     private icapa.models.ConfigurationSettings _config = new icapa.models.ConfigurationSettings();
 

diff --git a/src/main/java/icapa/services/DelimiterReaderService.java b/src/main/java/icapa/services/DelimiterReaderService.java
@@ -3,6 +3,7 @@
 import com.opencsv.CSVParserBuilder;
 import com.opencsv.CSVReader;
 import com.opencsv.CSVReaderBuilder;
+import com.opencsv.RFC4180ParserBuilder;
 import icapa.Util;
 import icapa.models.DelimiterReaderParams;
 import org.apache.ctakes.typesystem.type.structured.DocumentID;
@@ -38,7 +39,8 @@ public static DelimiterReaderService from(DelimiterReaderParams params) {
         result._rowEnd = params.getRowEnd();
         // TODO: Remove hard dependency on CSVReader. Create a wrapper class/interface
         result._csvReader = new CSVReaderBuilder(result._reader)
-            .withCSVParser(new CSVParserBuilder().withSeparator(params.getDelimiter()).build())
+            // Make sure to use RFC4180 parser. That is the standard. CSVParser came out before the standard. Don't use CSVParser
+            .withCSVParser(new RFC4180ParserBuilder().withSeparator(params.getDelimiter()).build())
             .build();
         // Skip to the right row
         String[] headers = new String[0];

diff --git a/src/main/resources/default-pipeline.piper b/src/main/resources/default-pipeline.piper
@@ -1,5 +1,5 @@
 // Custom reader
-reader icapa.cr.DelimiterReader InputFile=C:/root/tmp/mimiciii/pyctakes_notes_in.mimiciii.top3.csv DocumentIdColumnName=note_id NoteColumnName=note DocumentIdColumnName=note_id
+reader icapa.cr.DelimiterReader InputFile=C:/root/tmp/mimiciii/pyctakes_notes_in.mimiciii.top2.csv DocumentIdColumnName=note_id NoteColumnName=note DocumentIdColumnName=note_id
 // Without specifying document id
 //reader icapa.cr.DelimiterReader InputFile=C:/root/tmp/mimiciii/pyctakes_notes_in.mimiciii.top3.csv NoteColumnName=note