From 484f96b6a0b044de0ecdc02711484b33af032d31 Mon Sep 17 00:00:00 2001 From: Markus Straub Date: Fri, 18 Nov 2022 12:31:13 +0100 Subject: [PATCH] update opencv also fix some resource leaks with try-with-resources --- pom.xml | 48 +++++++++---------- .../pt2matsim/editor/BasicScheduleEditor.java | 25 ++++++---- .../matsim/pt2matsim/gtfs/GtfsConverter.java | 1 - .../matsim/pt2matsim/gtfs/GtfsFeedImpl.java | 25 +++++++++- .../pt2matsim/hafas/lib/OperatorReader.java | 21 ++++---- .../org/matsim/pt2matsim/tools/CsvTools.java | 19 ++++---- .../org/matsim/pt2matsim/tools/GtfsTools.java | 8 ++-- .../matsim/pt2matsim/tools/ShapeTools.java | 9 ++-- .../pt2matsim/mapping/PTMapperTest.java | 1 - .../OsmMultimodalNetworkConverterTest.java | 1 - 10 files changed, 95 insertions(+), 63 deletions(-) diff --git a/pom.xml b/pom.xml index 0dc2689f..b4a18c17 100644 --- a/pom.xml +++ b/pom.xml @@ -11,10 +11,10 @@ - pt2matsim-releases - pt2matsim Releases Maven Repository - https://repo.matsim.org/repository/matsim-releases/ - + pt2matsim-releases + pt2matsim Releases Maven Repository + https://repo.matsim.org/repository/matsim-releases/ + @@ -23,14 +23,14 @@ https://repo.matsim.org/repository/matsim - osgeo - https://repo.osgeo.org/repository/release + osgeo + https://repo.osgeo.org/repository/release - - 24.2 - + + 24.2 + @@ -47,13 +47,13 @@ com.opencsv opencsv - 3.10 + 5.3 - commons-io - commons-io - 2.11.0 - + commons-io + commons-io + 2.11.0 + de.grundid.opendatalab geojson-jackson @@ -117,16 +117,16 @@ - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + diff --git a/src/main/java/org/matsim/pt2matsim/editor/BasicScheduleEditor.java b/src/main/java/org/matsim/pt2matsim/editor/BasicScheduleEditor.java index adcc6805..9be64219 100644 --- a/src/main/java/org/matsim/pt2matsim/editor/BasicScheduleEditor.java +++ b/src/main/java/org/matsim/pt2matsim/editor/BasicScheduleEditor.java @@ -18,7 +18,11 @@ package org.matsim.pt2matsim.editor; +import com.opencsv.CSVParserBuilder; import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; + import org.apache.log4j.Logger; import org.matsim.api.core.v01.Coord; import org.matsim.api.core.v01.Id; @@ -93,15 +97,20 @@ public TransitSchedule getSchedule() { */ @Override public void parseCommandCsv(String filePath) throws IOException { - CSVReader reader = new CSVReader(new FileReader(filePath), ';'); - - String[] line = reader.readNext(); - while(line != null) { - log.info(CollectionUtils.arrayToString(line)); - executeCmdLine(line); - line = reader.readNext(); + try (CSVReader reader = new CSVReaderBuilder(new FileReader(filePath)) + .withCSVParser(new CSVParserBuilder() + .withSeparator(';') + .build()) + .build()) { + String[] line = reader.readNext(); + while (line != null) { + log.info(CollectionUtils.arrayToString(line)); + executeCmdLine(line); + line = reader.readNext(); + } + } catch (CsvValidationException e) { + throw new RuntimeException(e); } - reader.close(); } /** diff --git a/src/main/java/org/matsim/pt2matsim/gtfs/GtfsConverter.java b/src/main/java/org/matsim/pt2matsim/gtfs/GtfsConverter.java index 08d203da..cca21103 100644 --- a/src/main/java/org/matsim/pt2matsim/gtfs/GtfsConverter.java +++ b/src/main/java/org/matsim/pt2matsim/gtfs/GtfsConverter.java @@ -20,7 +20,6 @@ import org.apache.log4j.Logger; import org.matsim.api.core.v01.Id; -import org.matsim.core.utils.misc.OptionalTime; import org.matsim.core.utils.misc.Time; import org.matsim.pt.transitSchedule.api.*; import org.matsim.pt2matsim.gtfs.lib.*; diff --git a/src/main/java/org/matsim/pt2matsim/gtfs/GtfsFeedImpl.java b/src/main/java/org/matsim/pt2matsim/gtfs/GtfsFeedImpl.java index 3e2a6df6..a53c13a4 100755 --- a/src/main/java/org/matsim/pt2matsim/gtfs/GtfsFeedImpl.java +++ b/src/main/java/org/matsim/pt2matsim/gtfs/GtfsFeedImpl.java @@ -19,6 +19,8 @@ package org.matsim.pt2matsim.gtfs; import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvValidationException; + import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; import org.apache.commons.io.input.BOMInputStream; @@ -225,6 +227,8 @@ protected void loadStops() throws IOException { reader.close(); } catch (ArrayIndexOutOfBoundsException e) { throw new RuntimeException("Line " + l + " in stops.txt is empty or malformed."); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } log.info("... stops.txt loaded"); } @@ -271,6 +275,8 @@ protected boolean loadCalendar() { return false; } catch (ArrayIndexOutOfBoundsException i) { throw new RuntimeException("Line " + l + " in calendar.txt is empty or malformed."); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } log.info("... calendar.txt loaded"); return true; @@ -324,6 +330,8 @@ protected boolean loadCalendarDates() { return false; } catch (ArrayIndexOutOfBoundsException i) { throw new RuntimeException("Line " + l + " in calendar_dates.txt is empty or malformed."); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } return true; } @@ -367,6 +375,8 @@ protected void loadShapes() { log.info("... no shapes file found."); } catch (ArrayIndexOutOfBoundsException i) { throw new RuntimeException("Line " + l + " in shapes.txt is empty or malformed."); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } } @@ -410,6 +420,8 @@ protected void loadRoutes() throws IOException { reader.close(); } catch (ArrayIndexOutOfBoundsException i) { throw new RuntimeException("Line " + l + " in routes.txt is empty or malformed."); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } log.info("... routes.txt loaded"); } @@ -470,6 +482,8 @@ protected void loadTrips() throws IOException { reader.close(); } catch (ArrayIndexOutOfBoundsException i) { throw new RuntimeException("Line " + l + " in trips.txt is empty or malformed."); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } log.info("... trips.txt loaded"); } @@ -555,6 +569,8 @@ protected void loadStopTimes() throws IOException { reader.close(); } catch (ArrayIndexOutOfBoundsException i) { throw new RuntimeException("Line " + l + " in stop_times.txt is empty or malformed."); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } log.info("... stop_times.txt loaded"); } @@ -609,6 +625,8 @@ protected void loadFrequencies() { throw new RuntimeException("Line " + l + " in frequencies.txt is empty or malformed."); } catch (IOException e) { e.printStackTrace(); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } } @@ -652,6 +670,8 @@ protected void loadTransfers() { transfersFileFound = false; } catch (IOException e) { e.printStackTrace(); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } if(transfersFileFound) { log.info("... transfers.txt loaded"); @@ -662,14 +682,15 @@ protected String unzip(String compressedZip) { String unzippedFolder = compressedZip.substring(0, compressedZip.length() - 4) + "/"; log.info("Unzipping " + compressedZip + " to " + unzippedFolder); - try { - ZipFile zipFile = new ZipFile(compressedZip); + try (ZipFile zipFile = new ZipFile(compressedZip)) { if(zipFile.isEncrypted()) { throw new RuntimeException("Zip file is encrypted"); } zipFile.extractAll(unzippedFolder); } catch (ZipException e) { e.printStackTrace(); + } catch(IOException e) { + throw new RuntimeException(e); } return unzippedFolder; } diff --git a/src/main/java/org/matsim/pt2matsim/hafas/lib/OperatorReader.java b/src/main/java/org/matsim/pt2matsim/hafas/lib/OperatorReader.java index b0e924ad..489c13a4 100644 --- a/src/main/java/org/matsim/pt2matsim/hafas/lib/OperatorReader.java +++ b/src/main/java/org/matsim/pt2matsim/hafas/lib/OperatorReader.java @@ -37,16 +37,17 @@ public class OperatorReader { public static Map readOperators(String BETRIEB_DE) throws IOException { Map operators = new HashMap<>(); - BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(BETRIEB_DE), "utf-8")); - String newLine = readsLines.readLine(); - while (newLine != null) { - String abbrevationOperator = newLine.split("\"")[1].replace(" ",""); - newLine = readsLines.readLine(); - if (newLine == null) break; - String operatorId = newLine.substring(8, 14).trim(); - operators.put(operatorId, abbrevationOperator); - // read the next operator: - newLine = readsLines.readLine(); + try(BufferedReader readsLines = new BufferedReader(new InputStreamReader(new FileInputStream(BETRIEB_DE), "utf-8"))) { + String newLine = readsLines.readLine(); + while (newLine != null) { + String abbrevationOperator = newLine.split("\"")[1].replace(" ",""); + newLine = readsLines.readLine(); + if (newLine == null) break; + String operatorId = newLine.substring(8, 14).trim(); + operators.put(operatorId, abbrevationOperator); + // read the next operator: + newLine = readsLines.readLine(); + } } return operators; } diff --git a/src/main/java/org/matsim/pt2matsim/tools/CsvTools.java b/src/main/java/org/matsim/pt2matsim/tools/CsvTools.java index 60b03708..29258185 100644 --- a/src/main/java/org/matsim/pt2matsim/tools/CsvTools.java +++ b/src/main/java/org/matsim/pt2matsim/tools/CsvTools.java @@ -20,6 +20,8 @@ package org.matsim.pt2matsim.tools; import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvValidationException; + import org.matsim.core.utils.collections.MapUtils; import org.matsim.core.utils.collections.Tuple; @@ -142,15 +144,16 @@ public static void writeNestedMapToFile(Object[] header, Map> readNestedMapFromFile(String fileName, boolean ignoreFirstLine) throws IOException { Map> map = new HashMap<>(); - - CSVReader reader = new CSVReader(new FileReader(fileName)); - if(ignoreFirstLine) reader.readNext(); - String[] line = reader.readNext(); - while(line != null) { - MapUtils.getMap(line[0], map).put(line[1], line[2]); - line = reader.readNext(); + try(CSVReader reader = new CSVReader(new FileReader(fileName))) { + if(ignoreFirstLine) reader.readNext(); + String[] line = reader.readNext(); + while(line != null) { + MapUtils.getMap(line[0], map).put(line[1], line[2]); + line = reader.readNext(); + } + } catch (CsvValidationException e) { + throw new RuntimeException(e); } - reader.close(); return map; } diff --git a/src/main/java/org/matsim/pt2matsim/tools/GtfsTools.java b/src/main/java/org/matsim/pt2matsim/tools/GtfsTools.java index 0ea38f21..409873f2 100644 --- a/src/main/java/org/matsim/pt2matsim/tools/GtfsTools.java +++ b/src/main/java/org/matsim/pt2matsim/tools/GtfsTools.java @@ -137,7 +137,7 @@ public static Map> getTripsOndates(GtfsFeed feed) { * usually the largest file. */ public static void writeStopTimes(Collection trips, String folder) throws IOException { - CSVWriter stopTimesWriter = new CSVWriter(new FileWriter(folder + GtfsDefinitions.Files.STOP_TIMES.fileName), ','); + CSVWriter stopTimesWriter = new CSVWriter(new FileWriter(folder + GtfsDefinitions.Files.STOP_TIMES.fileName)); String[] header = GtfsDefinitions.Files.STOP_TIMES.columns; stopTimesWriter.writeNext(header, true); @@ -161,7 +161,7 @@ public static void writeStopTimes(Collection trips, String folder) throws * Experimental class to write stops.txt (i.e. after filtering for one date) */ public static void writeStops(Collection stops, String path) throws IOException { - CSVWriter stopsWriter = new CSVWriter(new FileWriter(path + GtfsDefinitions.Files.STOPS.fileName), ','); + CSVWriter stopsWriter = new CSVWriter(new FileWriter(path + GtfsDefinitions.Files.STOPS.fileName)); String[] header = GtfsDefinitions.Files.STOPS.columns; stopsWriter.writeNext(header, true); for(Stop stop : stops) { @@ -180,7 +180,7 @@ public static void writeStops(Collection stops, String path) throws IOExce * Experimental class to write trips.txt (i.e. after filtering for one date) */ public static void writeTrips(Collection trips, String path) throws IOException { - CSVWriter tripsWriter = new CSVWriter(new FileWriter(path + GtfsDefinitions.Files.TRIPS.fileName), ','); + CSVWriter tripsWriter = new CSVWriter(new FileWriter(path + GtfsDefinitions.Files.TRIPS.fileName)); String[] header = GtfsDefinitions.Files.TRIPS.columns; tripsWriter.writeNext(header, true); for(Trip trip : trips) { @@ -198,7 +198,7 @@ public static void writeTrips(Collection trips, String path) throws IOExce * Experimental class to write transfers.txt (i.e. after creating additional walk transfer) */ public static void writeTransfers(Collection transfers, String path) throws IOException { - CSVWriter transfersWiter = new CSVWriter(new FileWriter(path + GtfsDefinitions.Files.TRANSFERS.fileName), ','); + CSVWriter transfersWiter = new CSVWriter(new FileWriter(path + GtfsDefinitions.Files.TRANSFERS.fileName)); String[] columns = GtfsDefinitions.Files.TRANSFERS.columns; String[] optionalColumns = GtfsDefinitions.Files.TRANSFERS.optionalColumns; String[] header = Stream.concat(Arrays.stream(columns), Arrays.stream(optionalColumns)).toArray(String[]::new); diff --git a/src/main/java/org/matsim/pt2matsim/tools/ShapeTools.java b/src/main/java/org/matsim/pt2matsim/tools/ShapeTools.java index c30aec90..ad1c3e18 100644 --- a/src/main/java/org/matsim/pt2matsim/tools/ShapeTools.java +++ b/src/main/java/org/matsim/pt2matsim/tools/ShapeTools.java @@ -19,6 +19,8 @@ package org.matsim.pt2matsim.tools; import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvValidationException; + import org.locationtech.jts.geom.Coordinate; import org.matsim.api.core.v01.Coord; import org.matsim.api.core.v01.Id; @@ -263,9 +265,7 @@ public static Map, RouteShape> readShapesFile(String shapeFile, S Map, RouteShape> shapes = new HashMap<>(); CoordinateTransformation ct = TransformationFactory.getCoordinateTransformation("WGS84", outputCoordinateSystem); - CSVReader reader; - try { - reader = new CSVReader(new FileReader(shapeFile)); + try (CSVReader reader = new CSVReader(new FileReader(shapeFile))) { String[] header = reader.readNext(); Map col = getIndices(header, GtfsDefinitions.Files.SHAPES.columns); String[] line = reader.readNext(); @@ -280,7 +280,6 @@ public static Map, RouteShape> readShapesFile(String shapeFile, S currentShape.addPoint(ct.transform(point), Integer.parseInt(line[col.get(GtfsDefinitions.SHAPE_PT_SEQUENCE)])); line = reader.readNext(); } - reader.close(); } catch (FileNotFoundException e) { e.printStackTrace(); throw new RuntimeException("File not found!"); @@ -288,6 +287,8 @@ public static Map, RouteShape> readShapesFile(String shapeFile, S throw new RuntimeException("Emtpy line found file!"); } catch (IOException e) { e.printStackTrace(); + } catch (CsvValidationException e) { + throw new RuntimeException(e); } return shapes; } diff --git a/src/test/java/org/matsim/pt2matsim/mapping/PTMapperTest.java b/src/test/java/org/matsim/pt2matsim/mapping/PTMapperTest.java index 78004810..bec1b13c 100644 --- a/src/test/java/org/matsim/pt2matsim/mapping/PTMapperTest.java +++ b/src/test/java/org/matsim/pt2matsim/mapping/PTMapperTest.java @@ -11,7 +11,6 @@ import org.matsim.pt.transitSchedule.api.TransitSchedule; import org.matsim.pt.transitSchedule.api.TransitStopFacility; import org.matsim.pt.utils.TransitScheduleValidator; -import org.matsim.pt2matsim.config.OsmConverterConfigGroup; import org.matsim.pt2matsim.config.PublicTransitMappingConfigGroup; import org.matsim.pt2matsim.config.PublicTransitMappingStrings; import org.matsim.pt2matsim.run.CreateDefaultPTMapperConfig; diff --git a/src/test/java/org/matsim/pt2matsim/osm/OsmMultimodalNetworkConverterTest.java b/src/test/java/org/matsim/pt2matsim/osm/OsmMultimodalNetworkConverterTest.java index d26bb33b..39c80bf8 100644 --- a/src/test/java/org/matsim/pt2matsim/osm/OsmMultimodalNetworkConverterTest.java +++ b/src/test/java/org/matsim/pt2matsim/osm/OsmMultimodalNetworkConverterTest.java @@ -18,7 +18,6 @@ import org.matsim.pt2matsim.osm.lib.OsmDataImpl; import org.matsim.pt2matsim.osm.lib.OsmFileReader; import org.matsim.pt2matsim.run.CreateDefaultOsmConfig; -import org.matsim.pt2matsim.run.CreateDefaultPTMapperConfig; /** * @author polettif