From 3ed5fa653d679183c6dfb5b6622627bd4052924a Mon Sep 17 00:00:00 2001 From: Arne Seime Date: Wed, 5 Apr 2017 16:35:17 +0200 Subject: [PATCH 1/2] Added input parameter to disable parsing of GTFS ids in format PREFIX.123. Defaults to original behaviour --- .../gtfs/importer/GtfsImportParameters.java | 4 +++ .../gtfs/parser/AbstractConverter.java | 18 ++++++++----- .../gtfs/parser/GtfsAgencyParser.java | 2 +- .../gtfs/parser/GtfsCalendarParser.java | 4 +-- .../exchange/gtfs/parser/GtfsRouteParser.java | 4 +-- .../exchange/gtfs/parser/GtfsStopParser.java | 4 +-- .../gtfs/parser/GtfsTransferParser.java | 6 ++--- .../exchange/gtfs/parser/GtfsTripParser.java | 12 ++++----- .../validation/GtfsValidationReporter.java | 6 ++--- .../gtfs/parser/AbstractConverterTest.java | 27 +++++++++++++++++++ 10 files changed, 61 insertions(+), 26 deletions(-) create mode 100644 mobi.chouette.exchange.gtfs/src/test/java/mobi/chouette/exchange/gtfs/parser/AbstractConverterTest.java diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java index 03d3008731..fa0a02a9f0 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java @@ -32,6 +32,10 @@ public class GtfsImportParameters extends AbstractImportParameter { @XmlElement(name = "object_id_prefix", required=true) private String objectIdPrefix; + @Getter@Setter + @XmlElement(name = "split_id_on_dot", defaultValue="true") + private boolean splitIdOnDot = true; + @Getter@Setter @XmlElement(name = "max_distance_for_connection_link", defaultValue="0") private int maxDistanceForConnectionLink = 0; diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/AbstractConverter.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/AbstractConverter.java index 3c59fc1805..b6686d685a 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/AbstractConverter.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/AbstractConverter.java @@ -5,6 +5,7 @@ import java.util.TimeZone; import mobi.chouette.common.Constant; +import mobi.chouette.exchange.gtfs.importer.GtfsImportParameters; import mobi.chouette.exchange.gtfs.model.GtfsTime; import org.apache.log4j.Logger; @@ -34,16 +35,19 @@ public static Time getTime(GtfsTime gtfsTime) { return time; } - public static String composeObjectId(String prefix, String type, String id, Logger logger) { + public static String composeObjectId(GtfsImportParameters configuration, String type, String id, Logger logger) { if (id == null || id.isEmpty() ) return ""; - String[] tokens = id.split("\\."); - if (tokens.length == 2) { - // id should be produced by Chouette - return tokens[0].trim().replaceAll("[^a-zA-Z_0-9]", "_") + ":" + type + ":" - + tokens[1].trim().replaceAll("[^a-zA-Z_0-9\\-]", "_"); + + if(configuration.isSplitIdOnDot()) { + String[] tokens = id.split("\\."); + if (tokens.length == 2) { + // id should be produced by Chouette + return tokens[0].trim().replaceAll("[^a-zA-Z_0-9]", "_") + ":" + type + ":"+ tokens[1].trim().replaceAll("[^a-zA-Z_0-9\\-]", "_"); + } } - return prefix + ":" + type + ":" + id.trim().replaceAll("[^a-zA-Z_0-9\\-]", "_"); + return configuration.getObjectIdPrefix() + ":" + type + ":" + id.trim().replaceAll("[^a-zA-Z_0-9\\-]", "_"); + } public static String toString(URL url) { diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsAgencyParser.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsAgencyParser.java index 9091c3ad33..dd072956d4 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsAgencyParser.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsAgencyParser.java @@ -102,7 +102,7 @@ public void parse(Context context) throws Exception { GtfsImportParameters configuration = (GtfsImportParameters) context.get(CONFIGURATION); for (GtfsAgency gtfsAgency : importer.getAgencyById()) { - String objectId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), Company.COMPANY_KEY, + String objectId = AbstractConverter.composeObjectId(configuration, Company.COMPANY_KEY, gtfsAgency.getAgencyId(), log); Company company = ObjectFactory.getCompany(referential, objectId); convert(context, gtfsAgency, company); diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsCalendarParser.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsCalendarParser.java index d33240c784..8f69355f09 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsCalendarParser.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsCalendarParser.java @@ -188,7 +188,7 @@ public void parse(Context context) throws Exception { if (importer.hasCalendarImporter()) { for (GtfsCalendar gtfsCalendar : importer.getCalendarByService()) { - String objectId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String objectId = AbstractConverter.composeObjectId(configuration, Timetable.TIMETABLE_KEY, gtfsCalendar.getServiceId(), log); Timetable timetable = ObjectFactory.getTimetable(referential, objectId); convert(context, gtfsCalendar, timetable); @@ -199,7 +199,7 @@ public void parse(Context context) throws Exception { for (String serviceId : importer.getCalendarDateByService().keys()) { - String objectId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String objectId = AbstractConverter.composeObjectId(configuration, Timetable.TIMETABLE_KEY, serviceId, log); Timetable timetable = referential.getTimetables().get(objectId); diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsRouteParser.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsRouteParser.java index 1f3027bca5..8fb36be2d3 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsRouteParser.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsRouteParser.java @@ -186,7 +186,7 @@ public void parse(Context context) throws Exception { Index routes = importer.getRouteById(); GtfsRoute gtfsRoute = routes.getValue(gtfsRouteId); - String lineId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), Line.LINE_KEY, + String lineId = AbstractConverter.composeObjectId(configuration, Line.LINE_KEY, gtfsRouteId, log); Line line = ObjectFactory.getLine(referential, lineId); convert(context, gtfsRoute, line); @@ -199,7 +199,7 @@ public void parse(Context context) throws Exception { // Company if (gtfsRoute.getAgencyId() != null) { - String companyId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String companyId = AbstractConverter.composeObjectId(configuration, Company.COMPANY_KEY, gtfsRoute.getAgencyId(), log); Company company = ObjectFactory.getCompany(referential, companyId); line.setCompany(company); diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsStopParser.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsStopParser.java index 3ccf8087dc..6ddb1fa14a 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsStopParser.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsStopParser.java @@ -113,7 +113,7 @@ public void parse(Context context) throws Exception { for (GtfsStop gtfsStop : importer.getStopById()) { if (gtfsStop.getLocationType() != GtfsStop.LocationType.Access) { - String objectId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String objectId = AbstractConverter.composeObjectId(configuration, StopArea.STOPAREA_KEY, gtfsStop.getStopId(), log); StopArea stopArea = ObjectFactory.getStopArea(referential, objectId); @@ -149,7 +149,7 @@ protected void convert(Context context, GtfsStop gtfsStop, StopArea stopArea) { } stopArea.setAreaType(ChouetteAreaEnum.BoardingPosition); if (gtfsStop.getParentStation() != null) { - String parenId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String parenId = AbstractConverter.composeObjectId(configuration, StopArea.STOPAREA_KEY, gtfsStop.getParentStation(), log); StopArea parent = ObjectFactory.getStopArea(referential, parenId); stopArea.setParent(parent); diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTransferParser.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTransferParser.java index 6b6c8bb3b2..f619337185 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTransferParser.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTransferParser.java @@ -105,7 +105,7 @@ public void parse(Context context) throws Exception { for (GtfsTransfer gtfsTransfer : importer.getTransferByFromStop()) { - String objectId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String objectId = AbstractConverter.composeObjectId(configuration, ConnectionLink.CONNECTIONLINK_KEY, gtfsTransfer.getFromStopId() + "_" + gtfsTransfer.getToStopId(), log); ConnectionLink connectionLink = ObjectFactory.getConnectionLink(referential, objectId); @@ -119,10 +119,10 @@ protected void convert(Context context, GtfsTransfer gtfsTransfer, ConnectionLin GtfsImportParameters configuration = (GtfsImportParameters) context.get(CONFIGURATION); StopArea startOfLink = ObjectFactory.getStopArea(referential, AbstractConverter.composeObjectId( - configuration.getObjectIdPrefix(), StopArea.STOPAREA_KEY, gtfsTransfer.getFromStopId(), log)); + configuration, StopArea.STOPAREA_KEY, gtfsTransfer.getFromStopId(), log)); connectionLink.setStartOfLink(startOfLink); StopArea endOfLink = ObjectFactory.getStopArea(referential, AbstractConverter.composeObjectId( - configuration.getObjectIdPrefix(), StopArea.STOPAREA_KEY, gtfsTransfer.getToStopId(), log)); + configuration, StopArea.STOPAREA_KEY, gtfsTransfer.getToStopId(), log)); connectionLink.setEndOfLink(endOfLink); connectionLink.setCreationTime(Calendar.getInstance().getTime()); connectionLink.setLinkType(ConnectionLinkTypeEnum.Overground); diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTripParser.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTripParser.java index 5ef7ab1b75..a1808a508a 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTripParser.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/parser/GtfsTripParser.java @@ -511,7 +511,7 @@ public void parse(Context context) throws Exception { if (!hasTimes) continue; - String objectId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String objectId = AbstractConverter.composeObjectId(configuration, VehicleJourney.VEHICLEJOURNEY_KEY, gtfsTrip.getTripId(), log); VehicleJourney vehicleJourney = ObjectFactory.getVehicleJourney(referential, objectId); convert(context, gtfsTrip, vehicleJourney); @@ -537,7 +537,7 @@ public void parse(Context context) throws Exception { Collections.sort(vehicleJourney.getVehicleJourneyAtStops(), VEHICLE_JOURNEY_AT_STOP_COMPARATOR); // Timetable - String timetableId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String timetableId = AbstractConverter.composeObjectId(configuration, Timetable.TIMETABLE_KEY, gtfsTrip.getServiceId(), log); if (afterMidnight) { timetableId += GtfsCalendarParser.AFTER_MIDNIGHT_SUFFIX; @@ -588,7 +588,7 @@ private void createJourneyFrequencies(Context context, Referential referential, for (GtfsFrequency frequency : importer.getFrequencyByTrip().values(gtfsTrip.getTripId())) { vehicleJourney.setJourneyCategory(JourneyCategoryEnum.Frequency); - String timeBandObjectId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), + String timeBandObjectId = AbstractConverter.composeObjectId(configuration, Timeband.TIMETABLE_KEY, gtfsTrip.getTripId() + "-" + count++, log); Timeband timeband = ObjectFactory.getTimeband(referential, timeBandObjectId); timeband.setName(getTimebandName(frequency)); @@ -813,14 +813,14 @@ private List createRouteSections(Context context, Referential refe * @return */ private Route createRoute(Referential referential, GtfsImportParameters configuration, GtfsTrip gtfsTrip) { - String lineId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), Line.LINE_KEY, + String lineId = AbstractConverter.composeObjectId(configuration, Line.LINE_KEY, gtfsTrip.getRouteId(), log); Line line = ObjectFactory.getLine(referential, lineId); String routeKey = gtfsTrip.getRouteId() + "_" + gtfsTrip.getDirectionId().ordinal(); if (gtfsTrip.getShapeId() != null && !gtfsTrip.getShapeId().isEmpty()) routeKey += "_" + gtfsTrip.getShapeId(); routeKey += "_" + line.getRoutes().size(); - String routeId = AbstractConverter.composeObjectId(configuration.getObjectIdPrefix(), Route.ROUTE_KEY, + String routeId = AbstractConverter.composeObjectId(configuration, Route.ROUTE_KEY, routeKey, log); Route route = ObjectFactory.getRoute(referential, routeId); @@ -910,7 +910,7 @@ private void createStopPoint(Route route, JourneyPattern journeyPattern, List Date: Wed, 5 Apr 2017 17:34:57 +0200 Subject: [PATCH 2/2] Added missing field name in xmltype --- .../chouette/exchange/gtfs/importer/GtfsImportParameters.java | 1 + 1 file changed, 1 insertion(+) diff --git a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java index fa0a02a9f0..c53c3a8594 100644 --- a/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java +++ b/mobi.chouette.exchange.gtfs/src/main/java/mobi/chouette/exchange/gtfs/importer/GtfsImportParameters.java @@ -21,6 +21,7 @@ @ToString(callSuper=true) @XmlAccessorType(XmlAccessType.FIELD) @XmlType(propOrder={"objectIdPrefix", + "splitIdOnDot", "maxDistanceForConnectionLink", "maxDistanceForCommercial", "ignoreEndChars",