add ignore fields

usdot-fhwa-stol · Nov 28, 2023 · eed4b0e · eed4b0e
1 parent c095540
commit eed4b0e
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 25 deletions.
diff --git a/...m/telematic/telematic_cloud_messaging/message_converters/JSON2KeyValuePairsConverter.java b/...m/telematic/telematic_cloud_messaging/message_converters/JSON2KeyValuePairsConverter.java
@@ -10,10 +10,10 @@
 public class JSON2KeyValuePairsConverter {
     /**
      * @param json_str String JSON format consumed from NATS subject
-     * @param to_str_values Array of String to match in the JSON and convert the match to String data type value
+     * @param to_str_fields Array of String to match in the JSON and convert the match to String data type value
      * @return String of key value pairs separated by commas.
      */
-    public String convertJson2KeyValuePairs(String json_str, List<String> to_str_values) {
+    public String convertJson2KeyValuePairs(String json_str, List<String> to_str_fields, List<String> ignore_fields) {
         String pairs = "";
         JSONParser parser = new JSONParser();
         try {
@@ -22,21 +22,29 @@ public String convertJson2KeyValuePairs(String json_str, List<String> to_str_val
             for (Object key : json.keySet()) {
                 key_count++;
                 Object value = json.get(key.toString());
+                boolean is_ignored = false;
 
                 if (value == null || value.toString().isEmpty())
                 {
                     pairs += key + "=\"NA\"";
                 }
                 else {
-                    boolean is_skip = false;
-                    for (String value_item: to_str_values)
+                    for(String field: ignore_fields)
                     {
-                        if (key.toString().strip().equals(value_item)){
+                        if(key.toString().strip().equalsIgnoreCase(field))   
+                        {
+                            is_ignored = true;
+                        }
+                    }
+                    boolean is_processed = false;
+                    for (String field: to_str_fields)
+                    {
+                        if (key.toString().strip().equalsIgnoreCase(field)){
                             pairs += key + "=\"" + value.toString().replaceAll("\\s", "") + "\"";
-                            is_skip = true;
+                            is_processed = true;
                         }
                     }
-                    if(!is_skip)
+                    if(!is_processed && !is_ignored)
                     {
                         // Regex matching integers
                         if (value.toString().matches("[-+]?\\d*")) {
@@ -62,10 +70,15 @@ else if (value.toString().toLowerCase().strip().equals("nan")){
                     }                    
                 }              
 
-                if (json.keySet().size() != key_count) {
+                if (!is_ignored && json.keySet().size() != key_count) {
                     pairs += ",";
                 }
-            }
+                //If last field is ignored, remove the tail comma
+                if(is_ignored && json.keySet().size() == key_count && pairs.length() >= 1)
+                {
+                   pairs = pairs.substring(0, pairs.length()-1);
+                }
+            }//Loop through JSON keys
         } catch (ParseException e) {
             e.printStackTrace();
         }

diff --git a/.../src/main/java/com/telematic/telematic_cloud_messaging/nats_influx_connection/Config.java b/.../src/main/java/com/telematic/telematic_cloud_messaging/nats_influx_connection/Config.java
@@ -63,8 +63,10 @@ public enum BucketType{
     String streets_unit_id_list;
     // List of cloud unit ids
     String cloud_unit_id_list;
-    //List of values in the stream that should only be set to string data type
-    List<String> to_str_values;
+    //List of fields in the stream that should only be set to string data type
+    List<String> to_str_fields;
+    //List of fields in the stream that should be ignored
+    List<String> ignore_fields;
 
     public Config(){}
 
@@ -93,7 +95,8 @@ public String ToString(){
         "\nvehicle_unit_id_list: " + vehicle_unit_id_list +
         "\nstreets_unit_id_list: " + streets_unit_id_list +
         "\ncloud_unit_id_list: " + cloud_unit_id_list + 
-        "\nto_str_values:" + to_str_values.toString());
+        "\nto_str_fields:" + to_str_fields.toString() +
+        "\nignore_fields:" + ignore_fields.toString());
 
         return config_str;
 

diff --git a/...java/com/telematic/telematic_cloud_messaging/nats_influx_connection/InfluxDataWriter.java b/...java/com/telematic/telematic_cloud_messaging/nats_influx_connection/InfluxDataWriter.java
@@ -194,7 +194,7 @@ public String influxStringConverter(String publishData) {
         JSONObject payloadJson = publishDataJson.getJSONObject("payload");
 
         String flattenedPayloadJson = jsonFlattener.flattenJsonStr(payloadJson.toString());
-        String keyValuePairs = keyValueConverter.convertJson2KeyValuePairs(flattenedPayloadJson, config_.to_str_values);
+        String keyValuePairs = keyValueConverter.convertJson2KeyValuePairs(flattenedPayloadJson, config_.to_str_fields, config_.ignore_fields);
 
         String unit_id = publishDataJson.getString("unit_id").replaceAll("\\s", "_");
         String unit_type = publishDataJson.getString("unit_type").replaceAll("\\s", "_");

diff --git a/...n/java/com/telematic/telematic_cloud_messaging/nats_influx_connection/NatsInfluxPush.java b/...n/java/com/telematic/telematic_cloud_messaging/nats_influx_connection/NatsInfluxPush.java
@@ -73,7 +73,8 @@ static Config getConfigValues() {
             config.vehicle_unit_id_list = prop.getProperty("VEHICLE_UNIT_ID_LIST");
             config.streets_unit_id_list = prop.getProperty("STREETS_UNIT_ID_LIST");
             config.cloud_unit_id_list = prop.getProperty("CLOUD_UNIT_ID_LIST");
-            config.to_str_values = Arrays.asList(prop.getProperty("TO_STR_VALUES").split(","));
+            config.to_str_fields = Arrays.asList(prop.getProperty("TO_STR_FIELDS").split(","));
+            config.ignore_fields = Arrays.asList(prop.getProperty("IGNORE_FIELDS").split(","));
 
             try{
                 config.influx_bucket_type = BucketType.valueOf(prop.getProperty("INFLUX_BUCKET_TYPE"));

diff --git a/telematic_system/telematic_cloud_messaging/src/main/resources/application.properties b/telematic_system/telematic_cloud_messaging/src/main/resources/application.properties
@@ -27,7 +27,11 @@ INFLUX_ORG=my-org
 INFLUX_ORG_ID=<INFLUX_ORG_ID>
 INFLUX_TOKEN=<INFLUXDB-TOKEN>
 #Edge case for hostBSMId, sender_bsm_id, core_data.id and TCR/TCM ID where the Ids can be all digits or alpha characters
-TO_STR_VALUES=hostBSMId,TrafficControlRequest.reqid,tcmV01.reqid,m_header.sender_bsm_id,core_data.id
+TO_STR_FIELDS=hostBSMId,TrafficControlRequest.reqid,tcmV01.reqid,m_header.sender_bsm_id,core_data.id
+# Ignore data fields from message and do not save it into the influxDB. 
+# This is to prevent the data type conflicts: Some fields have random values like characters or numbers overtime. 
+# This confuse the influxDB as it does not know what type of field should be created to accommodate different values.
+IGNORE_FIELDS=payload.MessageFrame.value.PersonalSafetyMessage.id
 # Connection timeout to influx bucket. Unit: milliseconds
 INFLUX_CONNECT_TIMEOUT=1000
 # Timeout while writing data to influx. Unit: milliseconds