Merge branch 'dev' into feature/set-msgRepeatOptions-based-on-context

docker-compose-confluent-cloud.yml

@@ -26,10 +26,6 @@ services:
       DATA_SIGNING_ENABLED_RSU: ${DATA_SIGNING_ENABLED_RSU}
       DATA_SIGNING_ENABLED_SDW: ${DATA_SIGNING_ENABLED_SDW}
       DEFAULT_SNMP_PROTOCOL: ${DEFAULT_SNMP_PROTOCOL}
-      # Commented out, will use SDW depositor module by default
-      #ODE_DEPOSIT_SDW_MESSAGES_OVER_WEBSOCKET: ${ODE_DEPOSIT_SDW_MESSAGES_OVER_WEBSOCKET}
-      #ODE_DDS_CAS_USERNAME: ${ODE_DDS_CAS_USERNAME}
-      #ODE_DDS_CAS_PASSWORD: ${ODE_DDS_CAS_PASSWORD}
     volumes:
       - ${DOCKER_SHARED_VOLUME}:/jpo-ode
       - ${DOCKER_SHARED_VOLUME}/uploads:/home/uploads

docker-compose-ppm-nsv.yml

@@ -22,15 +22,22 @@ services:
       KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
       ALLOW_PLAINTEXT_LISTENER: "yes"
       KAFKA_CFG_NODE_ID: "1"
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
-      KAFKA_CREATE_TOPICS: "topic.OdeBsmPojo:1:1,topic.OdeSpatTxPojo:1:1,topic.OdeSpatPojo:1:1,topic.OdeSpatJson:1:1,topic.FilteredOdeSpatJson:1:1,topic.OdeSpatRxJson:1:1,topic.OdeSpatRxPojo:1:1,topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.OdeTimBroadcastJson:1:1,topic.J2735TimBroadcastJson:1:1,topic.OdeDriverAlertJson:1:1,topic.Asn1DecoderInput:1:1,topic.Asn1DecoderOutput:1:1,topic.Asn1EncoderInput:1:1,topic.Asn1EncoderOutput:1:1,topic.SDWDepositorInput:1:1,topic.OdeTIMCertExpirationTimeJson:1:1,topic.OdeRawEncodedBSMJson:1:1,topic.OdeRawEncodedSPATJson:1:1,topic.OdeRawEncodedTIMJson:1:1,topic.OdeRawEncodedMAPJson:1:1,topic.OdeMapTxPojo:1:1,topic.OdeMapJson:1:1,topic.OdeRawEncodedSSMJson:1:1,topic.OdeSsmPojo:1:1,topic.OdeSsmJson:1:1,topic.OdeRawEncodedSRMJson:1:1,topic.OdeSrmTxPojo:1:1,topic.OdeSrmJson:1:1,topic.OdeRawEncodedPSMJson:1:1,topic.OdePsmTxPojo:1:1,topic.OdePsmJson:1:1"
       KAFKA_CFG_DELETE_TOPIC_ENABLE: "true"
       KAFKA_CFG_LOG_RETENTION_HOURS: 2
     logging:
       options:
         max-size: "10m"  
         max-file: "5"
 
+  kafka_init:
+    image: bitnami/kafka:latest
+    depends_on:
+      kafka:
+        condition: service_started
+    volumes:
+      - ./scripts/kafka/kafka_init.sh:/kafka_init.sh
+    entrypoint: ["/bin/sh", "kafka_init.sh"]
+
   ode:
     build: .
     image: jpoode_ode:latest
@@ -51,10 +58,6 @@ services:
       DATA_SIGNING_ENABLED_RSU: ${DATA_SIGNING_ENABLED_RSU}
       DATA_SIGNING_ENABLED_SDW: ${DATA_SIGNING_ENABLED_SDW}
       DEFAULT_SNMP_PROTOCOL: ${DEFAULT_SNMP_PROTOCOL}
-      # Commented out, will use SDW depositor module by default
-      #ODE_DEPOSIT_SDW_MESSAGES_OVER_WEBSOCKET: ${ODE_DEPOSIT_SDW_MESSAGES_OVER_WEBSOCKET}
-      #ODE_DDS_CAS_USERNAME: ${ODE_DDS_CAS_USERNAME}
-      #ODE_DDS_CAS_PASSWORD: ${ODE_DDS_CAS_PASSWORD}
     depends_on:
       - kafka
     volumes:
@@ -73,6 +76,7 @@ services:
       - kafka
     volumes:
       - ${DOCKER_SHARED_VOLUME}/adm:/asn1_codec_share
+    restart: on-failure
 
   aem:
     build: ./asn1_codec
@@ -107,6 +111,7 @@ services:
       - ${DOCKER_SHARED_VOLUME}/ppm_bsm:/ppm_data
     depends_on:
       - kafka
+    restart: on-failure
 
   ppm_tim:
     build:

docker-compose.yml

@@ -18,15 +18,22 @@ services:
       KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
       ALLOW_PLAINTEXT_LISTENER: "yes"
       KAFKA_CFG_NODE_ID: "1"
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
-      KAFKA_CREATE_TOPICS: "topic.OdeBsmPojo:1:1,topic.OdeSpatTxPojo:1:1,topic.OdeSpatPojo:1:1,topic.OdeSpatJson:1:1,topic.FilteredOdeSpatJson:1:1,topic.OdeSpatRxJson:1:1,topic.OdeSpatRxPojo:1:1,topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.OdeTimBroadcastJson:1:1,topic.J2735TimBroadcastJson:1:1,topic.OdeDriverAlertJson:1:1,topic.Asn1DecoderInput:1:1,topic.Asn1DecoderOutput:1:1,topic.Asn1EncoderInput:1:1,topic.Asn1EncoderOutput:1:1,topic.SDWDepositorInput:1:1,topic.OdeTIMCertExpirationTimeJson:1:1,topic.OdeRawEncodedBSMJson:1:1,topic.OdeRawEncodedSPATJson:1:1,topic.OdeRawEncodedTIMJson:1:1,topic.OdeRawEncodedMAPJson:1:1,topic.OdeMapTxPojo:1:1,topic.OdeMapJson:1:1,topic.OdeRawEncodedSSMJson:1:1,topic.OdeSsmPojo:1:1,topic.OdeSsmJson:1:1,topic.OdeRawEncodedSRMJson:1:1,topic.OdeSrmTxPojo:1:1,topic.OdeSrmJson:1:1,topic.OdeRawEncodedPSMJson:1:1,topic.OdePsmTxPojo:1:1,topic.OdePsmJson:1:1"
       KAFKA_CFG_DELETE_TOPIC_ENABLE: "true"
       KAFKA_CFG_LOG_RETENTION_HOURS: 2
     logging:
       options:
         max-size: "10m"  
         max-file: "5"
 
+  kafka_init:
+    image: bitnami/kafka:latest
+    depends_on:
+      kafka:
+        condition: service_started
+    volumes:
+      - ./scripts/kafka/kafka_init.sh:/kafka_init.sh
+    entrypoint: ["/bin/sh", "kafka_init.sh"]
+
   ode:
     build: .
     image: jpoode_ode:latest
@@ -52,10 +59,6 @@ services:
       DATA_SIGNING_ENABLED_RSU: ${DATA_SIGNING_ENABLED_RSU}
       DATA_SIGNING_ENABLED_SDW: ${DATA_SIGNING_ENABLED_SDW}
       DEFAULT_SNMP_PROTOCOL: ${DEFAULT_SNMP_PROTOCOL}
-      # Commented out, will use SDW depositor module by default
-      #ODE_DEPOSIT_SDW_MESSAGES_OVER_WEBSOCKET: ${ODE_DEPOSIT_SDW_MESSAGES_OVER_WEBSOCKET}
-      #ODE_DDS_CAS_USERNAME: ${ODE_DDS_CAS_USERNAME}
-      #ODE_DDS_CAS_PASSWORD: ${ODE_DDS_CAS_PASSWORD}
     depends_on:
       - kafka
     volumes:
@@ -83,6 +86,7 @@ services:
       options:
         max-size: "10m"
         max-file: "5"
+    restart: on-failure
 
   aem:
     build: ./asn1_codec
@@ -101,6 +105,7 @@ services:
       options:
         max-size: "10m"
         max-file: "5"
+    restart: on-failure
 
   ppm_bsm:
     build:

docs/Architecture.md

@@ -12,7 +12,7 @@ Booz Allen Hamilton\
 8283 Greensboro Drive\
 McLean, VA 22102
 
-_Last updated June 11, 2021_
+_Last updated April 26th, 2024_
 
 # Contents
 
@@ -32,7 +32,7 @@ _Last updated June 11, 2021_
 - [6 - Appendix](#appendix)
   - [6.1 - Glossary](#glossary)
 
-<a name="version-history">
+<a name="version-history"></a>
 
 # Version History
 
@@ -43,7 +43,7 @@ _Last updated June 11, 2021_
 | 1.1       | Hamid Musavi   | 4/17/2017                                         | Updated to reflect ORNL De-identification service |
 | 1.2       | Hamid Musavi   | 12/21/2018                                        | General Update                                    |
 
-<a name="introduction">
+<a name="introduction"></a>
 
 # 1 - Introduction
 
@@ -62,7 +62,7 @@ This document is a living document and will be updated throughout the
 life of the JPO ODE project to reflect the most recent changes in the
 ODE design and stakeholder feedback.
 
-<a name="project-overview">
+<a name="project-overview"></a>
 
 # 2 - Project Overview
 
@@ -79,13 +79,13 @@ brokering, processing and routing data from various data sources,
 including connected vehicles, field devices, Transportation Management
 Center (TMC) applications and a variety of other data users. Data users
 include but not limited to transportation software applications,
-Research Data Exchange (RDE), US DOT Situation Data Warehouse.
+Research Data Exchange (RDE), and the [Situational Data Exchange (SDX)](https://sdx.trihydro.com/).
 
 As a data provisioning service, the ODE can provision data from
 disparate data sources to software applications that have placed data
 subscription requests to the ODE. On the other direction, the ODE can
 accept data from CV applications and broadcast them to field devices
-through Road Side Units (RSU) and US DOT Situation Data Warehouse which
+through Road Side Units (RSU)s and the Situational Data Exchange (SDX) which
 in turn will transmit the data to Sirius XM satellites for delivery to
 the connected vehicles in the field.
 
@@ -102,7 +102,7 @@ validation and sanitization.
     compromise the privacy of the individual(s) that might be linked to
     the data.
 
-<a name="system-overview">
+<a name="system-overview"></a>
 
 # 3 - System Overview
 
@@ -112,7 +112,7 @@ operational, monitoring, and research purposes. The system will enable
 applications to submit data through a variety standard interfaces as
 illustrated in the figure below.
 
-<a name="ode-technology-stack">
+<a name="ode-technology-stack"></a>
 
 ### 3.1 - ODE Technology Stack
 
@@ -121,20 +121,20 @@ each technology layer.
 
 ![](images/architecture/figure1.png)
 
-Figure 1 - ODE Technology Stack
+_Figure 1 - ODE Technology Stack_
 
 Data Integration later of JPO-ODE supports the producers and consumers
-of CV data as illustrated in Figure 2 below. Not all components or
+of CV data as illustrated in Figure 2 below. It is possible that not all components or
 services shown in this diagram have been implemented. *The
 implementation timeline for the identified interfaces will depend on the
 needs of the JPO ODE customers and the priority of these capabilities to
 the JPO-ODE product owner.*
 
 ![](images/architecture/figure2.png)
 
-Figure 2 - ODE Data Integration Clients
+_Figure 2 - ODE Data Integration Clients_
 
-<a name="producer-mechanisms">
+<a name="producer-mechanisms"></a>
 
 ### 3.2 - Producer Mechanisms
 
@@ -174,12 +174,12 @@ Notation (JSON), environmental and various other system logs.
     and process new records. *This interface is suitable only to
     applications residing in the private network domain.*
 
-<a name="consumer-mechanisms">
+<a name="consumer-mechanisms"></a>
 
 ### 3.3 - Consumer Mechanisms
 
 The JPO-ODE is designed to support the following mechanisms for
-outputting decoded BSM, Map and Signal Phase and Timing (SPaT) data as
+outputting decoded BSM, Map, and Signal Phase & Timing (SPaT) data as
 well as encoded TIM data.
 
 -   **Streaming Data Consumers (Direct):** Applications can subscribe
@@ -191,33 +191,20 @@ well as encoded TIM data.
     to the messaging service through the use of a standard WebSocket
     API. *This interface is suitable to all applications whether
     residing in the private network domain or in the cloud. For cloud
-    applications Secure WebSocket (wss) protocol should be utillized.*
-
--   **RESTful API Data Consumers:** Applications can connect directly
-    with a RESTful API and submit messages to the messaging service
-    through HTTP commands. *This interface is suitable to all
-    applications whether residing in the private network domain or in
-    the cloud. For cloud applications Secure HTTP (https) protocol
-    should be utilized.*
-
--   **File System Data Consumers:** Through the use of a shared file
-    repository, applications can monitor collection of data messages.
-    *This interface is suitable to applications residing in the private
-    network domain or in the cloud. This interface should be utilized
-    through Secure Copy (scp) protocol.*
+    applications Secure WebSocket (wss) protocol should be utilized.*
 
 -   **Database Data Consumers**: Data messages can be directly inserted
     into a shared application database and made available for queries.
 
-<a name="ode-management-console">
+<a name="ode-management-console"></a>
 
 ### 3.4 - ODE Management Console
 
 ODE is envisioned to provide a management console for functions such as
 SNMP device management and provisioning. Other configuration functions
 can be included in a future management console.
 
-<a name="architecture-pattern">
+<a name="architecture-pattern"></a>
 
 # 4 - Architecture Pattern
 
@@ -228,7 +215,7 @@ and service-oriented architectures.
 
 <sup>1</sup> _In software engineering, a **monolithic application** describes a single-tiered software **application** in which the user interface and data access code are combined into a single program from a single platform. A **monolithic application** is self-contained, and independent from other computing **applications**._
 
-<a name="pattern-description">
+<a name="pattern-description"></a>
 
 ### 4.1 - Pattern Description
 
@@ -257,7 +244,7 @@ The micro-services pattern consists of three major concepts:
 
 _Figure 3 - Basic Micro-services architecture pattern_
 
-<a name="pattern-topology">
+<a name="pattern-topology"></a>
 
 ### 4.2 - Pattern Topology
 
@@ -351,7 +338,7 @@ designed as depicted in Figure 6.
 
 _Figure 6 - Kafka Publish/Subscribe Model_
 
-<a name="jpo-ode-micro-services-topology">
+<a name="jpo-ode-micro-services-topology"></a>
 
 # 5 - JPO ODE Micro-services Topology
 
@@ -363,22 +350,22 @@ the major architectural components with which these services interact.
 
 _Figure 7 - JPO ODE Micro-services Topology_
 
-<a name="deployments">
+<a name="deployments"></a>
 
 ### 5.1 - Deployments
 
 Docker is utilized as the primary deployment mechanism to
 compartmentalize each of the designed micro-services into separate
 containers. Docker is used to package all components in a composite of
 containers each running a distinct service. The ODE application runs in
-one container and other major frameworks such as Kafka run
-in their own separate containers.
+one container, its submodules run in separate containers and other major
+frameworks such as Kafka run in their own separate containers.
 
-<a name="appendix">
+<a name="appendix"></a>
 
 # 6 - Appendix
 
-<a name="glossary">
+<a name="glossary"></a>
 
 ### 6.1 - Glossary