diff --git a/.gitmodules b/.gitmodules index ff394be..095c39f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "pugixml"] path = pugixml url = https://github.com/zeux/pugixml.git -[submodule "asn1c"] - path = asn1c - url = https://github.com/mouse07410/asn1c +[submodule "usdot-asn1c"] + path = usdot-asn1c + url = https://github.com/usdot-fhwa-stol/usdot-asn1c diff --git a/Dockerfile b/Dockerfile index b670165..b3d5478 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ADD ./pugixml /asn1_codec/pugixml RUN cd /asn1_codec/pugixml && mkdir -p build && cd build && cmake .. && make && make install # Build and install asn1c submodule -ADD ./asn1c /asn1_codec/asn1c +ADD ./usdot-asn1c /asn1_codec/asn1c RUN cd asn1c && test -f configure || autoreconf -iv && ./configure && make && make install # Make generated files available to the build & compile example diff --git a/Dockerfile.dev b/Dockerfile.dev index 9421e78..65efec3 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -23,7 +23,7 @@ ADD ./pugixml /asn1_codec/pugixml RUN cd /asn1_codec/pugixml && mkdir -p build && cd build && cmake .. && make && make install # Build and install asn1c submodule -ADD ./asn1c /asn1_codec/asn1c +ADD ./usdot-asn1c /asn1_codec/asn1c RUN cd asn1c && test -f configure || autoreconf -iv && ./configure && make && make install # Make generated files available to the build & compile example diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 5188063..2624714 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -22,7 +22,7 @@ ADD ./pugixml /asn1_codec/pugixml RUN cd /asn1_codec/pugixml && mkdir -p build && cd build && cmake .. && make && make install # Build and install asn1c submodule -ADD ./asn1c /asn1_codec/asn1c +ADD ./usdot-asn1c /asn1_codec/asn1c RUN cd asn1c && test -f configure || autoreconf -iv && ./configure && make && make install # Make generated files available to the build & compile example diff --git a/README.md b/README.md index 1ebbd8d..a311a66 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ one of two functions depending on how it is started: 1. **Decode**: This function is used to process messages *from* the connected vehicle environment *to* ODE subscribers. Specifically, the ACM extacts binary -data from consumed messages (ODE Metatdata Messages) and decodes the binary +data from consumed messages (ODE Metadata Messages) and decodes the binary ASN.1 data into a structure that is subsequently encoded into an alternative format more suitable for ODE subscribers (currently XML using XER). @@ -17,22 +17,25 @@ is subsequently *encoded into ASN.1 binary data*. ![ASN.1 Codec Operations](docs/graphics/asn1codec-operations.png) -# Table of Contents +## Table of Contents +**README.md** 1. [Release Notes](#release-notes) 1. [Getting Involved](#getting-involved) 1. [Documentation](#documentation) +1. [Generating C Files from ASN.1 Definitions](#generating-c-files-from-asn1-definitions) +1. [Confluent Cloud Integration](#confluent-cloud-integration) + +**Other Documents** 1. [Installation](docs/installation.md) 1. [Configuration and Operation](docs/configuration.md) 1. [Interface](docs/interface.md) 1. [Testing](docs/testing.md) -1. [Project Management](#project-management) -1. [Confluent Cloud Integration](#confluent-cloud-integration) ## Release Notes The current version and release history of the asn1_codec: [asn1_codec Release Notes]() -# Getting Involved +## Getting Involved This project is sponsored by the U.S. Department of Transportation and supports Operational Data Environment data type conversions. Here are some ways you can start getting involved in this project: @@ -42,7 +45,7 @@ conversions. Here are some ways you can start getting involved in this project: - If you would like to improve this code base or the documentation, [fork the project](https://github.com/usdot-jpo-ode/asn1_codec#fork-destination-box) and submit a pull request. - If you find a problem with the code or the documentation, please submit an [issue](https://github.com/usdot-jpo-ode/asn1_codec/issues/new). -## Introduction +### Introduction This project uses the [Pull Request Model](https://help.github.com/articles/using-pull-requests). This involves the following project components: @@ -56,7 +59,7 @@ request to the organization asn1_codec project. One the project's main developer or, if there are issues, discuss them with the submitter. This will ensure that the developers have a better understanding of the code base *and* we catch problems before they enter `master`. The following process should be followed: -## Initial Setup +### Initial Setup 1. If you do not have one yet, create a personal (or organization) account on GitHub (assume your account name is ``). 1. Log into your personal (or organization) account. @@ -66,19 +69,19 @@ understanding of the code base *and* we catch problems before they enter `master $ git clone https://github.com//asn1_codec.git ``` -## Additional Resources for Initial Setup +### Additional Resources for Initial Setup - [About Git Version Control](http://git-scm.com/book/en/v2/Getting-Started-About-Version-Control) - [First-Time Git Setup](http://git-scm.com/book/en/Getting-Started-First-Time-Git-Setup) - [Article on Forking](https://help.github.com/articles/fork-a-repo) -# Documentation +## Documentation This documentation is in the `README.md` file. Additional information can be found using the links in the [Table of Contents](#table-of-contents). All stakeholders are invited to provide input to these documents. Stakeholders should direct all input on this document to the JPO Product Owner at DOT, FHWA, or JPO. -## Code Documentation +### Code Documentation Code documentation can be generated using [Doxygen](https://www.doxygen.org) by following the commands below: @@ -89,36 +92,30 @@ $ doxygen ``` The documentation is in HTML and is written to the `/asn1_codec/docs/html` directory. Open `index.html` in a -browser. - -## Project Management +browser. -This project is managed using the Jira tool. +## Generating C Files from ASN.1 Definitions +Check here for instructions on how to generate C files from ASN.1 definitions: [ASN.1 C File Generation](asn1c_combined/README.md) -- [Jira Project Portal](https://usdotjpoode.atlassian.net/secure/Dashboard.jsp) +This should only be necessary if the ASN.1 definitions change. The generated files are already included in the repository. -# Confluent Cloud Integration +## Confluent Cloud Integration Rather than using a local kafka instance, the ACM can utilize an instance of kafka hosted by Confluent Cloud via SASL. -## Environment variables -### Purpose & Usage +### Environment variables +#### Purpose & Usage - The DOCKER_HOST_IP environment variable is used to communicate with the bootstrap server that the instance of Kafka is running on. - The KAFKA_TYPE environment variable specifies what type of kafka connection will be attempted and is used to check if Confluent should be utilized. - The CONFLUENT_KEY and CONFLUENT_SECRET environment variables are used to authenticate with the bootstrap server. -### Values +#### Values - DOCKER_HOST_IP must be set to the bootstrap server address (excluding the port) - KAFKA_TYPE must be set to "CONFLUENT" - CONFLUENT_KEY must be set to the API key being utilized for CC - CONFLUENT_SECRET must be set to the API secret being utilized for CC -## CC Docker Compose File +### CC Docker Compose File There is a provided docker-compose file (docker-compose-confluent-cloud.yml) that passes the above environment variables into the container that gets created. Further, this file doesn't spin up a local kafka instance since it is not required. -## Note -This has only been tested with Confluent Cloud but technically all SASL authenticated Kafka brokers can be reached using this method. - -# Generating C Files from ASN.1 Definitions -Check here for instructions on how to generate C files from ASN.1 definitions: [ASN.1 C File Generation](asn1c_combined/README.md) - -This should only be necessary if the ASN.1 definitions change. The generated files are already included in the repository. \ No newline at end of file +### Note +This has only been tested with Confluent Cloud but technically all SASL authenticated Kafka brokers can be reached using this method. \ No newline at end of file diff --git a/asn1c_combined/README.md b/asn1c_combined/README.md index dac9bd7..11abcc6 100644 --- a/asn1c_combined/README.md +++ b/asn1c_combined/README.md @@ -6,6 +6,18 @@ This script expects the necessary files to have already been generated by the `g ## Generating the C Code The necessary files can be generated using the `generate-files.sh` script. This script will reference the necessary ASN.1 files from the `j2735-asn-files` directory from the specified year and generate the C code in the `generated-files` directory. +### Installing asn1c +The `generate-files.sh` script requires the `asn1c` compiler to be installed. The `asn1c` compiler can be installed in WSL using the following commands: + +```bash +cd ./usdot-asn1c +aclocal +test -f configure || autoreconf -iv +./configure +make +sudo make install +``` + ### J2735 ASN Files The 'j2735-asn-files' subdirectory should contain the ASN.1 files for the J2735 standard. These are organized by year. diff --git a/asn1c_combined/generated-files/2016.tar.gz b/asn1c_combined/generated-files/2016.tar.gz index dc37ac5..95ea9bd 100644 Binary files a/asn1c_combined/generated-files/2016.tar.gz and b/asn1c_combined/generated-files/2016.tar.gz differ diff --git a/asn1c_combined/generated-files/2020.tar.gz b/asn1c_combined/generated-files/2020.tar.gz index 1609ae2..4de059a 100644 Binary files a/asn1c_combined/generated-files/2020.tar.gz and b/asn1c_combined/generated-files/2020.tar.gz differ diff --git a/build_local.sh b/build_local.sh index 2cc0eb9..bc952f5 100644 --- a/build_local.sh +++ b/build_local.sh @@ -19,9 +19,9 @@ initializeSubmodules(){ buildAsn1c(){ # build asn1c echo "${GREEN}Building asn1c${NC}" - cd ./asn1c + cd ./usdot-asn1c git reset --hard - git pull origin master + git pull origin vlm_master aclocal test -f ./configure || autoreconf -iv ./configure diff --git a/docs/Release_notes.md b/docs/Release_notes.md index 6a3ee13..2ddaa40 100644 --- a/docs/Release_notes.md +++ b/docs/Release_notes.md @@ -1,6 +1,20 @@ asn1_codec Release Notes ---------------------------- +Version 2.1.0, released June 2024 +---------------------------------------- +### **Summary** +The changes for the asn1_codec 2.1.0 release include revised documentation for accuracy and improved clarity and readability. Additionally, there has been a transition to using `usdot-fhwa-stol/usdot-asn1c` instead of `mouse07410/asn1c` for the ASN.1 Compiler. + +Enhancements in this release: +- CDOT PR 25: Revised documentation for accuracy & improved clarity/readability +- CDOT PR 26: Transitioned to using `usdot-fhwa-stol/usdot-asn1c` instead of `mouse07410/asn1c` for ASN.1 Compiler + +Known Issues: +- The do_kafka_test.sh script in the project's root directory is currently not running successfully. The issue is being investigated and will be addressed in a future update. +- According to Valgrind, a minor memory leak has been detected. The development team is aware of this and is actively working on resolving it. + + Version 2.0.0, released February 2024 ---------------------------------------- diff --git a/docs/configuration.md b/docs/configuration.md index daf1831..d879f52 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,4 +1,4 @@ -# ACM Operation +# ACM Configuration & Operation The ASN.1 Codec Module (ACM) processes Kafka data streams that preset [ODE Metadata](http://github.com/usdot-jpo-ode/jpo-ode/blob/develop/docs/Metadata_v3.md) wrapped ASN.1 data. It can perform @@ -6,7 +6,7 @@ one of two functions depending on how it is started: 1. **Decode**: This function is used to process messages *from* the connected vehicle environment *to* ODE subscribers. Specifically, the ACM extacts binary -data from consumed messages (ODE Metatdata Messages) and decodes the binary +data from consumed messages (ODE Metadata Messages) and decodes the binary ASN.1 data into a structure that is subsequently encoded into an alternative format more suitable for ODE subscribers (currently XML using XER). @@ -15,7 +15,7 @@ the connected vehicle environment. Specifically, the ACM extracts human-readable data from ODE Metadata and decodes it into a structure that is subsequently *encoded into ASN.1 binary data*. -![ASN.1 Codec Operations](https://github.com/usdot-jpo-ode/asn1_codec/blob/master/docs/graphics/asn1codec-operations.png) +![ASN.1 Codec Operations](graphics/asn1codec-operations.png) ## ACM Command Line Options @@ -41,7 +41,23 @@ line options override parameters specified in the configuration file.** The foll -v | --log-level : The info log level [trace,debug,info,warning,error,critical,off] ``` -# ACM Deployment +## Environment Variables +The following environment variables are used by the ACM: + +------------------- +| Variable | Description | +| --- | --- | +| `DOCKER_HOST_IP` | The IP address of the machine running the kafka cluster. | +| `ACM_LOG_TO_CONSOLE` | Whether or not to log to the console. | +| `ACM_LOG_TO_FILE` | Whether or not to log to a file. | +| `ACM_LOG_LEVEL` | The log level to use. Valid values are: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "OFF" | +| `KAFKA_TYPE` | If unset, a local kafka broker will be targeted. If set to "CONFLUENT", the application will target a Confluent Cloud cluster. | +| `CONFLUENT_KEY` | Confluent Cloud Integration (if KAFKA_TYPE is set to "CONFLUENT") | +| `CONFLUENT_SECRET` | Confluent Cloud Integration (if KAFKA_TYPE is set to "CONFLUENT") | + +The `sample.env` file contains the default values for some of these environment variables. To use these values, copy the `sample.env` file to `.env` and modify the values as needed. + +## ACM Deployment Once the ACM is [installed and configured](installation.md) it operates as a background service. The ACM can be started before or after other services. If started before the other services, it may produce some error messages while it waits @@ -55,7 +71,7 @@ Multiple ACM processes can be started. Each ACM will use its own configuration f deploy a decoder and an encoder, start two separate ACM services with different `-T` options. In this case different topics should be specified in the configuration files. -# ACM ASN.1 Data Sources +## ACM ASN.1 Data Sources The ACM receives XML data from the ODE; the schema for this XML message is described in [Metadata.md](https://github.com/usdot-jpo-ode/jpo-ode/blob/develop/docs/Metadata_v3.md) on the ODE. Currently, the ACM @@ -84,7 +100,7 @@ can handle: \* Denotes the message should already contain hex data, according the the ASN.1 specification for that message. For instance, IEEE 1609.2 must contain hex data in its `unsecuredData` tag. If the hex data is missing or invalid, -the ACM with likely generate an error when doing constraint checking. +the ACM will likely generate an error when doing constraint checking. - After ENCODING this text is changed to: `us.dot.its.jpo.ode.model.OdeHexByteArray` - After DECODING this text is changed to: `us.dot.its.jpo.ode.model.OdeXml` @@ -92,7 +108,7 @@ the ACM with likely generate an error when doing constraint checking. Both the ENCODER and DECODER will check the ASN.1 constraints for the C structures that are built as data passes through the module. -# ACM Kafka Limitations +## ACM Kafka Limitations With regard to the Apache Kafka architecture, each ACM process does **not** provide a way to take advantage of Kafka's scalable architecture. In other words, each ACM process will consume data from a single Kafka topic and a single partition within @@ -100,7 +116,7 @@ that topic. One way to consume topics with multiple partitions is to launch one configuration file will allow you to designate the partition. In the future, the ACM may be updated to automatically handle multiple partitions within a single topic. -# ACM Logging +## ACM Logging ACM operations are optionally logged to the console and/or to a file. The file is a rotating log file, i.e., a set number of log files will be used to record the ACM's information. By default, the file is in a `logs` directory from where the ACM is launched and the file is @@ -136,7 +152,7 @@ preceeded with a date and time stamp and the level of the log message. [171011 18:25:55.221442] [trace] ending configure() ``` -# ACM Configuration +## ACM Configuration The ACM configuration file is a text file with a prescribed format. It can be used to configure Kafka as well as the ACM. Comments can be added to the configuration file by starting a line with the '#' character. Configuration lines consist @@ -171,7 +187,7 @@ Example configuration files can be found in the [asn1_codec/config](../config) d The details of the settings and how they affect the function of the ACM follow: -## ODE Kafka Interface +### ODE Kafka Interface - `asn1.topic.producer` : The Kafka topic name where the ACM will write its output. **The name is case sensitive.** @@ -194,20 +210,18 @@ The details of the settings and how they affect the function of the ACM follow: - `compression.type` : The type of compression to use for writing to Kafka topics. Currently, this should be set to none. -# ACM Testing with Kafka +## ACM Testing with Kafka -There are four steps that need to be started / run as separate processes. +The necessary services for testing the ACM with Kafka are provided in the `docker-compose.yml` file. The following steps will guide you through the process of testing the ACM with Kafka. -1. Start the `kafka-docker` container to provide the basic kafka data streaming services. - -```bash -$ docker-compose up --no-recreate -d +1. Start the Kafka & ACM services via the provided `docker-compose.yml` file. +``` +$ docker compose up --build -d ``` -1. Start the ACM (here we are starting a decoder). - -```bash -$ ./acm -c config/example.properties -T decode +1. Exec into the Kafka container to gain access to the Kafka command line tools. +``` +$ docker exec -it asn1_codec_kafka_1 /bin/bash ``` 1. Use the provided `kafka-console-producer.sh` script (provided with the Apache Kafka installation) to send XML @@ -225,4 +239,9 @@ $ ./bin/kafka-console-consumer.sh --bootstrap-server ${SERVER_IP} --topic ${ACM_ ``` The log files will provide more information about the details of the processing that is taking place and document any -errors. +errors. To view log files, exec into the ACM container and use the `tail` command to view the log file. + +```bash +$ docker exec -it asn1_codec_acm_1 /bin/bash +$ tail -f logs/log +``` diff --git a/docs/installation.md b/docs/installation.md index b8c26ac..23b5a89 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,61 +1,126 @@ # Installation and Setup +## Table of Contents +1. [Docker Installation](#docker-installation) +1. [Manual Installation](#manual-installation) + +## Docker Installation + +### 1. Install [Docker](https://www.docker.com) + +- When following the website instructions, setup the Docker repos and follow the Linux post-install instructions. +- The CE version seems to work fine. +- [Docker installation instructions](https://docs.docker.com/engine/installation/linux/ubuntu/#install-using-the-repository) + +#### ORNL Specific Docker Configuration +- *ORNL specific, but may apply to others with organizational security* + - Correct for internal Google DNS blocking + - As root (`$ sudo su`), create a `daemon.json` file in the `/etc/docker` directory that contains the following information: +```bash + { + "debug": true, + "default-runtime": "runc", + "dns": ["160.91.126.23","160.91.126.28"], + "icc": true, + "insecure-registries": [], + "ip": "0.0.0.0", + "log-driver": "json-file", + "log-level": "info", + "max-concurrent-downloads": 3, + "max-concurrent-uploads": 5, + "oom-score-adjust": -500 + } +``` +- NOTE: The DNS IP addresses are ORNL specific. + +Be sure to restart the docker daemon to consume the new configuration file. + +```bash +$ service docker stop +$ service docker start +``` + +Check the configuration using the command below to confirm the updates above are taken if needed: + +```bash +$ docker info +``` + +### 2. Configure environment variables +Configure the environment variables for the ACM to communicate with the Kafka instance. Copy or rename the `sample.env` file to `.env`. + +```bash +$ cp sample.env .env +``` + +Edit the `.env` file to include the necessary information. + +```bash +$ vi .env +``` + +For more information on the environment variables, see the 'Environment Variables' section in the [configuration.md](configuration.md) file. + +### 3. Spin up Kafka & the ACM in Docker +To spin up the ACM and Kafka in Docker, use the following commands: + +```bash +docker compose up --build +``` + +## Manual Installation The ACM can be executed after following these high-level steps: 1. Obtain the necessary build tools. 1. Obtain the ACM code, submodules, and documentation. 1. Build and install the submodules. This includes the ASN.1 compiler. -1. Location or purchase the necessary ASN.1 specifications. -1. Use the ASN.1 Compiler to compile the needed ASN.1 specifications into C source code. -1. Use the ASN.1 Compiler generated `Makefile.am.sample` to generate an ASN.1-specific library for the ACM. +1. Extract previously generated header/implementation files. 1. Build the ACM. 1. Run the ACM. Docker may also be used to run the ACM. The [Docker Container](#docker-installation) instructions have more information. -## 1. Install [Git](https://git-scm.com/) - -When asked if you wish to continue after running the command below, type 'Y'. +### 1. Install [Git](https://git-scm.com/) ```bash -$ sudo apt install git +$ sudo apt install -y git ``` -## 2. Install [CMake](https://cmake.org) +### 2. Install [CMake](https://cmake.org) -The ACM uses CMake to build. When asked if you wish to continue after running the command below, type 'Y'. +The ACM uses CMake to build. ```bash -$ sudo apt install cmake +$ sudo apt install -y cmake ``` -## 3. Install Requirements for Autotools +### 3. Install Requirements for Autotools -The ASN.1 Compiler (asn1c) uses autotools to build. When asked if you wish to continue after running the command below, type 'Y'. +The ASN.1 Compiler (asn1c) uses autotools to build. ```bash -$ sudo apt install autoconf -$ sudo apt install libtool +$ sudo apt install -y autoconf +$ sudo apt install -y libtool ``` -## 4. Install librdkafka +### 4. Install librdkafka -Talking to a Kafka instance, subscribing and producting to topics requires the use of a third party library. We use the librdkafka library as a c/c++ implementation. This can be imported as a submodule, or installed in your development machine: +Talking to a Kafka instance, subscribing and producting to topics requires the use of a third party library. We use the librdkafka library as a c/c++ implementation. This can be installed as a package. ```bash -$ sudo apt install libsasl2-dev -$ sudo apt install libsasl2-modules -$ sudo apt install libssl-dev -$ sudo apt install librdkafka-dev +$ sudo apt install -y libsasl2-dev +$ sudo apt install -y libsasl2-modules +$ sudo apt install -y libssl-dev +$ sudo apt install -y librdkafka-dev ``` -## 5. Create a base directory from which to install all the necessary components to test the ACM. +### 5. Create a base directory from which to install all the necessary components to test the ACM. ```bash $ export GIT_REPOS=~/some/dir/you/want/to/put/this/stuff ``` -## 6. Install the ACM Code, Submodules, and Documentation +### 6. Install the ACM Code, Submodules, and Documentation ```bash $ cd $GIT_REPOS @@ -64,7 +129,7 @@ $ git clone --recurse-submodules https://github.com/usdot-jpo-ode/asn1_codec.git The ACM uses code from several other open source projects, namely `librdkafka`, `asn1c`, `pugixml`, `scms-asn`, `spdlog`, and `catch`. `spdlog` and `catch` are header only. The versions of these headers that were tested with the project are -included in the ACM project. The other repositories are submodules that must be built; the `--recurse-submodules` option +included in the ACM project. The other repositories are submodules that must be built (excluding librdkafka); the `--recurse-submodules` option retrieves these submodules. The following instructions will help you build these libraries (`asn1c` and `pugixml`). After the ACM has been cloned the following commands can be used to update the submodules. @@ -74,9 +139,9 @@ $ cd $GIT_REPOS/asn1_codec $ git submodule update --init --recursive ``` -## 7. Build the Submodules +### 7. Build the Submodules -### a. Build the ASN.1 Compiler (asn1c). +#### a. Build the ASN.1 Compiler (asn1c). The documentation for building and installing the [ASN.1 Compiler](https://github.com/vlm/asn1c) (asn1c) is on github. The top level documentation is in the `README.md` file in the root directory of the repository you just pulled and @@ -102,10 +167,9 @@ $ make $ sudo make install ``` -Note: If you are having additional problems check the [requirements page](https://github.com/vlm/asn1c/blob/master/REQUIREMENTS.md), -although I did not need `bison` or `flex` to get asn1c to build. +Note: If you are having additional problems check the [requirements page](https://github.com/vlm/asn1c/blob/master/REQUIREMENTS.md), although `bison` or `flex` may not be needed to get asn1c to build. -### b. Build the pugixml library. +#### b. Build the pugixml library. The ACM uses [pugixml](https://pugixml.org) to process XML (ASN.1 XER); XML is the ACM's human-readable format. The pugixml library is built using CMake. Although the [pugixml quickstart](https://pugixml.org/docs/quickstart.html) @@ -125,41 +189,22 @@ Note: an out of source build is also an option and works. This should install these libraries in the usual locations, i.e., `/usr/local/lib` `/usr/local/include` -## 8. Compile the J2735 and IEEE 1609.2 ASN.1 Specification Files and Build the CV ASN.1 Codec Library - -The ACM project's `asn1c_combined` subdirectory is where a library that performs connected vehicle specific ASN.1 -encoding and decoding will be built. More specifically, this library decodes various ASN.1 binary encodings (e.g., UPER, -OER) into C structures that conform the ASN.1 specification files that were compiled. It also encodes the C structures -back into other encodings (e.g., XER, or XML). This subdirectory must contain the ASN.1 specification files needed to -encode and decode connected vehicle communications. It contains a script to compile the specifications and build the -library (and a command line tool that can be used independently to encode and decode ASN.1 files). +### 8. Extract Previously Generated Header/Implementation Files +The `asn1c_combined` directory contains the header and implementation files that were generated from the ASN.1 compiler. These files are included in the repository, but if you need to regenerate them, follow the instructions in the [ASN.1 C File Generation](asn1c_combined/README.md) file. -### a. Obtain the Necessary ASN.1 Specification Files - -The above subdirectory must contain the below specification files; all of these are included except the j2735 -specification that needs to be [purchased from SAE](https://saemobilus.sae.org/content/j2735_201603). These -specifications are subject to change and should be updated periodically. - -- [IEEE 1609.2](https://github.com/wwhyte-si/1609dot2-asn.git) - - `1609dot2-base-types.asn` - - `1609dot2-schema.asn` -- Southeast Michigan Testbed Specification - - `SEMI_v2.3.0_070616.asn` -- SAE J2735 - - `J2735_201603DA.ASN` - -### b. Compile the Specifications and Build the Library +The header and implementation files can be extracted by running the `doIt.sh` script in the `asn1c_combined` directory. ```bash $ cd $GIT_REPOS/asn1_codec/asn1c_combined $ ./doIt.sh ``` -During compilation numerous source code files will be generated in or moved into this directory. It will also create the -static library, `libasncodec.a`, and the command line tool, `converter-example`. The latter is not -needed, but useful. Do not remove the header files that are generated in this directory +During compilation numerous source code files will be generated in or moved into this directory. It will also compile the command line tool, `converter-example`, which is not +needed, but useful. + +Do not remove the header files that are generated in this directory -## 9. Build the ACM +### 9. Build the ACM Finally, build the ACM. The ACM is build using CMake. @@ -171,7 +216,7 @@ $ cmake .. $ make ``` -## Additional information +### Additional information The following projects are used by the ACM; however, they are header-only and those headers are included in the ACM repository. You can pull fresh headers from the following locations: @@ -179,96 +224,3 @@ ACM repository. You can pull fresh headers from the following locations: - The ACM uses [spdlog](https://github.com/gabime/spdlog) for logging; it is a header-only library and the headers are included in the repository. - The ACM uses [Catch](https://github.com/philsquared/Catch) for unit testing, but it is a header-only library included in the repository. - Currently, the ACM does not use [RapidJSON](https://github.com/miloyip/rapidjson), but this header-only library may be used in the future. - -# Docker Installation - -## 1. Install [Docker](https://www.docker.com) - -- When following the website instructions, setup the Docker repos and follow the Linux post-install instructions. -- The CE version seems to work fine. -- [Docker installation instructions](https://docs.docker.com/engine/installation/linux/ubuntu/#install-using-the-repository) -- *ORNL specific, but may apply to others with organizational security* - - Correct for internal Google DNS blocking - - As root (`$ sudo su`), create a `daemon.json` file in the `/etc/docker` directory that contains the following information: -```bash - { - "debug": true, - "default-runtime": "runc", - "dns": ["160.91.126.23","160.91.126.28"], - "icc": true, - "insecure-registries": [], - "ip": "0.0.0.0", - "log-driver": "json-file", - "log-level": "info", - "max-concurrent-downloads": 3, - "max-concurrent-uploads": 5, - "oom-score-adjust": -500 - } -``` -- NOTE: The DNS IP addresses are ORNL specific. - -## 2. Restart the docker daemon to consume the new configuration file. - -```bash -$ service docker stop -$ service docker start -``` - -## 3. Check the configuration using the command below to confirm the updates above are taken if needed: - -```bash -$ docker info -``` - -## 4. Install Docker Compose -- Comprehensive instructions can be found on this [website](https://www.digitalocean.com/community/tutorials/how-to-install-docker-compose-on-ubuntu-16-04) -- Follow steps 1 and 2. - -## 5. Install [`kafka-docker`](https://github.com/wurstmeister/kafka-docker) so kafka and zookeeper can run in a separate container. - -- Get your host IP address. The address is usually listed under an ethernet adapter, e.g., `en`. - -```bash -$ ifconfig -$ export DOCKER_HOST_IP= -$ export ASN1_CODEC_HOME=${GIT_REPOS}/asn1_codec -``` -- Get the kafka and zookeeper images. - -```bash -$ cd $GIT_REPOS -$ git clone https://github.com/wurstmeister/kafka-docker.git -$ cd kafka-docker -$ vim docker-compose.yml // Set karka: ports: to 9092:9092 -``` -- The `docker-compose.yml` file may need to be changed; the ports for kafka should be 9092:9092. -- Startup the kafka and zookeeper containers and make sure they are running. - -```bash -$ docker-compose up --no-recreate -d -$ docker-compose ps -``` -- **When you want to stop kafka and zookeeper, execute the following commands.** - -```bash -$ cd $GIT_REPOS/kafka-docker -$ docker-compose down -``` - -## 6. Download and install the Kafka **binary**. - -- The Kafka binary provides a producer and consumer tool that can act as surrogates for the ODE (among other items). -- [Kafka Binary](https://kafka.apache.org/downloads) -- [Kafka Quickstart](https://kafka.apache.org/quickstart) is a very useful reference. -- Move and unpack the Kafka code as follows: - -```bash -$ cd $GIT_REPOS -$ wget http://apache.claz.org/kafka/0.10.2.1/kafka_2.12-0.10.2.1.tgz // mirror and kafka version may change; check website. -$ tar -xzf kafka_2.12-0.10.2.1.tgz // the kafka version may be different. -$ mv kafka_2.12-0.10.2.1 kafka -``` - - - - diff --git a/docs/interface.md b/docs/interface.md index 7c5c466..6287713 100644 --- a/docs/interface.md +++ b/docs/interface.md @@ -23,14 +23,14 @@ - Currently, the SAE J2735 MessageFrame is encoded by the ODE using XER rules. UPER rules are used to encode these payload for transmission to the CV environment. -# Instructions to the ACM +## Instructions to the ACM - The ACM does not maintain any state with regard to a particular message; each message is handled based on the information provided in the XML input. - The ACM handles XML messages whose root node is named ``. - The root node has two child element nodes: `` and ``. - The `` node contains the data to encode or decode. - The `` node provides instructions to the ACM. - In the `` node, the child element `` specifies how encodings/decodings will be performed and in which order. - The following example should explain the heirarchy: + The following example should explain the hierarchy: ```xml @@ -54,7 +54,7 @@ - If the path is not found or another error occurs, an XML document will be returned that describes the error and where it occured. - `` will indicate the encoding/decoding rule to use. -# ASN.1 Specification Type Names +## ASN.1 Specification Type Names - [asn1c](https://github.com/vlm/asn1c) generates a `pdu_collection.c` file when you compile your ASN.1 specification files. It defines the various types of encodings that can be made when using the compiled ASN.1 specification files. This file also contains an array of pointers to these structures. The structures contain two useful fields for the purpose of discovering @@ -70,10 +70,9 @@ - It would be nice for the name of the mangled name of the structure to be included in the structure, possibly the name string, but this may create other unknown problems in asn1c. -# Produced Encodings and Decodings +## Produced Encodings and Decodings - When the ACM completes its task, it will produce the result as an XML document containing the same elements as it was sent with the following exceptions: - When a message is encoded, the XML will be encoded into binary, transformed into a hex string and written to the `` `` child element. - When a message is encoded, the `` `` element will be changed to: `us.dot.its.jpo.ode.model.OdeHexByteArray` - When a message is decoded, the hex string will be converted into binary, decoded, and the XER (XML) written to the `` `` child element. - When a message is decoded, the `` `` element will be changed to the name of the decoded data element, e.g., `MessageFrame` - diff --git a/docs/testing.md b/docs/testing.md index 6302aa8..bc1bd0e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,13 +1,38 @@ # Testing the ACM -There is currently two ways to test the capabilities of the ACM. +There are currently two ways to test the capabilities of the ACM. - [Unit Testing](#unit-testing) - [Standalone Operation / Testing](#standalone-testing) -## Test Files +## Unit Testing +### Testing On Local Machine +The following command can be run to build & run the unit tests: -Several example JSON message test files are in the [asn1_codec/data](../data) directory. These files can be edited to generate -your own test cases. +```bash +$ ./build_local.sh +``` + + +### Testing Using Docker +Start by building the Docker image: + +```bash +$ docker build -t acm . +``` + +Then run the Docker container: + +```bash +$ docker run -it --name acm acm +``` + +Exec into the container and run the tests: + +```bash +$ docker exec -it acm /bin/bash +$ cd /build +$ ./acm_tests +``` ## Standalone Operation / Testing @@ -19,10 +44,7 @@ newlines. The following is an example command: $ ./acm -F -c config/example.properties -T decode ../data/InputData.Ieee1609Dot2Data.packed.xml ``` -## Unit Testing - -Unit tests are built when the ACM is compiled during installation. Those tests can be run using the following command: +### Test Files -```bash -$ ./acm_tests -``` +Several example JSON message test files are in the [asn1_codec/data](../data) directory. These files can be edited to generate +your own test cases. \ No newline at end of file diff --git a/sample.env b/sample.env index adb4451..f071c7d 100644 --- a/sample.env +++ b/sample.env @@ -2,14 +2,14 @@ DOCKER_HOST_IP= # Whether or not to log to the console. -ACM_LOG_TO_CONSOLE= +ACM_LOG_TO_CONSOLE=true # Whether or not to log to a file. -ACM_LOG_TO_FILE= +ACM_LOG_TO_FILE=true # The log level to use. # Valid values are: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "OFF" -ACM_LOG_LEVEL= +ACM_LOG_LEVEL=INFO # If unset, a local kafka broker will be targeted. # If set to "CONFLUENT", the application will target a Confluent Cloud cluster. diff --git a/src/acm.cpp b/src/acm.cpp index 60f0cfe..73ed9f6 100644 --- a/src/acm.cpp +++ b/src/acm.cpp @@ -1810,6 +1810,9 @@ bool ASN1_Codec::filetest() { logger->info(output_msg_stream.str()); + // Send output directly to stdout. + std::cout << output_msg_stream.str() << std::endl; + } else { logger->trace("Read an empty file."); } diff --git a/src/tests.cpp b/src/tests.cpp index a325ebf..479c385 100644 --- a/src/tests.cpp +++ b/src/tests.cpp @@ -38,7 +38,7 @@ const char *ONE609_BSM_HEX = "038081B1001480AD562FA8400039E8E717090F9665FE1BACC3 const char *ASD_ONE609_HEX = "44400000000084782786283B90A7148D2B0A89C49F8A85A7763BF8423C13C2107E1C0C6F7E2C0C6F16A070103620029015AAC5F50800073D1CE2E121F2CCBFC375986FFFFFFFFE0007775FBF43F4200FFFF000000000004042983820082FFFFFFFD049C20147FFFFFFFD128420408FFFFFFFD2454204D480287FFD2BAC2084680BAFFFD4EAC2064580B33FFD5BF42072380D6BFFD6FCC2079681407FFDA424206778115BFFDB34C205D0811CBFFDBC5C2057B8110BFFDBE142001780337FFEFE643FFFF800BBFFF8AB43FFFFFFFFFFFFBA3420080FFFFFFFFC345FFFC00000"; /** - * After switching to using `mouse07410/asn1c` (a fork of `vlm/asn1c`), this unit test is failing. + * This unit test is currently failing with the fork of asn1c that we're using (currently `usdot-fhwa-stol/usdot-asn1c`). * Since encoding BSMs is not a desired feature of the ACM, this test case has been commented out. */ // TEST_CASE("Encode BSM", "[encoding]" ) { // TODO: fix test case failing @@ -74,7 +74,7 @@ TEST_CASE("Encode ASD", "[encoding]" ) { } /** - * After switching to using `mouse07410/asn1c` (a fork of `vlm/asn1c`), this unit test is failing. + * This unit test is currently failing with the fork of asn1c that we're using (currently `usdot-fhwa-stol/usdot-asn1c`). * Since encoding BSMs is not a desired feature of the ACM, this test case has been commented out. */ // TEST_CASE("Encode ASD_BSM", "[encoding]" ) { // TODO: fix test case failing @@ -132,7 +132,7 @@ TEST_CASE("Encode ASD_1609", "[encoding]" ) { } /** - * After switching to using `mouse07410/asn1c` (a fork of `vlm/asn1c`), this unit test is failing. + * This unit test is currently failing with the fork of asn1c that we're using (currently `usdot-fhwa-stol/usdot-asn1c`). * Since encoding BSMs is not a desired feature of the ACM, this test case has been commented out. */ // TEST_CASE("Encode 1609_BSM", "[encoding]") { // TODO: fix test case failing @@ -154,7 +154,7 @@ TEST_CASE("Encode ASD_1609", "[encoding]" ) { // } /** - * After switching to using `mouse07410/asn1c` (a fork of `vlm/asn1c`), this unit test is failing. + * This unit test is currently failing with the fork of asn1c that we're using (currently `usdot-fhwa-stol/usdot-asn1c`). * Since encoding BSMs is not a desired feature of the ACM, this test case has been commented out. */ // TEST_CASE("Encode ASD_1609_BSM", "[encoding]") { // TODO: fix test case failing diff --git a/asn1c b/usdot-asn1c similarity index 100% rename from asn1c rename to usdot-asn1c