Skip to content

Commit

Permalink
Merge pull request #492 from mtconnect/484_json_ingest_error_recovery
Browse files Browse the repository at this point in the history
Added error handling for json ingest
  • Loading branch information
wsobel authored Oct 3, 2024
2 parents 89517ca + d439e64 commit 04be9db
Show file tree
Hide file tree
Showing 2 changed files with 177 additions and 5 deletions.
9 changes: 4 additions & 5 deletions src/mtconnect/pipeline/json_mapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -887,11 +887,10 @@ namespace mtconnect::pipeline {
}
else
{
// Otherwise we have a problem, we cannot consume content
// if we don't know the data item since we cannot handle tables
// and data sets.
LOG(warning) << "Cannot map properties without data item";
return false;
LOG(warning) << "Cannot map properties without data item, consuming erronous data";
ErrorHandler handler;
if (!handler(reader, buff))
return false;
}
m_props.clear();
m_dataItem.reset();
Expand Down
173 changes: 173 additions & 0 deletions test_package/json_mapping_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,15 @@ class JsonMappingTest : public testing::Test
Properties ps(props);
ErrorList errors;
auto di = DataItem::make(ps, errors);
if (errors.size() > 0)
{
cerr << "Errors occurred during make data item" << endl;
for (auto &e : errors)
{
cerr << " " << e->getEntity() << ": " << e->what() << endl;
}
return nullptr;
}
m_dataItems.emplace(di->getId(), di);

dev->second->addDataItem(di, errors);
Expand Down Expand Up @@ -1017,6 +1026,7 @@ TEST_F(JsonMappingTest, should_skip_erroneous_values)
Properties props {{"VALUE", R"(
{
"timestamp": "2023-11-09T11:20:00Z",
"c": "BAD_ITEM",
"a": {
"r1": {
"k1": 123.45
Expand Down Expand Up @@ -1047,5 +1057,168 @@ TEST_F(JsonMappingTest, should_skip_erroneous_values)
ASSERT_EQ("MANUAL", obs->getValue<string>());
}

/// @test if observation is incorrect, skip levels
TEST_F(JsonMappingTest, should_skip_erroneous_array)
{
auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}});
makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}});
makeDataItem("device", {{"id", "b"s}, {"type", "CONTROLLER_MODE"s}, {"category", "EVENT"s}});

Properties props {{"VALUE", R"(
{
"timestamp": "2023-11-09T11:20:00Z",
"c": [1.0, 2.0, 3.0, 4.0, 5.0],
"a": {
"r1": {
"k1": 123.45
},
"r2": {
"k2": "ABCDEF",
"k3": 6789
}
},
"b": "MANUAL"
})"s}};

auto jmsg = std::make_shared<JsonMessage>("JsonMessage", props);
jmsg->m_device = dev;

auto res = (*m_mapper)(std::move(jmsg));
ASSERT_TRUE(res);

auto value = res->getValue();
ASSERT_TRUE(std::holds_alternative<EntityList>(value));
auto list = get<EntityList>(value);
ASSERT_EQ(1, list.size());

auto obs = dynamic_pointer_cast<Observation>(list.front());
ASSERT_TRUE(obs);
ASSERT_EQ("ControllerMode", obs->getName());
ASSERT_EQ("b", obs->getDataItem()->getId());
ASSERT_EQ("MANUAL", obs->getValue<string>());
}

/// @test if observation is incorrect, skip levels
TEST_F(JsonMappingTest, should_skip_erroneous_table)
{
auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}});
makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}});
makeDataItem("device", {{"id", "b"s}, {"type", "CONTROLLER_MODE"s}, {"category", "EVENT"s}});

Properties props {{"VALUE", R"(
{
"timestamp": "2023-11-09T11:20:00Z",
"c": {
"r1": {
"k1": 123.45
},
"r2": {
"k2": "ABCDEF",
"k3": 6789
}
},
"a": {
"r1": {
"k1": 123.45
},
"r2": {
"k2": "ABCDEF",
"k3": 6789
}
},
"b": "MANUAL"
})"s}};

auto jmsg = std::make_shared<JsonMessage>("JsonMessage", props);
jmsg->m_device = dev;

auto res = (*m_mapper)(std::move(jmsg));
ASSERT_TRUE(res);

auto value = res->getValue();
ASSERT_TRUE(std::holds_alternative<EntityList>(value));
auto list = get<EntityList>(value);
ASSERT_EQ(1, list.size());

auto obs = dynamic_pointer_cast<Observation>(list.front());
ASSERT_TRUE(obs);
ASSERT_EQ("ControllerMode", obs->getName());
ASSERT_EQ("b", obs->getDataItem()->getId());
ASSERT_EQ("MANUAL", obs->getValue<string>());
}

// if the tag "blah" is not found in the device file, none of the other tags get processed
TEST_F(JsonMappingTest, should_ignore_Invalid_Tag_DataItem)
{
auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}});
makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}});
makeDataItem("device", {{"id", "c"s}, {"type", "CONTROLLER_MODE"s}, {"category", "EVENT"s}});

Properties props {{"VALUE", R"(
{
"timestamp": "2023-11-09T11:20:00Z",
"c": "BLAH",
"a": {
"r1": {
"k1": 123.45
},
"r2": {
"k2": "ABCDEF",
"k3": 6789
}
},
"b": "MANUAL"
})"s}};

auto jmsg = std::make_shared<JsonMessage>("JsonMessage", props);
jmsg->m_device = dev;

auto res = (*m_mapper)(std::move(jmsg));
ASSERT_TRUE(res);

auto value = res->getValue();
ASSERT_TRUE(std::holds_alternative<EntityList>(value));
auto list = get<EntityList>(value);
ASSERT_EQ(1, list.size());

auto obs = dynamic_pointer_cast<Observation>(list.front());
ASSERT_TRUE(obs);
ASSERT_EQ("ControllerMode", obs->getName());
ASSERT_EQ("c", obs->getDataItem()->getId());
ASSERT_EQ("BLAH", obs->getValue<string>());
}

TEST_F(JsonMappingTest, should_ignore_Invalid_make_DataItem)
{
auto dev = makeDevice("Device", {{"id", "device"s}, {"name", "device"s}, {"uuid", "device"s}});
makeDataItem("device", {{"id", "a"s}, {"type", "EXECUTION"s}, {"category", "EVENT"s}});
makeDataItem("device", {{"id", "b"s}, {"blah", "BLAH"s}, {"category", "EVENT"s}});

Properties props {{"VALUE", R"(
{
"timestamp": "2023-11-09T11:20:00Z",
"c": "Blah",
"a": "ACTIVE",
"b": "MANUAL"
})"s}};

auto jmsg = std::make_shared<JsonMessage>("JsonMessage", props);
jmsg->m_device = dev;

auto res = (*m_mapper)(std::move(jmsg));
ASSERT_TRUE(res);

auto value = res->getValue();
ASSERT_TRUE(std::holds_alternative<EntityList>(value));
auto list = get<EntityList>(value);
ASSERT_EQ(1, list.size());

auto obs = dynamic_pointer_cast<Observation>(list.front());
ASSERT_TRUE(obs);
ASSERT_EQ("Execution", obs->getName());
ASSERT_EQ("a", obs->getDataItem()->getId());
ASSERT_EQ("ACTIVE", obs->getValue<string>());
}

/// @test verify the json mapper can an asset in json
TEST_F(JsonMappingTest, should_parse_json_asset) { GTEST_SKIP(); }

0 comments on commit 04be9db

Please sign in to comment.