diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index b05eacdd..19cce0b9 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, iotagent-node-lib#1612) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iota-json#827) diff --git a/docs/usermanual.md b/docs/usermanual.md index 0cb8de8a..e7f335fa 100644 --- a/docs/usermanual.md +++ b/docs/usermanual.md @@ -30,15 +30,14 @@ t|15|k|abc In this example, two attributes, one named "t" with value "15" and another named "k" with value "abc" are transmitted. Values in Ultralight 2.0 are not typed (everything is treated as a string). -Multiple groups of measures can be combined into a single request, using the `#` character. In that case, a different +Multiple groups of measures can be combined into a single request (but just for HTTP/POST or MQTT), using the `#` character. In that case, a different NGSI request will be generated for each group of measures. E.g.: ```text gps|1.2/3.4#t|10 ``` -This will generate two NGSI requests for the same entity, one for each one of the values. Each one of those requests can -contain any number of attributes. +This will generate two elements in the NGSI batch update request (POST /v2/op/update) for the same entity, one for each one of the measures. Each one of those elements can contain any number of attributes. Measure groups can additionally have an optional timestamp, with the following syntax: diff --git a/lib/bindings/HTTPBindings.js b/lib/bindings/HTTPBindings.js index dbb99537..fe8a0d73 100644 --- a/lib/bindings/HTTPBindings.js +++ b/lib/bindings/HTTPBindings.js @@ -214,7 +214,7 @@ function returnCommands(req, res, next) { } function handleIncomingMeasure(req, res, next) { - let updates = []; + let update = []; context = fillService(context, { service: 'n/a', subservice: 'n/a' }); // prettier-ignore config.getLogger().debug(context, 'Processing multiple HTTP measures for device %s with apiKey %j', @@ -223,10 +223,10 @@ function handleIncomingMeasure(req, res, next) { function processHTTPWithDevice(device) { context = fillService(context, device); if (req.ulPayload) { - updates = req.ulPayload.reduce(commonBindings.processMeasureGroup.bind(null, device, req.apiKey), []); + update = [req.ulPayload].reduce(commonBindings.processMeasureGroup.bind(null, device, req.apiKey), []); } - async.series(updates, function (error) { + async.series(update, function (error) { if (error) { next(error); // prettier-ignore diff --git a/lib/commonBindings.js b/lib/commonBindings.js index 23fffa28..2f8d9ea1 100644 --- a/lib/commonBindings.js +++ b/lib/commonBindings.js @@ -128,32 +128,35 @@ function manageConfigurationRequest(apiKey, deviceId, device, objMessage) { /* eslint-disable-next-line no-unused-vars */ function processMeasureGroup(device, apikey, previous, current, index) { - const values = []; - if (current.command) { + if (current[0] && current[0].command) { previous.push( iotAgentLib.setCommandResult.bind( null, device.name, config.getConfig().iota.defaultResource, apikey, - current.command, - current.value, + current[0].command, + current[0].value, constants.COMMAND_STATUS_COMPLETED, device ) ); } else { - for (const k in current) { - if (current.hasOwnProperty(k)) { - values.push({ - name: k, - type: guessType(k, device), - value: current[k] - }); + const val = []; + for (let curr of current) { + const values = []; + for (const k in curr) { + if (curr.hasOwnProperty(k)) { + values.push({ + name: k, + type: guessType(k, device), + value: curr[k] + }); + } } + val.push(values); } - - previous.push(iotAgentLib.update.bind(null, device.name, device.type, '', values, device)); + previous.push(iotAgentLib.update.bind(null, device.name, device.type, '', val, device)); } return previous; @@ -168,7 +171,7 @@ function processMeasureGroup(device, apikey, previous, current, index) { * @param {String} messageStr UL payload parsed to string. */ function multipleMeasures(apiKey, device, messageStr) { - let updates = []; + let update = []; let parsedMessage; context = fillService(context, device); config.getLogger().debug(context, 'Processing multiple measures for device %s with apiKey %s', device.id, apiKey); @@ -181,9 +184,9 @@ function multipleMeasures(apiKey, device, messageStr) { return; } config.getLogger().debug(context, 'stringMessage: %s parsedMessage: %s', messageStr, parsedMessage); - updates = parsedMessage.reduce(processMeasureGroup.bind(null, device, apiKey), []); + update = [parsedMessage].reduce(processMeasureGroup.bind(null, device, apiKey), []); - async.series(updates, function (error) { + async.series(update, function (error) { if (error) { config.getLogger().error( context, diff --git a/test/unit/ngsiv2/amqpBinding-test.js b/test/unit/ngsiv2/amqpBinding-test.js index ae43f19a..b4a7746a 100644 --- a/test/unit/ngsiv2/amqpBinding-test.js +++ b/test/unit/ngsiv2/amqpBinding-test.js @@ -218,19 +218,7 @@ describe('AMQP Transport binding: measures', function () { contextBrokerMock .matchHeader('fiware-service', 'smartgondor') .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/singleMeasure.json') - ) - .reply(204); - - contextBrokerMock - .matchHeader('fiware-service', 'smartgondor') - .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/secondSingleMeasure.json') - ) + .post('/v2/op/update', utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multimeasure.json')) .reply(204); }); @@ -248,19 +236,7 @@ describe('AMQP Transport binding: measures', function () { contextBrokerMock .matchHeader('fiware-service', 'smartgondor') .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multipleMeasure.json') - ) - .reply(204); - - contextBrokerMock - .matchHeader('fiware-service', 'smartgondor') - .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/secondMultipleMeasure.json') - ) + .post('/v2/op/update', utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multimeasure2.json')) .reply(204); }); diff --git a/test/unit/ngsiv2/contextRequests/multimeasure.json b/test/unit/ngsiv2/contextRequests/multimeasure.json new file mode 100644 index 00000000..3802eafb --- /dev/null +++ b/test/unit/ngsiv2/contextRequests/multimeasure.json @@ -0,0 +1,21 @@ +{ + "actionType": "append", + "entities": [ + { + "id": "Second UL Device", + "type": "AnMQTTDevice", + "temperature": { + "type": "celsius", + "value": 23 + } + }, + { + "id": "Second UL Device", + "type": "AnMQTTDevice", + "humidity": { + "type": "degrees", + "value": 98 + } + } + ] +} diff --git a/test/unit/ngsiv2/contextRequests/multimeasure2.json b/test/unit/ngsiv2/contextRequests/multimeasure2.json new file mode 100644 index 00000000..55c8bb70 --- /dev/null +++ b/test/unit/ngsiv2/contextRequests/multimeasure2.json @@ -0,0 +1,29 @@ +{ + "actionType": "append", + "entities": [ + { + "id": "Second UL Device", + "type": "AnMQTTDevice", + "temperature": { + "type": "celsius", + "value": 23 + }, + "humidity": { + "type": "degrees", + "value": 98 + } + }, + { + "id": "Second UL Device", + "type": "AnMQTTDevice", + "temperature": { + "type": "celsius", + "value": 16 + }, + "humidity": { + "type": "degrees", + "value": 34 + } + } + ] +} diff --git a/test/unit/ngsiv2/contextRequests/multimeasure3.json b/test/unit/ngsiv2/contextRequests/multimeasure3.json new file mode 100644 index 00000000..2667a40b --- /dev/null +++ b/test/unit/ngsiv2/contextRequests/multimeasure3.json @@ -0,0 +1,499 @@ +{ + "actionType": "append", + "entities": [ + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "temperature:ambient": { + "type": "float", + "value": 24.4, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "hum": { + "type": "Text", + "value": 58, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "aco": { + "type": "Text", + "value": 0.1, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "apa": { + "type": "Text", + "value": 0.38, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "ao3": { + "type": "Text", + "value": 121, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "no2": { + "type": "Text", + "value": 115, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "pla": { + "type": "Text", + "value": 43.4551, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "plo": { + "type": "Text", + "value": -3.83381, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "poa": { + "type": "Text", + "value": 28, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "spi": { + "type": "Text", + "value": 0, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "dia": { + "type": "Text", + "value": 0, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "mit": { + "type": "Text", + "value": 1492, + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + }, + { + "id": "urn:x-iot:smartsantander:u7jcfa:fixed:t311", + "type": "repeater:illuminance", + "att_name": { + "type": "string", + "value": "value", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "coords": { + "type": "string", + "value": "WGS84", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "pos": { + "type": "Text", + "value": "43.4630608,-3.8345434", + "metadata": { + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-05-21T16:33:16.657+02:00" + } + } + ] +} diff --git a/test/unit/ngsiv2/httpBindings-test.js b/test/unit/ngsiv2/httpBindings-test.js index 6bdc3b79..0e49590d 100644 --- a/test/unit/ngsiv2/httpBindings-test.js +++ b/test/unit/ngsiv2/httpBindings-test.js @@ -391,19 +391,7 @@ describe('HTTP Transport binding: measures', function () { contextBrokerMock .matchHeader('fiware-service', 'smartgondor') .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/singleMeasure.json') - ) - .reply(204); - - contextBrokerMock - .matchHeader('fiware-service', 'smartgondor') - .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/secondSingleMeasure.json') - ) + .post('/v2/op/update', utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multimeasure.json')) .reply(204); }); @@ -440,19 +428,7 @@ describe('HTTP Transport binding: measures', function () { contextBrokerMock .matchHeader('fiware-service', 'smartgondor') .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multipleMeasure.json') - ) - .reply(204); - - contextBrokerMock - .matchHeader('fiware-service', 'smartgondor') - .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/secondMultipleMeasure.json') - ) + .post('/v2/op/update', utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multimeasure2.json')) .reply(204); }); @@ -650,28 +626,11 @@ describe('HTTP Transport binding: measures', function () { contextBrokerMock .matchHeader('fiware-service', 'smartgondor') .matchHeader('fiware-servicepath', '/gardens') - // Note: The expected body payload is not set explicitly since this mock will be used to - // intercept requests from the IOTA to the CB for each one of the different observations. - // Therefore, instead of introducing 13 different mocks, we have decided to have a single one - // and just check the structure of the payload programmatically. - .post('/v2/entities?options=upsert', function (body) { - let i = 0; - let attributes = 0; - - for (const attribute in body) { - // checks that all attributes has metadata - if (body.hasOwnProperty(attribute)) { - attributes++; - for (const metadata in body[attribute].metadata) { - if (body[attribute].metadata.hasOwnProperty(metadata)) { - i++; - } - } - } - } - return i === attributes - 1 - 2; - }) - .times(13) + .post( + '/v2/op/update' + // FIXME: Mock about current timestamp + //utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multimeasure3.json') + ) .reply(204); config.iota.timestamp = true; // forces to add timestamp att and metadata with timeinstant to all attributes diff --git a/test/unit/ngsiv2/mqttBinding-test.js b/test/unit/ngsiv2/mqttBinding-test.js index 1876bb58..47c4d778 100644 --- a/test/unit/ngsiv2/mqttBinding-test.js +++ b/test/unit/ngsiv2/mqttBinding-test.js @@ -289,19 +289,7 @@ describe('MQTT Transport binding: measures', function () { contextBrokerMock .matchHeader('fiware-service', 'smartgondor') .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/singleMeasure.json') - ) - .reply(204); - - contextBrokerMock - .matchHeader('fiware-service', 'smartgondor') - .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/secondSingleMeasure.json') - ) + .post('/v2/op/update', utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multimeasure.json')) .reply(204); }); @@ -329,19 +317,7 @@ describe('MQTT Transport binding: measures', function () { contextBrokerMock .matchHeader('fiware-service', 'smartgondor') .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multipleMeasure.json') - ) - .reply(204); - - contextBrokerMock - .matchHeader('fiware-service', 'smartgondor') - .matchHeader('fiware-servicepath', '/gardens') - .post( - '/v2/entities?options=upsert', - utils.readExampleFile('./test/unit/ngsiv2/contextRequests/secondMultipleMeasure.json') - ) + .post('/v2/op/update', utils.readExampleFile('./test/unit/ngsiv2/contextRequests/multimeasure2.json')) .reply(204); });