From 058bb40e351468e455befb872b026ca2d8c23a4b Mon Sep 17 00:00:00 2001 From: Fernando Lopez Date: Sun, 6 Oct 2019 17:33:52 +0200 Subject: [PATCH 1/6] Update IoTA with the last changes of the IoTAgent Node Lib --- .jshintrc | 1 + CHANGES_NEXT_RELEASE | 6 +- bin/lwm2mAgent.js | 10 +- config.js | 6 + docker/README.md | 40 ++++++ docs/administrationGuide.md | 231 ++++++++++++++++++++++++------ docs/configurationProvisioning.md | 2 +- docs/deprecated.md | 2 +- docs/deviceProvisioning.md | 8 +- docs/index.md | 2 +- docs/limits.conf | 33 +++++ docs/roadmap.md | 2 +- docs/staticConfiguration.md | 2 +- docs/sysctl.conf | 184 ++++++++++++++++++++++++ docs/userGuide.md | 48 +++---- package.json | 2 +- test/.jshintrc | 1 + 17 files changed, 499 insertions(+), 81 deletions(-) create mode 100644 docs/limits.conf create mode 100644 docs/sysctl.conf diff --git a/.jshintrc b/.jshintrc index 851a48a..6dd58ca 100644 --- a/.jshintrc +++ b/.jshintrc @@ -9,6 +9,7 @@ "quotmark": "single", "undef": true, "unused": true, + "esversion": 6, "trailing": true, "maxparams": 8, "maxdepth": 4, diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index c265b6c..719a716 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,7 @@ - Set Nodejs 8 as minimum version in packages.json (effectively removing Nodev6 from supported versions) - ADD PM2_ENABLED flag to Docker -- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues \ No newline at end of file +- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues +- Configure multithreading using cluster nodejs module (through multiCore config.js parameter or IOTA_MULTI_CORE +environment variable) +- Correct typos in documentation files +- Add lwm2m agent version in the description of /iot/about request diff --git a/bin/lwm2mAgent.js b/bin/lwm2mAgent.js index 55f28e8..caeac0f 100755 --- a/bin/lwm2mAgent.js +++ b/bin/lwm2mAgent.js @@ -24,14 +24,16 @@ */ 'use strict'; -var iotAgent = require('../lib/iotAgentLwm2m'), +let iotAgent = require('../lib/iotAgentLwm2m'), + iotAgentLib = require('iotagent-node-lib'), + info = require('../package.json'), context = { op: 'IOTAgent.Executable' }, logger = require('logops'); function start() { - var config; + let config; if (process.argv.length === 3) { config = require('../' + process.argv[2]); @@ -39,7 +41,9 @@ function start() { config = require('../config'); } - iotAgent.start(config, function(error) { + config.ngsi.iotaVersion = info.version; + + iotAgentLib.startServer(config, iotAgent, function(error) { if (error) { logger.error(context, 'Error starting Agent: [%s] Exiting process', error); } else { diff --git a/config.js b/config.js index 7fc09d6..04f5bf2 100644 --- a/config.js +++ b/config.js @@ -148,4 +148,10 @@ config.ngsi = { //clientSecret: 'IOTA_AUTH_CLIENT_SECRET' //}; +/** + * flag indicating whether the node server will be executed in multi-core option (true) or it will be a + * single-thread one (false). + */ +config.multiCore = true; + module.exports = config; diff --git a/docker/README.md b/docker/README.md index 5fdc984..3c2ec13 100644 --- a/docker/README.md +++ b/docker/README.md @@ -179,3 +179,43 @@ Currently, this `_FILE` suffix is supported for: - `IOTA_AUTH_PASSWORD` - `IOTA_AUTH_CLIENT_ID` - `IOTA_AUTH_CLIENT_SECRET` + +## Best Practices + +### Increase ULIMIT in Docker Production Deployments + +Default settings for ulimit on a Linux system assume that several users would share the system. These settings limit the +number of resources used by each user. The default settings are generally very low for high performance servers and +should be increased. By default, we recommend, that the Lightweight OMA M2M IoTAgent server in high performance +scenarios, the following changes to ulimits: + +```console +ulimit -n 65535 # nofile: The maximum number of open file descriptors (most systems do not allow this + value to be set) +ulimit -c unlimited # core: The maximum size of core files created +ulimit -l unlimited # memlock: The maximum size that may be locked into memory +``` + +If you are just doing light testing and development, you can omit these settings, and everything will still work. + +To set the ulimits in your container, you will need to run Lightweight OMA M2M IoTAgent Docker containers with the +following additional --ulimit flags: + +```console +docker run --ulimit nofile=65535:65535 --ulimit core=100000000:100000000 --ulimit memlock=100000000:100000000 \ +--name iotagent -d fiware/lightweightm2m-iotagent +``` + +Since “unlimited” is not supported as a value, it sets the core and memlock values to 100 GB. If your system has more +than 100 GB RAM, you will want to increase this value to match the available RAM on the system. + +> Note: The --ulimit flags only work on Docker 1.6 or later. + +Nevertheless, you have to "request" more resources (i.e. multiple cores), which might be more difficult for orchestrates +([Docker Engine](https://docs.docker.com/engine) or [Kubernetes](https://kubernetes.io)) to schedule than a few +different containers requesting one core (or less...) each (which it can, in turn, schedule on multiple nodes, and not +necessarily look for one node with enough available cores). + +If you want to get more details about the configuration of the system and node.js for high performance scenarios, please +refer to the +[Administration Guide](https://fiware-iotagent-lwm2m.readthedocs.io/en/latest/administrationGuide/index.html). diff --git a/docs/administrationGuide.md b/docs/administrationGuide.md index d71f436..a9bde37 100644 --- a/docs/administrationGuide.md +++ b/docs/administrationGuide.md @@ -7,8 +7,9 @@ - [Packaging](#packaging) - [Sanity checks](#sanity-checks) - [Diagnosis procedures](#diagnosis-procedures) +- [High performance configuration usage](#high-performance-configuration-usage) -# Prerequisites +## Prerequisites The IoT Agent requires Node.js 0.10.x to work and uses npm as its package manager. Most Linux distributions offer packages to install it. For other OS, you can find instructions to install Node.js [here](https://nodejs.org/). @@ -16,9 +17,9 @@ packages to install it. For other OS, you can find instructions to install Node. NOTE: the current version of Node.js, 0.12.x has not been tested with the Agent, so we suggest to download and use the previous version (that process can be eased with utilities as `n` or `nvm`). -# Installation +## Installation -## Cloning the GitHub repository +### Cloning the GitHub repository Once the repository is cloned, from the root folder of the project execute: @@ -28,9 +29,9 @@ npm install This will download the dependencies for the project, and let it ready to the execution. -## Using the RPM +### Using the RPM -To see how to generate the RPM, follow the instructions in [Packaging](#rpm). +To see how to generate the RPM, follow the instructions in [Packaging](#packaging). To install the RPM, use the YUM tool: @@ -38,7 +39,7 @@ To install the RPM, use the YUM tool: yum localinstall --nogpg ``` -## Using Docker +### Using Docker There are automatic builds of the development version of the IoT Agent published in Docker hub. In order to install using the docker version, just execute the following: @@ -50,9 +51,9 @@ docker run --link orion:orion telefonicaiot/lightweightm2m-iotagent As you can see, the Lightweight M2M (as any other IoT Agent) requires a Context Broker to work. In order to link it, just use the option `--link` as shown in the example. -# Usage +## Usage -## GitHub installation +### GitHub installation In order to execute the IoT Agent, just issue the following command from the root folder of the cloned project: @@ -62,7 +63,7 @@ bin/lwm2mAgent.js [config file] The optional name of a config file is optional and described in the following section. -## RPM installation +### RPM installation The RPM installs a linux service that can be managed with the typical instructions: @@ -76,7 +77,7 @@ service iotagent-lwm2m stop In this mode, the log file is written in `/var/log/iotagent-lwm2m/iotagent-lwm2m.log`. -# Configuration +## Configuration There are two ways to provide the IoT Agent with a configuration set: passing the name of a config file (related to the root folder of the project) or customise the example `config.js` in the root. @@ -102,22 +103,22 @@ These are the specific LWM2M parameters that can be configured for the agent: - **types**: for IoT Agents with multiple southbound paths, this attribute maps attribute types (defined either in the configuration file or by using the Device Configuration API) to southbound interfaces. E.g.: -```json - { - name: 'Light', - url: '/light' - }, - { - name: 'Pressure', - url: '/pres' - }, - { - name: 'Arduino', - url: '/arduino' - } -``` - -# Packaging + ```javascript + { + name: 'Light', + url: '/light' + }, + { + name: 'Pressure', + url: '/pres' + }, + { + name: 'Arduino', + url: '/arduino' + } + ``` + +## Packaging The only package type allowed is RPM. In order to execute the packaging scripts, the RPM Build Tools must be available in the system. @@ -132,13 +133,13 @@ cd rpm Where `` is the version (x.y.z) you want the package to have and `` is an increasing number dependent on previous installations. -# Sanity checks +## Sanity checks The Sanity Check Procedures are the steps that a System Administrator will take to verify that an installation is ready to be tested. This is therefore a preliminary set of tests to ensure that obvious or basic malfunctioning is fixed before proceeding to unit tests, integration tests and user validation -## Checking the administrative interface is up +### Checking the administrative interface is up The first procedure that can be executed to check if the IoTAgent is running is to get the version from the administrative interface. A curl command can be used to do so: @@ -162,9 +163,9 @@ Connection: keep-alive {"version":"0.8.1","port":4041,"baseRoot":"/"} ``` -## List of running processes +### List of running processes -The Agent runs a single node.js process, executing the `bin/lwm2mAgent.js` script. Here is an example of the runnig +The Agent runs a single node.js process, executing the `bin/lwm2mAgent.js` script. Here is an example of the running process. ```text @@ -174,17 +175,17 @@ root 732 31786 8 10:14 pts/0 00:00:00 node bin/lwm2mAgent.js If the process is running as a service, the PID number can be found in the `/opt/iotagent-lwm2m/iotagent-lwm2m.pid` file. -## Databases +### Databases The Lightweight M2M can work with in-memory databases (for testing purposes) or with MongoDB, depending on the selected configuration. The host, port and database name are configured in the config file as well. Check the [Configuration](configurationProvisioning.md) section for details. Be aware that the North Port and the South Port interactions both make use of the DB, and that their configurations can differ. -## Network Interfaces Up & Open +### Network Interfaces Up & Open Using `netstat` in a Linux machine, the ports can be checked up. The IoT Agent should be listening in two ports: the -administration and provisioning port (tipically TCP 4041) and the Lightweight M2M port (typically 60001). +administration and provisioning port (typically TCP 4041) and the Lightweight M2M port (typically 60001). The administrative port can be checked with the following command: @@ -214,7 +215,7 @@ udp 0 0 0.0.0.0:60001 0.0.0.0:* where the `` corresponds to the PID of the IoT Agent. -## Checking the service is up +### Checking the service is up A quick way to check if the service is working is to use the `status` command of the service. Execute: @@ -226,42 +227,186 @@ This will tell you if the SO thinks the IoT Agent Service is up. Be aware that t checking, that doesn't check the actual working Agent, just the existence of the process. Querying the administrative interface is always a stronger check. -## End-to-end testing +### End-to-end testing In order to make a simple LWM2M Check, a LWM2M client should be installed. The best approach is to install the client of the same library the IoT Agent uses, the [Node.js LWM2M Library](https://github.com/telefonicaid/lwm2m-node-lib). This library contains a simple command-line Lightweight M2M Client that can be used to test simple scenarios. For examples on how to perform these kind of tests, see the How-To's in the -[Getting Started section of the User Manual](userGuide.md#gettingstarted) that shows simple registrations and send +[Getting Started section of the User Manual](userGuide.md#getting-started) that shows simple registrations and send measure tests. -# Diagnosis procedures +## Diagnosis procedures Whenever a problem is risen in the IoT Agent, or if the Sanity Checks fail, the administrator should look at the log -files in order to check what kind of problema has happened. If the IoT Agent has been deployed using the RPMs, logs will +files in order to check what kind of problem has happened. If the IoT Agent has been deployed using the RPMs, logs will be located in the `/var/log/iotagent-lwm2m` folder. If the IoT Agent has been started from the command line, logs are written to the stdout. Whenever you are trying to debug a log, change first the log level as explained in the configuration section. Be aware that the North Port and South Port interactions of the IoT Agent have different log levels, that can be set independently. -## Resource availability +### Resource availability To be filled soon by DevOps. -## Remote Service Access +### Remote Service Access Check the [User Manual](userGuide.md) for more information on the exposed APIs. -## Resource consumption +### Resource consumption To be filled soon by DevOps. -## I/O Flows +### I/O Flows The Lightweight M2M IoT Agent follows the standard I/O flows for agents shown in the Node.js IoT Agent library. The flow is reproduced here, for clarity: -![General](https://raw.githubusercontent.com/telefonicaid/iotagent-node-lib/master/img/ngsiInteractions.png "NGSI Interactions") +![General](https://raw.githubusercontent.com/telefonicaid/iotagent-node-lib/master/doc/img/ngsiInteractions.png "NGSI Interactions") The interaction in the South Bound follows the flows defined by the [OMA Lightweight M2M Specification](http://www.omaspecworks.org/about-oma/work-program/m2m-enablers/). + +## High performance configuration usage + +Node.js is single‑threaded and uses non-blocking I/O, allowing it to scale up to tens of thousands of concurrent +operations. Nevertheless, Node.js has a few weak points and vulnerabilities that can make Node.js‑based systems to offer +under-performance behaviour, specially when a Node.js web application experiences rapid traffic growth. + +Additionally, It is important to know the place in which the node.js server is running, because it has limitations. +There are two types of limits on the host: hardware and software. Hardware limits can be easy to spot. Your application +might be consuming all of the memory and needing to consume disk to continue working. Adding more memory by upgrading +your host, whether physical or virtual, seems to be the right choice. + +Moreover, Node.js applications have also a software memory limit (imposed by V8), therefore we cannot forget about these +limitations when we execute a service. In this case of 64-bit environment, your application would be running by default +at a 1 GB V8 limit. If your application is running in high traffic scenarios, you will need a higher limit. The same is +applied to other parameters. + +It means that we need to make some changes in the execution of node.js and in the configuration of the system: + +- **Node.js flags** + + - **--use-idle-notification** + + Turns of the use idle notification to reduce memory footprint. + + - **--expose-gc** + + Use the expose-gc command to enable manual control of the garbage collector from the own node.js server code. In + case of the IoTAgent, it is not implemented because it is needed to implement the calls to the garbage collector + inside the ser server, nevertheless the recommended value is every 30 seconds. + + - **--max-old-space-size=xxxx** + + In that case, we want to increase the limit for heap memory of each V8 node process in order to use max capacity + that it is possible instead of the 1,4Gb default on 64-bit machines (512Mb on a 32-bit machine). The + recommendation is at least to use half of the total memory of the physical or virtual instance. + +- **User software limits** + + Linux kernel provides some configuration about system related limits and maximums. In a distributed environment with + multiple users, usually you need to take into control the resources that are available for each of the users. + Nevertheless, when the case is that you have only one available user but this one request a lot of resources due to + a high performance application the default limits are not proper configured and need to be changed to resolve the + high performance requirements. These are like maximum file handler count, maximum file locks, maximum process count + etc. + + You can see the limits of your system executing the command: + + ```bash + ulimit -a + ``` + + You can define the corresponding limits inside the file limits.conf. This description of the configuration file + syntax applies to the `/etc/security/limits.conf` file and \*.conf files in the `/etc/security/limits.d` directory. + You can get more information about the limits.conf in the + [limits.con - linux man pages](http://man7.org/linux/man-pages/man5/limits.conf.5.html). The recommended values to + be changes are the following: + + - **core** + + Limits of the core file size in KB, we recommend to change to `unlimited` both hard and soft types. + + * soft core unlimited + * hard core unlimited + + - **data** + + Maximum data size in KB, we recommend to change to `unlimited` both hard and soft types. + + * soft data unlimited + * hard data unlimited + + - **fsize** + + Maximum filesize in KB, we recommend to change to `unlimited` both hard and soft types. + + * soft fsize unlimited + * hard fsize unlimited + + - **memlock** + + Maximum locked-in-memory address space in KB, we recommend to change to `unlimited` both hard and soft types. + + * memlock unlimited + * memlock unlimited + + - **nofile** + + Maximum number of open file descriptors, we recommend to change to `65535` both hard and soft types. + + * soft nofile 65535 + * hard nofile 65535 + + - **rss** + + Maximum resident set size in KB (ignored in Linux 2.4.30 and higher), we recommend to change to `unlimited` both + hard and soft types. + + * soft rss unlimited + * hard rss unlimited + + - **stack** + + Maximum stack size in KB, we recommend to change to `unlimited` both hard and soft types. + + * soft stack unlimited + * hard stack unlimited + + - **nproc** + + Maximum number of processes, we recommend to change to `unlimited` both hard and soft types. + + * soft nproc unlimited + * hard nproc unlimited + + You can take a look to the [limits.conf](limits.conf) file provided in this folder with all the values provided. + +- **Configure kernel parameters** + + sysctl is used to modify kernel parameters at runtime. We plan to modify the corresponding `/etc/sysctl.conf` file. + You can get more information in the corresponding man pages of + [sysctl](http://man7.org/linux/man-pages/man8/sysctl.8.html) and + [sysctl.conf](http://man7.org/linux/man-pages/man5/sysctl.conf.5.html). You can search all the kernel parameters by + using the command `sysctl -a` + + - **fs.file-max** + + The maximum file handles that can be allocated, the recommended value is `1000000`. + + fs.file-max = 1000000 + + - **fs.nr_open** + + Max amount of file handles that can be opened, the recommended value is `1000000`. + + fs.nr_open = 1000000 + + - **net.netfilter.nf_conntrack_max** + + Size of connection tracking table. Default value is nf_conntrack_buckets value \* 4. + + net.nf_conntrack_max = 1048576 + + For more details about any other kernel parameters, take a look to the example [sysctl.conf](sysctl.conf) file. diff --git a/docs/configurationProvisioning.md b/docs/configurationProvisioning.md index b8e012f..271bb35 100644 --- a/docs/configurationProvisioning.md +++ b/docs/configurationProvisioning.md @@ -16,7 +16,7 @@ This guide will use a Lightweight M2M client to simulate the interaction with th this client will be explained when appropriate. This guide will make use of the automatic OMA Registry mapping, with the example of the `WeatherBalloon` defined in the -[Getting Started](userGuide.md#gettingstarted) section. So, the attributes will be defined exclusively by their name, +[Getting Started](userGuide.md#getting-started) section. So, the attributes will be defined exclusively by their name, and the mapping from and to a full LWM2M Mapping (like /387/3/23) will be performed by the IoT Agent. # Installation diff --git a/docs/deprecated.md b/docs/deprecated.md index 19297bf..7fa7338 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -38,4 +38,4 @@ features: | --------------------- | ----------------------------------------------------------- | ----------------------------- | | NGSIv1 API | Not yet defined | Not yet defined | | Support to Node.js v6 | Not yet defined but it will be done by May 2019 | Not yet defined | -| Support to Node.js v4 | 0.4.0 | Novemeber 28th, 2017 | +| Support to Node.js v4 | 0.4.0 | November 28th, 2017 | diff --git a/docs/deviceProvisioning.md b/docs/deviceProvisioning.md index 7126aaa..a20feab 100644 --- a/docs/deviceProvisioning.md +++ b/docs/deviceProvisioning.md @@ -7,7 +7,7 @@ # Overview -This guide will show the process of using the IoT Agent with device preprovisioning. In this use case, the owner of the +This guide will show the process of using the IoT Agent with device pre-provisioning. In this use case, the owner of the devices, before connecting each device, will provision the device information into the agent. Then, the device can register itself in the server and start being used with the agent. @@ -83,8 +83,8 @@ bin/iotagent-lwm2m-client.js ## Provisioning the device -Before starting to use any device, the device must be provisioned. In this step, we'll be sending a preprovisioning -request for a 'Robot' device. To send the preprovisioning request we will be using the `curl` command that comes +Before starting to use any device, the device must be provisioned. In this step, we'll be sending a pre-provisioning +request for a 'Robot' device. To send the pre-provisioning request we will be using the `curl` command that comes installed with any Unix-like OS. This request has to be sent to the administrative port of the IoT Agent (default value 4041), not to the Lightweight M2M @@ -231,7 +231,7 @@ EOF Note that the headers of the request to the Context Broker should match the ones you used in the Device Provisioning. Another thing to note is the Entity ID: it is formed by the concatenation of the type and the Device ID, separated by a -colon. This convention can be overriden in the provisioning request. +colon. This convention can be overridden in the provisioning request. ### Updating the active attributes diff --git a/docs/index.md b/docs/index.md index 9a7016e..06f9744 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ [![](https://nexus.lab.fiware.org/repository/raw/public/badges/stackoverflow/iot-agents.svg)](https://stackoverflow.com/questions/tagged/fiware+iot) This Internet of Things Agent is a bridge that can be used to communicate devices using the OMA Lightweight M2M protocol -and NGSI Context Brokers (like Orion). Lighteweight M2M is a lightweight protocol aimed to constrained devices and +and NGSI Context Brokers (like Orion). Lightweight M2M is a lightweight protocol aimed to constrained devices and communications where the bandwidth and device memory may be limited resources. Github's [README.md](https://github.com/telefonicaid/lightweightm2m-iotagent/blob/master/README.md) provides a good diff --git a/docs/limits.conf b/docs/limits.conf new file mode 100644 index 0000000..f42d171 --- /dev/null +++ b/docs/limits.conf @@ -0,0 +1,33 @@ +### USER LIMITS TUNING ### + +# Maximum number of open file descriptors +* soft nofile 65535 +* hard nofile 65535 + +# Maximum number of processes +* soft nproc unlimited +* hard nproc unlimited + +# Maximum filesize in KB +* soft fsize unlimited +* hard fsize unlimited + +# Maximum data size in KB +* soft data unlimited +* hard data unlimited + +# Maximum stack size in KB +* soft stack unlimited +* hard stack unlimited + +# Maximum resident set size in KB (ignored in Linux 2.4.30 and higher) +* soft rss unlimited +* hard rss unlimited + +# Limits of the core file size in KB +* soft core unlimited +* hard core unlimited + +# Maximum locked-in-memory address space in KB +* memlock unlimited +* memlock unlimited diff --git a/docs/roadmap.md b/docs/roadmap.md index ee260c8..083e579 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -37,7 +37,7 @@ release(s) generated in the next 9 months after the next planned release: - Support to FIWARE complex data models. - Deployment instructions for high availability scenarios. -- Incremental introduccion of ECMAScript6 syntax (previous analysis of which sub-set of interesting aspect we want to +- Incremental introduction of ECMAScript6 syntax (previous analysis of which sub-set of interesting aspect we want to take) ### Long term diff --git a/docs/staticConfiguration.md b/docs/staticConfiguration.md index 7383e84..8974d0a 100644 --- a/docs/staticConfiguration.md +++ b/docs/staticConfiguration.md @@ -15,7 +15,7 @@ This guide will use a Lightweight M2M client to simulate the interaction with th this client will be explained when appropriate. In this guide we will provide an explicit mapping for all the device attributes, using the `Robot` example given in the -[Getting Started](userGuide.md#gettingstarted) section. Some of them could be mapped automatically using the OMA +[Getting Started](userGuide.md#getting-started) section. Some of them could be mapped automatically using the OMA Registry automatic mapping (but those will be covered in other step-by-step guides). # Installation diff --git a/docs/sysctl.conf b/docs/sysctl.conf new file mode 100644 index 0000000..2395c43 --- /dev/null +++ b/docs/sysctl.conf @@ -0,0 +1,184 @@ +### KERNEL TUNING ### + +# Increase size of file handles and inode cache +fs.file-max = 1000000 + +# Do less swapping +vm.swappiness = 10 +vm.dirty_ratio = 60 +vm.dirty_background_ratio = 2 + +# Sets the time before the kernel considers migrating a process to another core +kernel.sched_migration_cost_ns = 5000000 + + + +### GENERAL NETWORK SECURITY OPTIONS ### + +# Number of times SYNACKs for passive TCP connection. +net.ipv4.tcp_synack_retries = 2 + +# Control Syncookies, asociated with security +net.ipv4.tcp_syncookies = 0 + +# How many times to try to retransmit the initial SYN packet for an active TCP connection attempt. +net.ipv4.tcp_syn_retries = 2 + +# Maximum number of remembered connection requests, which are still did not receive an acknowledgment +# from connecting client. +net.ipv4.tcp_max_syn_backlog = 4096 + +# Allowed local port range +net.ipv4.ip_local_port_range = 1024 65535 + +# Decrease the time default value for connections to keep alive +net.ipv4.tcp_keepalive_time = 300 +net.ipv4.tcp_keepalive_probes = 5 +net.ipv4.tcp_keepalive_intvl = 15 + +# Protect Against TCP Time-Wait +net.ipv4.tcp_rfc1337 = 1 + +# Decrease the time default value for tcp_fin_timeout connection +net.ipv4.tcp_fin_timeout = 15 + +# The maximum number of packets which may be queued for each unresolved address by other network layers. +net.ipv4.neigh.default.unres_qlen = 101 + +# Size of connection tracking table. Default value is nf_conntrack_buckets value * 4. +net.netfilter.nf_conntrack_max = 1048576 + + + +### TUNING NETWORK PERFORMANCE ### + +# Increase number of incoming connections +net.core.somaxconn = 65535 + +# Increase the maximum amount of option memory buffers +net.core.optmem_max = 25165824 + +# Increase number of incoming connections backlog +net.core.netdev_max_backlog = 65536 + +# Maximum number of packets taken from all interfaces in one polling cycle (NAPI poll). +net.core.netdev_budget = 500 + +# Maximum Socket Send Buffer +net.core.wmem_max = 33554432 + +# Default Socket Send Buffer +net.core.wmem_default = 31457280 + +# Maximum Socket Receive Buffer +net.core.rmem_max = 33554432 + +# Default Socket Receive Buffer +net.core.rmem_default = 31457280 + +# The default queuing discipline to use for network devices. This allows +# overriding the default of pfifo_fast with an alternative. Since the default +# queuing discipline is created without additional parameters so is best suited +# to queuing disciplines that work well without configuration like stochastic +# fair queue (sfq), CoDel (codel) or fair queue CoDel (fq_codel). +net.core.default_qdisc = fq_codel + +# The maximum number of packets that kernel can handle on a NAPI interrupt, it's a Per-CPU variable. +net.core.dev_weight = 64 + +# Increase the write-buffer-space allocatable +net.ipv4.tcp_wmem = 4096 5242880 33554432 + +net.ipv4.udp_wmem_min = 16384 + +# Increase the read-buffer space allocatable +net.ipv4.tcp_rmem = 4096 5242880 33554432 + +# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks +net.ipv4.tcp_max_tw_buckets = 1440000 +net.ipv4.tcp_tw_recycle = 1 +net.ipv4.tcp_tw_reuse = 1 + +# Number of orphans +net.ipv4.tcp_max_orphans = 16384 +net.ipv4.tcp_orphan_retries = 0 + +# Increase the maximum total buffer-space allocatable +# This is measured in units of pages (4096 bytes) +net.ipv4.tcp_mem = 786432 1048576 26777216 +net.ipv4.udp_mem = 65536 131072 262144 + +# Minimal size, in bytes, of receive buffers used by UDP sockets in moderation. +net.ipv4.udp_rmem_min = 16384 + +# If set, provide RFC2861 behavior and time out the congestion window after an +# idle period. An idle period is defined at the current RTO. If unset, the +# congestion window will not be timed out after an idle period. Default: 1 +net.ipv4.tcp_slow_start_after_idle = 0 + +# By default, TCP saves various connection metrics in the route cache when +# the connection closes, so that connections established in the near future +# can use these to set initial conditions. +net.ipv4.tcp_no_metrics_save = 0 + +# Enable IP forwarding +net.ipv4.ip_forward = 1 + +# Enable timestamps as defined in RFC1323 +net.ipv4.tcp_timestamps = 1 + +# Enable window scaling as defined in RFC1323 +net.ipv4.tcp_window_scaling = 1 + +# Enable reuse of TIME-WAIT sockets for new connections when it is safe from protocol viewpoint. +# 0 - disable +# 1 - global enable +# 2 - enable for loopback traffic only +net.ipv4.tcp_tw_reuse = 1 + +# Maximum memory used to reassemble IP fragments before the kernel begins to remove incomplete fragment +# queues to free up resources. +net.ipv4.ipfrag_low_thresh = 196608 + +# Time in seconds to keep an IP fragment in memory. +net.ipv4.ipfrag_time = 30 + +# Count buffering overhead as bytes/2^tcp_adv_win_scale +# (if tcp_adv_win_scale > 0) or bytes-bytes/2^(-tcp_adv_win_scale), +# if it is <= 0. +# Possible values are [-31, 31], inclusive. +# Default: 1 +net.ipv4.tcp_adv_win_scale = 3 + +# Enable select acknowledgments (SACKS) +net.ipv4.tcp_sack = 1 + +# Maximum time-to-live of entries in seconds +net.ipv4.inet_peer_maxttl = 600 + +# Minimum time-to-live of entries in seconds +net.ipv4.inet_peer_minttl = 120 + +# The approximate size of the storage +net.ipv4.inet_peer_threshold = 65664 + +# Accept packets with SRR option conf/all/accept_source_route must also be set to TRUE to accept packets +# with SRR option on the interface. Default TRUE (router), FALSE (host) +net.ipv4.conf.all.accept_source_route = 0 + +# Network types +# low latency, high bandwidth - htcp, default = cubic +# Set the congestion control algorithm to be used for new connections. The algorithm "reno" is always available, but +# additional choices may be available based on kernel configuration. Default is set as part of kernel configuration. +# For passive connections, the listener congestion control choice is inherited. +# [see setsockopt(listenfd, SOL_TCP, TCP_CONGESTION, "name" ...) ] +net.ipv4.tcp_congestion_control = htcp + +# TCP queue length +net.ipv4.neigh.default.proxy_qlen = 64 + +# Disable IPv6, Note that you must list all of the targeted interfaces explicitly, as disabling all.disable_ipv6 +# does not apply to interfaces that are already "up" when sysctl settings are applied. Note 2, if disabling IPv6 +# by sysctl, you should comment out the IPv6 hosts in your /etc/hosts +net.ipv6.conf.all.disable_ipv6 = 1 +net.ipv6.conf.default.disable_ipv6 = 1 diff --git a/docs/userGuide.md b/docs/userGuide.md index eb8e89e..fca82e5 100644 --- a/docs/userGuide.md +++ b/docs/userGuide.md @@ -1,11 +1,11 @@ # OMA Lightweight M2M IoT Agent: User and Development Guide -- [Overview](#overview) +- [General Overview](#general-overview) - [Getting started](#getting-started) -- [Testing](#testing) +- [Project Testing](#project-testing) - [Development](#development) -# Overview +## General Overview The Lightweight M2M IoT Agent is a standard FIWARE IoT Agent that implements the bridge of the OMA Lightweight M2M protocol with the internal protocol for the FIWARE components (OMA NGSI). This IoT Agent is based in the public Node.js @@ -22,14 +22,14 @@ This project has, then, two APIs: - The API for traffic north of the IoT Agent (NGSI): information about the North Port NGSI mapping can be obtained in the same Node.JS IoT Agent Library documentation. -You will find examples and more detailed information in the Getting Started howtos below. +You will find examples and more detailed information in the Getting Started how-tos below. -# Getting started +## Getting started -This document links a set of howtos oriented to give a quick step-by-step example on how to use the agent with different -types of configurations. It's important to remark that those configuration options are not mutually exclusive: an IoT -Agent can have some device preprovisioned, some configuration groups defined and some static configurations also, each -for different types of devices. +This document links a set of how-tos oriented to give a quick step-by-step example on how to use the agent with +different types of configurations. It's important to remark that those configuration options are not mutually exclusive: +an IoT Agent can have some device pre-provisioned, some configuration groups defined and some static configurations +also, each for different types of devices. Some of the guides will share the use of a faked device type called `Robot` with the following characteristics: @@ -52,11 +52,11 @@ Each guide is presented with a brief explanation about its contents: - [Device Provisioning Guide](deviceProvisioning.md): this guide shows how to configure, launch and use an IoT Agent, provisioning each device before sending its measures. - [Configuration Provisioning Guide](configurationProvisioning.md): this guide shows how to configure a group of - devices for being autoprovisioned when they register in the agent. + devices for being auto-provisioned when they register in the agent. - [Static Configuration Guide](staticConfiguration.md): this guide shows how to configure static routes that map incoming devices to different statically configured types. -# Testing +## Testing The IoT Agent comes with a test suite to check the main functionalities. You can execute the tests using: @@ -73,11 +73,11 @@ executed so DO NOT EXECUTE THIS TESTS ON PRODUCTION MACHINES. It must be also noted that although the lightweightM2M-iotagent works with MongoDB replica sets, the unit testing suite and its scripts require using a single MongoDB instance. -# Development +## Development -## Contribution Guidelines +### Contribution Guidelines -### Overview +#### Overview Being an Open Source project, everyone can contribute, provided that it respect the following points: @@ -124,9 +124,9 @@ branch in the main lightweightm2m-iotagent repository, by following this steps ``` Contributions following this guidelines will be added to the `master` branch, and released in the next version. The -release process is explaind in the _Releasing_ section below. +release process is explained in the _Releasing_ section below. -### Branching model +#### Branching model In order to start developing a new feature or refactoring, a new branch should be created with name `task/`. This branch must be created from the current version of the `master` branch. Once the new functionality has been @@ -143,7 +143,7 @@ All the `task/*` and `bug/*` branches are temporary, and should be removed once There is another set of branches called `release/`, one for each version of the product. This branches point to each of the released versions of the project, they are permanent and they are created with each release. -### Change log +#### Change log The project contains a version changelog, called CHANGES_NEXT_RELEASE, that can be found in the root of the project. Whenever a new feature or bug fix is going to be merged with `master`, a new entry should be added to this changelog. @@ -152,19 +152,19 @@ The new entry should contain the reference number of the issue it is solving (if When a new version is released, the change log is cleared, and remains fixed in the last commit of that version. The content of the change log is also moved to the release description in the GitHub release. -### Releasing +#### Releasing The process of making a release consists of the following steps: -1. Create a new task branch changing the development version number in the package.json (with a sufix `-next`), to the - new target version (without any sufix), and PR into `master`. +1. Create a new task branch changing the development version number in the package.json (with a suffix `-next`), to the + new target version (without any suffix), and PR into `master`. 2. Create a tag from the last version of `master` named with the version number and push it to the repository. 3. Create the release in Github, from the created tag. In the description, add the contents of the change log. 4. Create a release branch from the last version of `master` named with the version number. -5. Create a new task for preparing the next release, adding the sufix `-next` to the current version number (to signal +5. Create a new task for preparing the next release, adding the suffix `-next` to the current version number (to signal this as the development version). -## Testing +## Project Testing ### Prerequisites @@ -248,7 +248,7 @@ npm run watch Istanbul -Analizes the code coverage of your tests. +Analyzes the code coverage of your tests. To generate an HTML coverage report under `site/coverage/` and to print out a summary, type @@ -269,7 +269,7 @@ npm run clean ### Prettify Code -Runs the [prettier](https://prettier.io) code formatter to ensure consistent code style (whitespacing, parameter +Runs the [prettier](https://prettier.io) code formatter to ensure consistent code style (whitespaces, parameter placement and breakup of long lines etc.) within the codebase. ```bash diff --git a/package.json b/package.json index ee44cf8..0c165a6 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "cheerio": "1.0.0-rc.2", "xmldom": "0.1.27", "logops": "2.1.0", - "iotagent-node-lib": "git://github.com/telefonicaid/iotagent-node-lib.git#master", + "iotagent-node-lib": "git://github.com/flopezag/iotagent-node-lib.git#feature/cluster-support", "lwm2m-node-lib": "git://github.com/telefonicaid/lwm2m-node-lib.git#master" }, "devDependencies": { diff --git a/test/.jshintrc b/test/.jshintrc index d442544..0d6cec8 100644 --- a/test/.jshintrc +++ b/test/.jshintrc @@ -9,6 +9,7 @@ "quotmark": "single", "undef": true, "unused": true, + "esversion": 6, "trailing": true, "maxparams": 6, "maxdepth": 4, From 555ccd7f553e74eb08fe19cbe87d73b7921dc272 Mon Sep 17 00:00:00 2001 From: Fernando Lopez Date: Fri, 18 Oct 2019 19:52:44 +0200 Subject: [PATCH 2/6] Update CNR --- CHANGES_NEXT_RELEASE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 719a716..d60fc0e 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,7 +1,7 @@ - Set Nodejs 8 as minimum version in packages.json (effectively removing Nodev6 from supported versions) - ADD PM2_ENABLED flag to Docker - Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues -- Configure multithreading using cluster nodejs module (through multiCore config.js parameter or IOTA_MULTI_CORE -environment variable) +- NOTE: in the "upgrade iotagent-node-lib from X to Y" include a mention to configure multi-threading using cluster +nodejs module (through multiCore config.js parameter or IOTA_MULTI_CORE) - Correct typos in documentation files - Add lwm2m agent version in the description of /iot/about request From c6ab4db6433d5bcd40c3fc431aba1c24eac907f5 Mon Sep 17 00:00:00 2001 From: Fernando Lopez Date: Fri, 18 Oct 2019 19:53:40 +0200 Subject: [PATCH 3/6] Comment the multiCore parameter --- config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.js b/config.js index 04f5bf2..0b22499 100644 --- a/config.js +++ b/config.js @@ -152,6 +152,6 @@ config.ngsi = { * flag indicating whether the node server will be executed in multi-core option (true) or it will be a * single-thread one (false). */ -config.multiCore = true; +// config.multiCore = true; module.exports = config; From c5be33fed27b73267cd181ed9ee8f2970fdb8829 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Tue, 22 Oct 2019 10:15:01 +0200 Subject: [PATCH 4/6] Minor changes to fix code comments --- CHANGES_NEXT_RELEASE | 5 +---- package.json | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e643c3d..71f64cb 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,8 +1,5 @@ -- Set Nodejs 8 as minimum version in packages.json (effectively removing Nodev6 from supported versions) -- ADD PM2_ENABLED flag to Docker -- Upgrade NodeJS version from 8.16.0 to 8.16.1 in Dockerfile due to security issues +- Fix: support to lazy attributes and commands based in NGSIv2 (#104) - NOTE: in the "upgrade iotagent-node-lib from X to Y" include a mention to configure multi-threading using cluster nodejs module (through multiCore config.js parameter or IOTA_MULTI_CORE) - Correct typos in documentation files - Add lwm2m agent version in the description of /iot/about request -- Fix: support to lazy attributes and commands based in NGSIv2 (#104) diff --git a/package.json b/package.json index 1c750ea..aa2e6de 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "cheerio": "1.0.0-rc.2", "xmldom": "0.1.27", "logops": "2.1.0", - "iotagent-node-lib": "git://github.com/flopezag/iotagent-node-lib.git#feature/cluster-support", + "iotagent-node-lib": "git://github.com/telefonicaid/iotagent-node-lib.git#master", "lwm2m-node-lib": "git://github.com/telefonicaid/lwm2m-node-lib.git#master" }, "devDependencies": { From c5378c39af0f6be8509cd961a09d05177ecdae78 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Tue, 22 Oct 2019 10:37:26 +0200 Subject: [PATCH 5/6] Re-add fork/branch for test - please remove when ready --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index aa2e6de..1c750ea 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "cheerio": "1.0.0-rc.2", "xmldom": "0.1.27", "logops": "2.1.0", - "iotagent-node-lib": "git://github.com/telefonicaid/iotagent-node-lib.git#master", + "iotagent-node-lib": "git://github.com/flopezag/iotagent-node-lib.git#feature/cluster-support", "lwm2m-node-lib": "git://github.com/telefonicaid/lwm2m-node-lib.git#master" }, "devDependencies": { From 02154088b7197809763450d44e2c59b630b0af7d Mon Sep 17 00:00:00 2001 From: Fermin Galan Marquez Date: Thu, 31 Oct 2019 11:22:32 +0100 Subject: [PATCH 6/6] FIX iotagent-node-lib dependency back to master --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1c750ea..aa2e6de 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "cheerio": "1.0.0-rc.2", "xmldom": "0.1.27", "logops": "2.1.0", - "iotagent-node-lib": "git://github.com/flopezag/iotagent-node-lib.git#feature/cluster-support", + "iotagent-node-lib": "git://github.com/telefonicaid/iotagent-node-lib.git#master", "lwm2m-node-lib": "git://github.com/telefonicaid/lwm2m-node-lib.git#master" }, "devDependencies": {