diff --git a/.flake8 b/.flake8 index fa5a54d..23de0c0 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,7 @@ [flake8] +# spellcheck +dictionaries = en_US,python,technical + ignore = # E201 whitespace after '(' E201, @@ -13,6 +16,11 @@ ignore = # E303 too many blank lines E303, + +per-file-ignores = + src/sml2mqtt/config/*:N805 + tests/*:E501 + max-line-length = 120 exclude = .git, @@ -23,7 +31,3 @@ exclude = conf, __init__.py, tests/conftest.py, - - # the interfaces will throw unused imports - HABApp/openhab/interface.py, - HABApp/openhab/interface_async.py, diff --git a/.github/workflows/publish-dockerhub.yml b/.github/workflows/publish-dockerhub.yml new file mode 100644 index 0000000..6b95333 --- /dev/null +++ b/.github/workflows/publish-dockerhub.yml @@ -0,0 +1,44 @@ +name: Publish sml2mqtt to dockerhub +on: + release: + types: [published] +# on: [push, pull_request] + + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: master + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + version: latest + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v4 + with: + push: true + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 + tags: | + spacemanspiff2007/sml2mqtt:latest + spacemanspiff2007/sml2mqtt:${{ github.ref_name }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/run_tox.yml b/.github/workflows/run_tox.yml index 752fb73..e13878f 100644 --- a/.github/workflows/run_tox.yml +++ b/.github/workflows/run_tox.yml @@ -3,10 +3,22 @@ name: Tests on: [push, pull_request] jobs: - test: + pre-commit: + name: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: pre-commit/action@v3.0.0 + + + tests: + needs: pre-commit runs-on: ubuntu-latest strategy: - max-parallel: 2 + max-parallel: 4 matrix: python-version: ['3.8', '3.9', '3.10', '3.11'] diff --git a/.gitignore b/.gitignore index 007972a..4ca5ce4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ __pycache__ /build /venv/ + +# sphinx +build +make.bat diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 974064c..e2f1f7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,17 +2,52 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: + - id: check-ast + - id: check-builtin-literals + - id: check-docstring-first + - id: check-merge-conflict +# - id: check-toml - id: check-yaml + - id: debug-statements - id: end-of-file-fixer - id: trailing-whitespace + - repo: https://github.com/pycqa/isort - rev: v5.11.3 + rev: 5.12.0 hooks: - id: isort name: isort (python) + - repo: https://github.com/PyCQA/flake8 rev: '6.0.0' hooks: - id: flake8 + additional_dependencies: + - flake8-bugbear==23.2.13 + - flake8-comprehensions==3.10.1 + - flake8-pytest-style==1.6 +# - flake8-spellcheck==0.28 +# - flake8-unused-arguments==0.0.12 + - flake8-noqa==1.3 + - pep8-naming==0.13.3 + + + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: ["--py38-plus"] + + + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + + + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..8b234d8 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,30 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +# Optionally set the version of Python and requirements required to build your docs +python: + install: + - requirements: requirements_setup.txt + - requirements: docs/requirements.txt + - method: pip + path: . diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e442291 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.10-alpine + +VOLUME /sml2mqtt + +COPY . /tmp/sml2mqtt_src + +RUN apk add --no-cache python3 py3-wheel py3-pip gcc musl-dev python3-dev && \ + # install sml2mqtt from local dir + pip install --no-cache-dir /tmp/sml2mqtt_src && \ + # cleanup + pip install --no-cache-dir pyclean && pyclean /usr && pip uninstall -y pyclean setuptools wheel pip && \ + apk del py3-wheel py3-pip gcc musl-dev python3-dev && \ + rm -fr /tmp/* + +WORKDIR /sml2mqtt +CMD [ "sml2mqtt", "--config", "/sml2mqtt/config.yml"] diff --git a/docs/_static/theme_changes.css b/docs/_static/theme_changes.css new file mode 100644 index 0000000..f566b61 --- /dev/null +++ b/docs/_static/theme_changes.css @@ -0,0 +1,7 @@ + +/* On screens that are 767px or more */ +@media screen and (min-width: 767px) { + .wy-nav-content { + max-width: 1100px !important; + } +} diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..09bc8b8 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,92 @@ +import os +import re +import sys + +RTD_BUILD = os.environ.get('READTHEDOCS') == 'True' + + +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'sml2mqtt' +copyright = '2023, spacemanspiff2007' +author = 'spacemanspiff2007' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + 'sphinx_exec_code', + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx_autodoc_typehints', + 'sphinxcontrib.autodoc_pydantic', +] + +templates_path = ['_templates'] +exclude_patterns = [] + + +# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-add_module_names +# use class name instead of FQN +add_module_names = False + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] +html_css_files = ['theme_changes.css'] + + +# -- Options for autodoc ------------------------------------------------- +autodoc_member_order = 'bysource' +autoclass_content = 'class' + +# so autodoc does find the source +sys.path.insert(0, os.path.join(os.path.abspath('..'), 'src')) + + +# -- Options for autodoc pydantic ------------------------------------------------- +# https://autodoc-pydantic.readthedocs.io/en/stable/ + +# No config on member +autodoc_pydantic_model_show_config_member = False +autodoc_pydantic_model_show_config_summary = False + +# No validators +autodoc_pydantic_model_show_validator_summary = False +autodoc_pydantic_model_show_validator_members = False + +# Model configuration +autodoc_pydantic_model_signature_prefix = 'settings' +autodoc_pydantic_model_show_json = False +autodoc_pydantic_model_show_field_summary = False +autodoc_pydantic_model_member_order = 'bysource' + +# Field config +autodoc_pydantic_field_show_alias = False +autodoc_pydantic_field_list_validators = False +autodoc_pydantic_field_swap_name_and_alias = True + +# -- Options for intersphinx ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html +if RTD_BUILD: + intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None) + } + +# -- Options for nitpick ------------------------------------------------- +# Don't show warnings for missing python references since these are created via intersphinx during the RTD build +if not RTD_BUILD: + nitpick_ignore_regex = [ + (re.compile(r'^py:class'), re.compile(r'pathlib\..+')), + (re.compile(r'^py:data'), re.compile(r'typing\..+')), + (re.compile(r'^py:class'), re.compile(r'pydantic\..+|.+Constrained(?:Str|Int)Value')), + ] diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 0000000..cc4bfcb --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,219 @@ +************************************** +Configuration & CLI +************************************** + +.. _COMMAND_LINE_INTERFACE: + +Command Line Interface +====================================== + +.. exec_code:: + :hide_code: + + import sml2mqtt.__args__ + sml2mqtt.__args__.get_command_line_args(['-h']) + + +Configuration +====================================== + +Configuration is done through ``config.yml`` The parent folder of the file can be specified with ``-c PATH`` or ``--config PATH``. +If nothing is specified the file ``config.yml`` is searched in the subdirectory ``sml2mqtt`` in + +* the current working directory +* the venv directory +* the user home + +If the config is specified and it does not yet exist a default configuration file will be created + +Example +-------------------------------------- + +.. code-block:: yaml + + logging: + level: INFO # Log level + file: sml2mqtt.log # Log file path (absolute or relative to config file) + + mqtt: + connection: + client id: sml2mqtt + host: localhost + port: 1883 + user: '' + password: '' + tls: false + tls insecure: false + + # MQTT default configuration + # All other topics use these values if no other values for qos/retain are set + # It's possible to override + # - topic (fragment that is used to build the full mqtt topic) + # - full_topic (will not build the topic from the fragments but rather use the configured value) + # - qos + # - retain + # for each (!) mqtt-topic entry + defaults: + qos: 0 + retain: false + topic prefix: sml2mqtt + + last will: + topic: status + + general: + Wh in kWh: true # Automatically convert Wh to kWh + republish after: 120 # Republish automatically after this time (if no other every filter is configured) + + # Serial port configurations for the sml readers + ports: + - url: COM1 + timeout: 3 + - url: /dev/ttyS0 + timeout: 3 + + + devices: + # Device configuration by OBIS value 0100000009ff or by url if the device does not report OBIS 0100000009ff + 11111111111111111111: + mqtt: + topic: DEVICE_TOPIC + + # OBIS IDs that will not be processed (optional) + skip: + - OBIS + - values + - to skip + + # Configuration how each OBIS value is reported. Create as many OBIS IDs (e.g. 0100010800ff as you like). + # Each sub entry (mqtt, workarounds, transformations, filters) is optional and can be omitted + values: + + OBIS: + # Sub topic how this value is reported. + mqtt: + topic: OBIS + + # Workarounds allow the enabling workarounds (e.g. if the device has strange behaviour) + # These are the available workarounds + workarounds: + - negative on energy meter status: true # activate this workaround + + # Transformations allow mathematical calculations on the obis value + # They are applied in order how they are defined + transformations: + - factor: 3 # multiply with factor + - offset: 100 # add offset + - round: 2 # round on two digits + + # Filters control how often a value is published over mqtt. + # If one filter is true the value will be published + filters: + - diff: 10 # report if value difference is >= 10 + - perc: 10 # report if percentage change is >= 10% + - every: 120 # report at least every 120 secs (overrides the value from general) + + + +Example devices +-------------------------------------- +One energy meter is connected to the serial port. The serial meter reports OBIS ``0100000009ff`` +as ``11111111111111111111``. + +For this device + +* the mqtt topic fragment is set to ``light`` +* the value ``0100010801ff`` will not be published +* The following values of the device are specially configured: + + * Energy value (OBIS ``0100010800ff``) + + * Will be rounded to one digit + * Will be published on change **or** at least every hour + * The mqtt topic used is ``sml2mqtt/light/energy``. + (Built through ``topic prefix`` + ``device mqtt`` + ``value mqtt``) + + + * Power value (OBIS ``0100100700ff``) + + * Will be rounded to one digit + * Will be published if at least a 5% power change occurred **or** at least every 2 mins + (default from ``general`` -> ``republish after``) + * The mqtt topic used is ``sml2mqtt/light/power`` + + +.. code-block:: yaml + + devices: + 11111111111111111111: + mqtt: + topic: light + skip: + - 0100010801ff + values: + 0100010800ff: + mqtt: + topic: energy + transformations: + - round: 1 + filters: + - every: 3600 + 0100100700ff: + mqtt: + topic: power + filters: + - perc: 5 + + +Configuration Reference +====================================== +All possible configuration options are described here. Not all entries are created by default in the config file +and one should take extra care when changing those entries. + +.. autopydantic_model:: sml2mqtt.config.config.Settings + +logging +-------------------------------------- + +.. autopydantic_model:: sml2mqtt.config.logging.LoggingSettings + +general +-------------------------------------- + +.. autopydantic_model:: sml2mqtt.config.config.GeneralSettings + +ports +-------------------------------------- + +.. autopydantic_model:: sml2mqtt.config.config.PortSettings + +mqtt +-------------------------------------- + +.. py:currentmodule:: sml2mqtt.config.mqtt + +.. autopydantic_model:: MqttConfig + +.. autopydantic_model:: MqttConnection + +.. autopydantic_model:: OptionalMqttPublishConfig + +.. autopydantic_model:: MqttDefaultPublishConfig + +devices +-------------------------------------- + +.. py:currentmodule:: sml2mqtt.config.device + +.. autopydantic_model:: SmlDeviceConfig + +.. autopydantic_model:: SmlValueConfig + +.. autoclass:: WorkaroundOptionEnum + :members: + +.. autoclass:: TransformOptionEnum + :members: + +.. autoclass:: FilterOptionEnum + :members: diff --git a/docs/getting_started.rst b/docs/getting_started.rst new file mode 100644 index 0000000..befcc3c --- /dev/null +++ b/docs/getting_started.rst @@ -0,0 +1,124 @@ +************************************** +Getting started +************************************** + +First install ``sml2mqtt`` e.g in a :ref:`virtual environment `. + +Run ``sml2mqtt`` with a path to a configuration file. +A new default configuration file will be created. + +Edit the configuration file and add the serial ports. + +Now run ``sml2mqtt`` with the path to the configuration file and the ``analyze`` option. +(see :ref:`command line interface `). +This will process one sml frame from the meter and report the output. +It's a convenient way to check what values will be reported. +It will also show how the configuration changes the sml values (e.g. if you add a transformation or a workaround). + +Example output for the meter data: + +.. code-block:: text + + SmlMessage + transaction_id: 17c77d6b + group_no : 0 + abort_on_error: 0 + message_body + codepage : None + client_id : None + req_file_id: 07ed29cd + server_id : 11111111111111111111 + ref_time : None + sml_version: None + crc16 : 25375 + SmlMessage + transaction_id: 17c77d6c + group_no : 0 + abort_on_error: 0 + message_body + client_id : None + sever_id : 11111111111111111111 + list_name : 0100620affff + act_sensor_time : 226361515 + val_list: list + + obis : 8181c78203ff + status : None + val_time : None + unit : None + scaler : None + value : ISK + value_signature: None + -> (Hersteller-Identifikation) + + obis : 0100000009ff + status : None + val_time : None + unit : None + scaler : None + value : 11111111111111111111 + value_signature: None + -> (Geräteeinzelidentifikation) + + obis : 0100010800ff + status : 386 + val_time : None + unit : 30 + scaler : -1 + value : 123456789 + value_signature: None + -> 12345678.9Wh (Zählerstand Total) + + obis : 0100010801ff + status : None + val_time : None + unit : 30 + scaler : -1 + value : 123456789 + value_signature: None + -> 12345678.9Wh (Zählerstand Tarif 1) + + obis : 0100010802ff + status : None + val_time : None + unit : 30 + scaler : -1 + value : 0 + value_signature: None + -> 0.0Wh (Zählerstand Tarif 2) + + obis : 0100100700ff + status : None + val_time : None + unit : 27 + scaler : 0 + value : 555 + value_signature: None + -> 555W (aktuelle Wirkleistung) + + obis : 8181c78205ff + status : None + val_time : None + unit : None + scaler : None + value : XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + value_signature: None + -> (Öffentlicher Schlüssel) + list_signature : None + act_gateway_time: None + crc16 : 22117 + SmlMessage + transaction_id: 17c77d6d + group_no : 0 + abort_on_error: 0 + message_body + global_signature: None + crc16 : 56696 + + +Check if the meter reports the serial number unter obis ``0100000009ff``. +If not it's possible to configure another number (of even multiple ones) for configuration matching. +If yes replace ``SERIAL_ID_HEX`` in the dummy configuration with the reported +serial number (here ``11111111111111111111``). +Modify the device configuration to your liking (see configuration documentation). +Run the analyze command again and see how the output changes and observe the reported values. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..bef04b1 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,19 @@ + +Welcome to sml2mqtt's documentation! +==================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + configuration + getting_started + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..e5af37a --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,142 @@ +************************************** +Installation +************************************** + +Virtual environment +====================================== + +.. _INSTALLATION_VENV: + +Installation +-------------------------------------- + +.. hint:: + On Windows use the ``python`` command instead of ``python3`` + + +Navigate to the folder in which the virtual environment shall be created (e.g.):: + + cd /opt/sml2mqtt + +If the folder does not exist yet you can create it with the ``mkdir`` command:: + + mkdir /opt/sml2mqtt + + +Create virtual environment (this will create a new subfolder "venv"):: + + python3 -m venv venv + + +Go into folder of virtual environment:: + + cd venv + + +#. Activate the virtual environment + + Linux:: + + source bin/activate + + Windows:: + + Scripts\activate + +#. Upgrade pip and setuptools:: + + python3 -m pip install --upgrade pip setuptools + +Install sml2mqtt + + python3 -m pip install sml2mqtt + +#. Run sml2mqtt:: + + sml2mqtt --config PATH_TO_CONFIGURAT_FILE + + +Upgrading +-------------------------------------- +#. Stop sml2mqtt + +#. Activate the virtual environment + + Navigate to the folder where sml2mqtt is installed:: + + cd /opt/sml2mqtt + + Activate the virtual environment + + Linux:: + + source bin/activate + + Windows:: + + Scripts\activate + +#. Run the following command in your activated virtual environment:: + + python3 -m pip install --upgrade sml2mqtt + +#. Start sml2mqtt + +#. Observe the log for errors in case there were changes + + +Autostart after reboot +-------------------------------------- + +To automatically start the sml2mqtt from the virtual environment after a reboot call:: + + nano /etc/systemd/system/sml2mqtt.service + + +and copy paste the following contents. If the user/group which is running sml2mqtt is not "openhab" replace accordingly. +If your installation is not done in "/opt/sml2mqtt/venv/bin" replace accordingly as well:: + + [Unit] + Description=sml2mqtt + Documentation=https://github.com/spacemanspiff2007/sml2mqtt + After=network-online.target + + [Service] + Type=simple + User=openhab + Group=openhab + Restart=on-failure + RestartSec=2min + ExecStart=/opt/sml2mqtt/venv/bin/sml2mqtt -c PATH_TO_CONFIGURATION_FILE + + [Install] + WantedBy=multi-user.target + + +Now execute the following commands to enable autostart:: + + sudo systemctl --system daemon-reload + sudo systemctl enable sml2mqtt.service + + +It is now possible to start, stop, restart and check the status of sml2mqtt with:: + + sudo systemctl start sml2mqtt.service + sudo systemctl stop sml2mqtt.service + sudo systemctl restart sml2mqtt.service + sudo systemctl status sml2mqtt.service + + +Docker +====================================== + + +Installation through `docker `_ is available: + +.. code-block:: bash + + docker pull spacemanspiff2007/sml2mqtt:latest + + +The docker image has one volume ``/sml2mqtt`` which has to be mounted. +There the ``config.yml`` will be used or a new ``config.yml`` will be created diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..1dfe91c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +# Packages required to build the documentation +sphinx >= 6, < 7 +sphinx-autodoc-typehints >= 1.22, < 2 +sphinx_rtd_theme == 1.2.0 +sphinx-exec-code == 0.10 +autodoc_pydantic >= 1.8, < 1.9 diff --git a/readme.md b/readme.md index 501a2b1..b7602f1 100644 --- a/readme.md +++ b/readme.md @@ -5,329 +5,17 @@ [![PyPI](https://img.shields.io/pypi/v/sml2mqtt)](https://pypi.org/project/sml2mqtt/) [![Downloads](https://pepy.tech/badge/sml2mqtt/month)](https://pepy.tech/project/sml2mqtt) -_A simple sml to mqtt bridge_ +_A simple yet flexible sml to mqtt bridge_ sml2mqtt is a asyncio application that can read multiple sml (Smart Message Language) streams from energy meters and report the values through mqtt. -## Installation -Navigate to the folder where the virtual environment shall be created (e.g.). -``` -cd /opt/sml2mqtt -``` -If the folder does not exist yet you can create it with the ``mkdir`` command. +# Documentation +[The documentation can be found at here](https://sml2mqtt.readthedocs.io) -Create virtual environment (this will create a new subfolder "venv"):: -``` -python3 -m venv venv -``` - -Go into folder of virtual environment -``` -cd venv -``` - -Activate the virtual environment -``` -source bin/activate -``` - -Install sml2mqtt -``` -python3 -m pip install --upgrade pip sml2mqtt -``` - - -Run sml2mqtt -``` -sml2mqtt --config PATH_TO_CONFIGURATION_FOLDER -``` -A good configuration path would be ```/opt/sml2mqtt```. -sml2mqtt will automatically create a default configuration and a logfile there. - -## Autostart - -To automatically start the sml2mqtt from the virtual environment after a reboot call:: -``` -nano /etc/systemd/system/sml2mqtt.service -``` - -and copy paste the following contents. If the user/group which is running sml2mqtt is not "openhab" replace accordingly. -If your installation is not done in "/opt/sml2mqtt/venv/bin" replace accordingly as well:: - -```text -[Unit] -Description=sml2mqtt -Documentation=https://github.com/spacemanspiff2007/sml2mqtt -After=network-online.target - -[Service] -Type=simple -User=openhab -Group=openhab -Restart=on-failure -RestartSec=2min -ExecStart=/opt/sml2mqtt/venv/bin/sml2mqtt -c PATH_TO_CONFIGURATION_FILE - -[Install] -WantedBy=multi-user.target -``` - -Now execute the following commands to enable autostart:: -``` -sudo systemctl --system daemon-reload -sudo systemctl enable sml2mqtt.service -``` - -It is now possible to start, stop, restart and check the status of sml2mqtt with:: -``` -sudo systemctl start sml2mqtt.service -sudo systemctl stop sml2mqtt.service -sudo systemctl restart sml2mqtt.service -sudo systemctl status sml2mqtt.service -``` - -## Analyze - -Starting sml2mqtt with the ``-a`` or ``--analyze`` arg will analyze a single sml frame and report the output. -It's a convenient way to check what values will be reported. -It will also show how the configuration changes the sml values (e.g. if you add a transformation or a workaround). - -``` -sml2mqtt --config PATH_TO_CONFIGURATION_FOLDER -a -``` -Output -``` -SmlMessage - transaction_id: 17c77d6b - group_no : 0 - abort_on_error: 0 - message_body - codepage : None - client_id : None - req_file_id: 07ed29cd - server_id : 11111111111111111111 - ref_time : None - sml_version: None - crc16 : 25375 -SmlMessage - transaction_id: 17c77d6c - group_no : 0 - abort_on_error: 0 - message_body - client_id : None - sever_id : 11111111111111111111 - list_name : 0100620affff - act_sensor_time : 226361515 - val_list: list - - obis : 8181c78203ff - status : None - val_time : None - unit : None - scaler : None - value : ISK - value_signature: None - -> (Hersteller-Identifikation) - - obis : 0100000009ff - status : None - val_time : None - unit : None - scaler : None - value : 11111111111111111111 - value_signature: None - -> (Geräteeinzelidentifikation) - - obis : 0100010800ff - status : 386 - val_time : None - unit : 30 - scaler : -1 - value : 123456789 - value_signature: None - -> 12345678.9Wh (Zählerstand Total) - - obis : 0100010801ff - status : None - val_time : None - unit : 30 - scaler : -1 - value : 123456789 - value_signature: None - -> 12345678.9Wh (Zählerstand Tarif 1) - - obis : 0100010802ff - status : None - val_time : None - unit : 30 - scaler : -1 - value : 0 - value_signature: None - -> 0.0Wh (Zählerstand Tarif 2) - - obis : 0100100700ff - status : None - val_time : None - unit : 27 - scaler : 0 - value : 555 - value_signature: None - -> 555W (aktuelle Wirkleistung) - - obis : 8181c78205ff - status : None - val_time : None - unit : None - scaler : None - value : XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - value_signature: None - -> (Öffentlicher Schlüssel) - list_signature : None - act_gateway_time: None - crc16 : 22117 -SmlMessage - transaction_id: 17c77d6d - group_no : 0 - abort_on_error: 0 - message_body - global_signature: None - crc16 : 56696 -``` - -## Configuration - -Configuration is done in the ``config.yml`` file. -It is possible to create a default configuration. Just use the ``-c`` command line switch and specify a path to load -the configuration from. If the file does not exist a sample file will be created. - -```yml -logging: - level: INFO # Log level - file: sml2mqtt.log # Log file path (absolute or relative to config file) - -mqtt: - connection: - client id: sml2mqtt - host: localhost - port: 1883 - user: '' - password: '' - tls: false - tls insecure: false - - # MQTT default configuration - # All other topics use these values if no other values for qos/retain are set - # It's possible to override - # - topic (fragment that is used to build the full mqtt topic) - # - full_topic (will not build the topic from the fragments but rather use the configured value) - # - qos - # - retain - # for each (!) mqtt-topic entry - defaults: - qos: 0 - retain: false - topic prefix: sml2mqtt - - last will: - topic: status - -general: - Wh in kWh: true # Automatically convert Wh to kWh - republish after: 120 # Republish automatically after this time (if no other every filter is configured) - -# Serial port configurations for the sml readers -ports: -- url: COM1 - timeout: 3 -- url: /dev/ttyS0 - timeout: 3 - - -devices: - # Device configuration by OBIS value 0100000009ff or by url if the device does not report OBIS 0100000009ff - 11111111111111111111: - mqtt: - topic: DEVICE_TOPIC - - # OBIS IDs that will not be processed (optional) - skip: - - OBIS - - values - - to skip - - # Configuration how each OBIS value is reported. Create as many OBIS IDs (e.g. 0100010800ff as you like). - # Each sub entry (mqtt, workarounds, transformations, filters) is optional and can be omitted - values: - - OBIS: - # Sub topic how this value is reported. - mqtt: - topic: OBIS - - # Workarounds allow the enabling workarounds (e.g. if the device has strange behaviour) - # These are the available workarounds - workarounds: - - negative on energy meter status: true # activate this workaround - - # Transformations allow mathematical calculations on the obis value - # They are applied in order how they are defined - transformations: - - factor: 3 # multiply with factor - - offset: 100 # add offset - - round: 2 # round on two digits - - # Filters control how often a value is published over mqtt. - # If one filter is true the value will be published - filters: - - diff: 10 # report if value difference is >= 10 - - perc: 10 # report if percentage change is >= 10% - - every: 120 # report at least every 120 secs (overrides the value from general) -``` - - -### Example devices configuration -One energy meter is connected to the serial port. The serial meter reports OBIS ``0100000009ff`` -as ``11111111111111111111``. - -For this device -- the mqtt topic fragment is set to ``light`` -- the value ``0100010801ff`` will not be published -- The following values of the device are specially configured: - - - Energy value (OBIS ``0100010800ff``) - - Will be rounded to one digit - - Will be published on change **or** at least every hour - - The mqtt topic used is ``sml2mqtt/light/energy`` - - - Power value (OBIS ``0100100700ff``) - - Will be rounded to one digit - - Will be published if at least a 5% power change occurred **or** at least every 2 mins - (default from ``general`` -> ``republish after``) - - The mqtt topic used is ``sml2mqtt/light/power`` - - -```yaml -devices: - 11111111111111111111: - mqtt: - topic: light - skip: - - 0100010801ff - values: - 0100010800ff: - mqtt: - topic: energy - transformations: - - round: 1 - filters: - - every: 3600 - 0100100700ff: - mqtt: - topic: power - filters: - - perc: 5 -``` +# Changelog +#### 2.0.0 (2023-03-22) +- Release rework diff --git a/requirements_setup.txt b/requirements_setup.txt index d0c6c6e..124c031 100644 --- a/requirements_setup.txt +++ b/requirements_setup.txt @@ -1,5 +1,5 @@ asyncio-mqtt == 0.16.1 pyserial-asyncio == 0.6 -easyconfig == 0.2.6 +easyconfig == 0.2.8 pydantic >= 1.10, <2.0 smllib == 1.2 diff --git a/src/sml2mqtt/__args__.py b/src/sml2mqtt/__args__.py index f228f95..6ea0721 100644 --- a/src/sml2mqtt/__args__.py +++ b/src/sml2mqtt/__args__.py @@ -25,7 +25,7 @@ def get_command_line_args(args=None) -> Type[CommandArgs]: parser.add_argument( '-a', '--analyze', - help='Processes exactly one sml message, shows the values of the message and what will be reported', + help='Process exactly one sml message, shows the values of the message and what will be reported', action='store_true', default=False ) @@ -61,13 +61,13 @@ def find_config_folder(config_file_str: Optional[str]) -> Path: return config_file else: config_file = Path(config_file_str).resolve() - # Add to check_path so we have a nice error message + # Add to check_path, so we have a nice error message check_path.append(config_file) if config_file.is_file(): return config_file - # folder exists but not the file -> create + # folder exists but not the file -> create file if config_file.parent.is_dir(): return config_file diff --git a/src/sml2mqtt/__init__.py b/src/sml2mqtt/__init__.py index 8285c04..9573cb7 100644 --- a/src/sml2mqtt/__init__.py +++ b/src/sml2mqtt/__init__.py @@ -8,8 +8,7 @@ # isort: split -from sml2mqtt import value -from sml2mqtt.process_value import process_value +from sml2mqtt import sml_value # isort: split diff --git a/src/sml2mqtt/__log__.py b/src/sml2mqtt/__log__.py index 214f946..10f5df6 100644 --- a/src/sml2mqtt/__log__.py +++ b/src/sml2mqtt/__log__.py @@ -20,7 +20,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.last_check: date = datetime.now().date() - def shouldRollover(self, record): + def shouldRollover(self, record): # noqa: N802 date = datetime.now().date() if date == self.last_check: return 0 diff --git a/src/sml2mqtt/__main__.py b/src/sml2mqtt/__main__.py index aa9ed42..e762728 100644 --- a/src/sml2mqtt/__main__.py +++ b/src/sml2mqtt/__main__.py @@ -4,27 +4,39 @@ import traceback import typing -from sml2mqtt.__args__ import get_command_line_args +from sml2mqtt import mqtt +from sml2mqtt.__args__ import CMD_ARGS, get_command_line_args from sml2mqtt.__log__ import log, setup_log -from sml2mqtt.__shutdown__ import get_return_code, setup_signal_handler, shutdown +from sml2mqtt.__shutdown__ import get_return_code, shutdown, signal_handler_setup from sml2mqtt.config import CONFIG from sml2mqtt.device import Device -from sml2mqtt.mqtt import BASE_TOPIC, connect async def a_main(): - await connect() - await asyncio.sleep(0.1) # Wait till mqtt is connected + devices = [] - # Create devices for port try: + if CMD_ARGS.analyze: + mqtt.patch_analyze() + else: + # initial mqtt connect + mqtt.start() + await mqtt.wait_for_connect(5) + + # Create devices for port for port_cfg in CONFIG.ports: - dev_mqtt = BASE_TOPIC.create_child(port_cfg.url) - await Device.create(port_cfg, port_cfg.timeout, set(), dev_mqtt) + dev_mqtt = mqtt.BASE_TOPIC.create_child(port_cfg.url) + device = await Device.create(port_cfg, port_cfg.timeout, set(), dev_mqtt) + devices.append(device) + + for device in devices: + device.start() except Exception as e: shutdown(e) + return await asyncio.gather(*devices, mqtt.wait_for_disconnect()) + def main() -> typing.Union[int, str]: try: @@ -34,35 +46,25 @@ def main() -> typing.Union[int, str]: return 7 # This is needed to make async-mqtt work + # see https://github.com/sbtinstruments/asyncio-mqtt if platform.system() == 'Windows': asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # Add possibility to stop program with Ctrl + c - setup_signal_handler() - loop = None + signal_handler_setup() try: setup_log() # setup mqtt base topic - BASE_TOPIC.cfg.topic_fragment = CONFIG.mqtt.topic - BASE_TOPIC.cfg.qos = CONFIG.mqtt.defaults.qos - BASE_TOPIC.cfg.retain = CONFIG.mqtt.defaults.retain - BASE_TOPIC.update() - - # setup loop - loop = asyncio.get_event_loop_policy().get_event_loop() - loop.create_task(a_main()) - loop.run_forever() + mqtt.setup_base_topic(CONFIG.mqtt.topic, CONFIG.mqtt.defaults.qos, CONFIG.mqtt.defaults.retain) + + asyncio.run(a_main()) except Exception as e: for line in traceback.format_exc().splitlines(): log.error(line) print(e) return str(e) - finally: - if loop is not None: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() return get_return_code() diff --git a/src/sml2mqtt/__shutdown__.py b/src/sml2mqtt/__shutdown__.py index 2754e59..216450c 100644 --- a/src/sml2mqtt/__shutdown__.py +++ b/src/sml2mqtt/__shutdown__.py @@ -1,11 +1,11 @@ import signal import traceback -from asyncio import create_task, get_event_loop, sleep, Task +from asyncio import create_task, Task from typing import Dict, Optional, Type, Union import sml2mqtt.mqtt from sml2mqtt.__log__ import log -from sml2mqtt.errors import AllDevicesFailed, DeviceSetupFailed +from sml2mqtt.errors import AllDevicesFailedError, DeviceSetupFailedError, InitialMqttConnectionFailedError # ---------------------------------------------------------------------------------------------------------------------- # Return code logic @@ -32,36 +32,47 @@ def get_return_code() -> int: # ---------------------------------------------------------------------------------------------------------------------- # Signal handlers so we can shutdown gracefully # ---------------------------------------------------------------------------------------------------------------------- -def shutdown_handler(sig, frame): +def _signal_handler_shutdown(sig, frame): set_return_code(0) - create_task(do_shutdown()) + do_shutdown() -def setup_signal_handler(): - signal.signal(signal.SIGINT, shutdown_handler) - signal.signal(signal.SIGTERM, shutdown_handler) +def signal_handler_setup(): + signal.signal(signal.SIGINT, _signal_handler_shutdown) + signal.signal(signal.SIGTERM, _signal_handler_shutdown) # ---------------------------------------------------------------------------------------------------------------------- # Actual shutdown logic # ---------------------------------------------------------------------------------------------------------------------- SHUTDOWN_TASK: Optional[Task] = None +SHUTDOWN_REQUESTED = False -async def do_shutdown(): +def do_shutdown(): + global SHUTDOWN_TASK, SHUTDOWN_REQUESTED + + if SHUTDOWN_REQUESTED: + return None + + if SHUTDOWN_TASK is None: + SHUTDOWN_TASK = create_task(_shutdown_task()) + + SHUTDOWN_REQUESTED = True + + +async def _shutdown_task(): global SHUTDOWN_TASK try: print('Shutting down ...') log.info('Shutting down ...') - for device in sml2mqtt.device.sml_device.ALL_DEVICES.values(): - device.shutdown() - - await sml2mqtt.mqtt.disconnect() - await sleep(0.1) + sml2mqtt.mqtt.cancel() - get_event_loop().stop() + # once all devices are stopped the main loop will exit + for device in sml2mqtt.device.sml_device.ALL_DEVICES.values(): + device.stop() finally: SHUTDOWN_TASK = None @@ -72,13 +83,18 @@ async def do_shutdown(): def shutdown(e: Union[Exception, Type[Exception]]): global SHUTDOWN_TASK - ret_map: Dict[int, Type[Exception]] = {10: DeviceSetupFailed, 20: AllDevicesFailed} + ret_map: Dict[int, Type[Exception]] = { + 10: DeviceSetupFailedError, + 11: InitialMqttConnectionFailedError, + 20: AllDevicesFailedError + } log_traceback = True # get return code based on the error - for ret_code, cls in ret_map.items(): + for ret_code, cls in ret_map.items(): # noqa: B007 if isinstance(e, cls): + log_traceback = False break if e is cls: @@ -93,5 +109,4 @@ def shutdown(e: Union[Exception, Type[Exception]]): set_return_code(ret_code) - if SHUTDOWN_TASK is None: - SHUTDOWN_TASK = create_task(do_shutdown()) + do_shutdown() diff --git a/src/sml2mqtt/__version__.py b/src/sml2mqtt/__version__.py index 923b987..afced14 100644 --- a/src/sml2mqtt/__version__.py +++ b/src/sml2mqtt/__version__.py @@ -1 +1 @@ -__version__ = '1.2.2' +__version__ = '2.0.0' diff --git a/src/sml2mqtt/config/config.py b/src/sml2mqtt/config/config.py index bd74b72..a1da81f 100644 --- a/src/sml2mqtt/config/config.py +++ b/src/sml2mqtt/config/config.py @@ -2,7 +2,7 @@ import serial from easyconfig import AppBaseModel, BaseModel, create_app_config -from pydantic import constr, Field, StrictFloat, StrictInt, validator +from pydantic import constr, Field, StrictFloat, StrictInt, StrictStr, validator from .device import REPUBLISH_ALIAS, SmlDeviceConfig, SmlValueConfig from .logging import LoggingSettings @@ -50,7 +50,6 @@ def _val_bytesize(cls, v): return v - class GeneralSettings(BaseModel): wh_in_kwh: bool = Field(True, description='Automatically convert Wh to kWh', alias='Wh in kWh') republish_after: int = Field( @@ -65,16 +64,16 @@ class GeneralSettings(BaseModel): False, description='Report the device id even though it does never change', alias='report device id', in_file=False ) - device_id_obis: str = Field( - '', description='Additional OBIS field for the serial number, default is 0100000009ff', + device_id_obis: List[StrictStr] = Field( + ['0100000009ff'], description='Additional OBIS fields for the serial number, default is 0100000009ff', alias='device id obis', in_file=False ) class Settings(AppBaseModel): - logging: LoggingSettings = LoggingSettings() - mqtt: MqttConfig = MqttConfig() - general: GeneralSettings = GeneralSettings() + logging: LoggingSettings = Field(default_factory=LoggingSettings) + mqtt: MqttConfig = Field(default_factory=MqttConfig) + general: GeneralSettings = Field(default_factory=GeneralSettings) ports: List[PortSettings] = [] devices: Dict[str, SmlDeviceConfig] = Field({}, description='Device configuration by ID or url',) diff --git a/src/sml2mqtt/config/device.py b/src/sml2mqtt/config/device.py index 5735240..4eb9fe3 100644 --- a/src/sml2mqtt/config/device.py +++ b/src/sml2mqtt/config/device.py @@ -14,15 +14,15 @@ class WorkaroundOptionEnum(str, Enum): class TransformOptionEnum(str, Enum): - factor = 'factor' - offset = 'offset' - round = 'round' + factor = 'factor' #: Use the value as a factor + offset = 'offset' #: Use the value as an offset + round = 'round' #: Round the result to the digits class FilterOptionEnum(str, Enum): - diff = 'diff' - perc = 'perc' - every = 'every' + diff = 'diff' #: Report when difference is greater equal than the value + perc = 'perc' #: Report when percentual change is greater equal the value + every = 'every' #: Report every x seconds TYPE_SML_VALUE_WORKAROUND_CFG = \ @@ -53,11 +53,18 @@ def len_1(cls, v): class SmlDeviceConfig(BaseModel): - mqtt: Optional[OptionalMqttPublishConfig] = OptionalMqttPublishConfig() - status: Optional[OptionalMqttPublishConfig] = OptionalMqttPublishConfig(topic='status') + """Configuration for a sml device""" + + mqtt: Optional[OptionalMqttPublishConfig] = Field( + default=None, description='Optional MQTT configuration for this meter.') + + status: Optional[OptionalMqttPublishConfig] = Field( + default=OptionalMqttPublishConfig(topic='status'), + description='Optional MQTT status topic configuration for this meter' + ) skip: Optional[Set[StrictStr]] = Field( default=None, description='OBIS codes (HEX) of values that will not be published (optional)') - values: Optional[Dict[str, SmlValueConfig]] = Field( - default=None, description='Special configurations for each of the values (optional)') + values: Dict[StrictStr, SmlValueConfig] = Field( + default={}, description='Special configurations for each of the values (optional)') diff --git a/src/sml2mqtt/config/mqtt.py b/src/sml2mqtt/config/mqtt.py index 493e344..bc6536d 100644 --- a/src/sml2mqtt/config/mqtt.py +++ b/src/sml2mqtt/config/mqtt.py @@ -14,10 +14,11 @@ class MqttDefaultPublishConfig(BaseModel): class OptionalMqttPublishConfig(BaseModel): - topic: TOPIC_STR = None - full_topic: TOPIC_STR = Field(None, alias='full topic') - qos: QOS = None - retain: StrictBool = None + topic: TOPIC_STR = Field(None, description='Topic fragment for building this topic with the parent topic') + full_topic: TOPIC_STR = Field( + None, alias='full topic', description='Full topic - will ignore the parent topic parts') + qos: QOS = Field(None, description='QoS for publishing this value (if set - otherwise use parent)') + retain: StrictBool = Field(None, description='Retain for publishing this value (if set - otherwise use parent)') @validator('topic', 'full_topic') def validate_topic(cls, value): @@ -51,8 +52,8 @@ class MqttConnection(BaseModel): class MqttConfig(BaseModel): - connection: MqttConnection = MqttConnection() + connection: MqttConnection = Field(default_factory=MqttConnection) topic: TOPIC_STR = Field('sml2mqtt', alias='topic prefix') - defaults: MqttDefaultPublishConfig = MqttDefaultPublishConfig() + defaults: MqttDefaultPublishConfig = Field(default_factory=MqttDefaultPublishConfig) last_will: OptionalMqttPublishConfig = Field( default_factory=lambda: OptionalMqttPublishConfig(topic='status'), alias='last will') diff --git a/src/sml2mqtt/device/sml_device.py b/src/sml2mqtt/device/sml_device.py index 3ae502e..f2c77df 100644 --- a/src/sml2mqtt/device/sml_device.py +++ b/src/sml2mqtt/device/sml_device.py @@ -1,4 +1,6 @@ +import logging import traceback +from asyncio import Event from binascii import b2a_hex from typing import Dict, Final, List, Optional, Set @@ -7,17 +9,20 @@ from smllib.sml import SmlListEntry import sml2mqtt -from sml2mqtt import process_value +from sml2mqtt import CMD_ARGS from sml2mqtt.__log__ import get_logger from sml2mqtt.__shutdown__ import shutdown from sml2mqtt.config import CONFIG from sml2mqtt.config.config import PortSettings -from sml2mqtt.config.device import SmlDeviceConfig +from sml2mqtt.config.device import SmlDeviceConfig, SmlValueConfig from sml2mqtt.device import DeviceStatus from sml2mqtt.device.watchdog import Watchdog -from sml2mqtt.errors import AllDevicesFailed, DeviceSetupFailed +from sml2mqtt.errors import AllDevicesFailedError, DeviceSetupFailedError, \ + ObisIdForConfigurationMappingNotFoundError, Sml2MqttConfigMappingError from sml2mqtt.mqtt import MqttObj -from sml2mqtt.process_value import VALUES as ALL_VALUES +from sml2mqtt.sml_value import SmlValue + +Event().set() ALL_DEVICES: Dict[str, 'Device'] = {} @@ -25,13 +30,19 @@ class Device: @classmethod async def create(cls, settings: PortSettings, timeout: float, skip_values: Set[str], mqtt_device: MqttObj): + device = None try: device = cls(settings.url, timeout, set(skip_values), mqtt_device) device.serial = await sml2mqtt.device.SmlSerial.create(settings, device) ALL_DEVICES[settings.url] = device + return device except Exception as e: - raise DeviceSetupFailed(e) from None + if device is None: + get_logger('device').error('Setup failed!') + else: + device.log.error('Setup failed') + raise DeviceSetupFailedError(e) from None def __init__(self, url: str, timeout: float, skip_values: Set[str], mqtt_device: MqttObj): self.stream = SmlStreamReader() @@ -47,10 +58,23 @@ def __init__(self, url: str, timeout: float, skip_values: Set[str], mqtt_device: self.device_url = url self.device_id: str = url.split("/")[-1] - self.device_id_set = False + + self.sml_values: Dict[str, SmlValue] = {} self.skip_values = skip_values + def start(self): + self.serial.start() + self.watchdog.start() + + def stop(self): + self.serial.cancel() + self.watchdog.cancel() + + def __await__(self): + yield from self.serial.wait_for_cancel().__await__() + yield from self.watchdog.wait_for_cancel().__await__() + def shutdown(self): if not self.status.is_shutdown_status(): self.set_status(DeviceStatus.SHUTDOWN) @@ -62,66 +86,96 @@ def set_status(self, new_status: DeviceStatus) -> bool: self.status = new_status self.log_status.info(f'{new_status:s}') - # Don't publish the port open because we don't have the correct name yet + # Don't publish the port open because we don't have the correct name from the config yet if new_status != DeviceStatus.PORT_OPENED: self.mqtt_status.publish(new_status.value) # If all ports are closed, or we have errors we shut down - if all(map(lambda x: x.status.is_shutdown_status(), ALL_DEVICES.values())): + if all(x.status.is_shutdown_status() for x in ALL_DEVICES.values()): # Stop reading from the serial port because we are shutting down self.serial.close() - shutdown(AllDevicesFailed) + self.watchdog.cancel() + shutdown(AllDevicesFailedError) return True - def set_device_id(self, serial: str): - assert isinstance(serial, str) - assert not self.device_id_set - self.device_id = serial - self.device_id_set = True + def _select_device_id(self, frame_values: Dict[str, SmlListEntry]) -> str: + # search frame and see if we get a match + for search_obis in CONFIG.general.device_id_obis: + if (obis_value := frame_values.get(search_obis)) is not None: + self.log.debug(f'Found obis id {search_obis:s} in the sml frame') + value = obis_value.get_value() + self.device_id = str(value) + return str(search_obis) + + searched = ', '.join(CONFIG.general.device_id_obis) + self.log.error(f'Found none of the following obis ids in the sml frame: {searched:s}') + raise ObisIdForConfigurationMappingNotFoundError() + + def _select_device_config(self) -> Optional[SmlDeviceConfig]: + device_cfg = CONFIG.devices.get(self.device_id) + if device_cfg is None: + self.log.warning(f'No configuration found for {self.device_id:s}') + return None - # config is optional - cfg: Optional[SmlDeviceConfig] = None - if CONFIG.devices is not None: - cfg = CONFIG.devices.get(serial) + self.log.debug(f'Configuration found for {self.device_id:s}') + return device_cfg - # Build the defaults but with the serial number instead of the url - if cfg is None: - self.mqtt_device.set_topic(serial) - return None + def _setup_device(self, frame_values: Dict[str, SmlListEntry]): + found_obis = self._select_device_id(frame_values) + cfg = self._select_device_config() + + # Global configuration option to ignore mapping value + if not CONFIG.general.report_device_id: + self.skip_values.add(found_obis) + + # Change the mqtt topic default from device url to the matched device id + self.mqtt_device.set_topic(self.device_id) - # config found -> load all configured values - self.mqtt_device.set_config(cfg.mqtt) - self.mqtt_status.set_config(cfg.status) - - if cfg.skip is not None: - self.skip_values.update(cfg.skip) - - def select_device_id(self, frame_values: Dict[str, SmlListEntry]): - obis_ids = ['0100000009ff'] - if CONFIG.general.device_id_obis: - obis_ids.append(CONFIG.general.device_id_obis) - - for obis_id in obis_ids: - entry = frame_values.pop(obis_id, None) - if entry is not None: - if not CONFIG.general.report_device_id: - self.skip_values.add(obis_id) - self.set_device_id(entry.get_value()) - break - else: - self.set_device_id(self.device_url) - - # remove additionally skipped frames - # this gets filled in set_device_id, so we have to do it afterwards! - for name in self.skip_values: - frame_values.pop(name, None) + # override from config + if cfg is not None: + # setup topics + self.mqtt_device.set_config(cfg.mqtt) + self.mqtt_status.set_config(cfg.status) + + # additional obis values that are ignored from the config + if cfg.skip is not None: + self.skip_values.update(cfg.skip) + + self._setup_sml_values(cfg, frame_values) + + def _setup_sml_values(self, device_config: Optional[SmlDeviceConfig], frame_values: Dict[str, SmlListEntry]): + log_level = logging.DEBUG if not CMD_ARGS.analyze else logging.INFO + values_config: Dict[str, SmlValueConfig] = device_config.values if device_config is not None else {} + + for obis_id in frame_values: + if obis_id in self.skip_values: + continue + + value_config = values_config.get(obis_id) + + if device_config is None or value_config is None: + self.log.log(log_level, f'Creating default value handler for {obis_id}') + value = SmlValue( + self.device_id, obis_id, self.mqtt_device.create_child(obis_id), + workarounds=[], transformations=[], filters=sml2mqtt.sml_value.filter_from_config(None) + ) + else: + self.log.log(log_level, f'Creating value handler from config for {obis_id}') + value = SmlValue( + self.device_id, obis_id, self.mqtt_device.create_child(obis_id).set_config(value_config.mqtt), + workarounds=sml2mqtt.sml_value.workaround_from_config(value_config.workarounds), + transformations=sml2mqtt.sml_value.transform_from_config(value_config.transformations), + filters=sml2mqtt.sml_value.filter_from_config(value_config.filters), + ) + + self.sml_values[obis_id] = value def serial_data_timeout(self): if self.set_status(DeviceStatus.MSG_TIMEOUT): self.stream.clear() self.log.warning('Timeout') - async def serial_data_read(self, data: bytes): + def serial_data_read(self, data: bytes): frame = None try: @@ -138,22 +192,25 @@ async def serial_data_read(self, data: bytes): return None # Process Frame - await self.process_frame(frame) - except Exception: + self.process_frame(frame) + except Exception as e: # dump frame if possible if frame is not None: self.log.error('Received Frame') self.log.error(f' -> {b2a_hex(frame.buffer)}') # Log exception - for line in traceback.format_exc().splitlines(): - self.log.error(line) + if isinstance(e, Sml2MqttConfigMappingError): + self.log.error(str(e)) + else: + for line in traceback.format_exc().splitlines(): + self.log.error(line) # Signal that an error occurred self.set_status(DeviceStatus.ERROR) return None - async def process_frame(self, frame: SmlFrame): + def process_frame(self, frame: SmlFrame): do_analyze = sml2mqtt.CMD_ARGS.analyze do_wh_in_kwh = CONFIG.general.wh_in_kwh @@ -200,20 +257,20 @@ async def process_frame(self, frame: SmlFrame): # Mark for publishing frame_values[name] = sml_obj - # We overwrite the device_id url (default) with the serial number if the device reports it - # Otherwise we still do the config lookup so the user can configure the mqtt topics - if not self.device_id_set: - self.select_device_id(frame_values) + # If we don't have the values we have to set up the device first + if not self.sml_values: + self._setup_device(frame_values) + for drop_obis in self.skip_values: + frame_values.pop(drop_obis, None) - # Process all values - for obis_value in frame_values.values(): - process_value(self, obis_value, frame_values) + for obis_id, frame_value in frame_values.items(): + self.sml_values[obis_id].set_value(frame_value, frame_values) # There was no Error -> OK self.set_status(DeviceStatus.OK) if do_analyze: - for obis, value in ALL_VALUES.get(self.device_id, {}).items(): + for value in self.sml_values.values(): self.log.info('') for line in value.describe().splitlines(): self.log.info(line) diff --git a/src/sml2mqtt/device/sml_device_status.py b/src/sml2mqtt/device/sml_device_status.py index 7e73a35..ff10c36 100644 --- a/src/sml2mqtt/device/sml_device_status.py +++ b/src/sml2mqtt/device/sml_device_status.py @@ -13,3 +13,6 @@ class DeviceStatus(str, Enum): def is_shutdown_status(self) -> bool: return self.value in (DeviceStatus.PORT_CLOSED, DeviceStatus.ERROR, DeviceStatus.SHUTDOWN) + + def __str__(self) -> str: + return self.value diff --git a/src/sml2mqtt/device/sml_serial.py b/src/sml2mqtt/device/sml_serial.py index 0833779..eafd11c 100644 --- a/src/sml2mqtt/device/sml_serial.py +++ b/src/sml2mqtt/device/sml_serial.py @@ -1,6 +1,6 @@ import asyncio -from asyncio import create_task, Task -from typing import Awaitable, Callable, Final, Optional, TYPE_CHECKING +from asyncio import CancelledError, create_task, Task +from typing import Optional, TYPE_CHECKING from serial_asyncio import create_serial_connection, SerialTransport @@ -36,9 +36,7 @@ def __init__(self) -> None: self.transport: Optional[SerialTransport] = None - self.pause_task: Optional[Task] = None - - self.on_data_cb: Final = Callable[[bytes], Awaitable] + self.task: Optional[Task] = None def connection_made(self, transport): self.transport = transport @@ -46,31 +44,44 @@ def connection_made(self, transport): self.device.set_status(DeviceStatus.PORT_OPENED) + def connection_lost(self, exc): + self.close() + + log.info(f'Port {self.url} was closed') + self.device.set_status(DeviceStatus.PORT_CLOSED) + def data_received(self, data: bytes): - self.pause_serial() - create_task(self.device.serial_data_read(data)) + self.transport.pause_reading() + self.device.serial_data_read(data) - async def resume_serial(self): - try: - await asyncio.sleep(0.4) + async def _chunk_task(self): + while True: + await asyncio.sleep(0.2) self.transport.resume_reading() - finally: - self.pause_task = None - def pause_serial(self): - self.transport.pause_reading() - self.pause_task = create_task(self.resume_serial()) + def start(self): + assert self.task is None + self.task = create_task(self._chunk_task(), name=f'Chunk task {self.url:s}') - def connection_lost(self, exc): + def cancel(self): self.close() - log.info(f'Port {self.url} was closed') - self.device.set_status(DeviceStatus.PORT_CLOSED) + async def wait_for_cancel(self): + if self.task is None: + return False + try: + await self.task + except CancelledError: + pass + return True def close(self): - if self.pause_task is not None: - self.pause_task.cancel() - self.pause_task = None - if not self.transport.is_closing(): self.transport.close() + + if (task := self.task) is None: + return None + + task.cancel() + self.task = None + return task diff --git a/src/sml2mqtt/device/sml_value_group.py b/src/sml2mqtt/device/sml_value_group.py new file mode 100644 index 0000000..8ede679 --- /dev/null +++ b/src/sml2mqtt/device/sml_value_group.py @@ -0,0 +1,19 @@ +from typing import Dict, Final + +from smllib.sml import SmlListEntry + +from sml2mqtt.sml_value import SmlValue + + +class SmlValueGroup: + def __init__(self, name: str): + self.name: Final = name + self.values: Dict[str, SmlValue] = {} + + def process_frame(self, frame_values: Dict[str, SmlListEntry]): + for obis, frame_value in frame_values.items(): + value = self.values[obis] + value.set_value(frame_value, frame_values) + + def __str__(self) -> str: + return f'<{self.__class__.__name__} {self.name:s}>' diff --git a/src/sml2mqtt/device/watchdog.py b/src/sml2mqtt/device/watchdog.py index e3018fd..98cd4d5 100644 --- a/src/sml2mqtt/device/watchdog.py +++ b/src/sml2mqtt/device/watchdog.py @@ -1,4 +1,4 @@ -from asyncio import create_task, Event, Task, TimeoutError, wait_for +from asyncio import CancelledError, create_task, Event, Task, TimeoutError, wait_for from typing import Any, Callable, Final, Optional @@ -12,14 +12,23 @@ def __init__(self, timeout: float, callback: Callable[[], Any]): self.task: Optional[Task] = None def start(self): - if self.task is None: - self.task = create_task(self.wd_task()) + assert self.task is None + self.task = create_task(self.wd_task()) def cancel(self): if self.task is not None: self.task.cancel() self.task = None + async def wait_for_cancel(self): + if self.task is None: + return False + try: + await self.task + except CancelledError: + pass + return True + def feed(self): self.event.set() diff --git a/src/sml2mqtt/errors.py b/src/sml2mqtt/errors.py index e864713..0b1241c 100644 --- a/src/sml2mqtt/errors.py +++ b/src/sml2mqtt/errors.py @@ -1,10 +1,32 @@ -class Sml2MqttException(Exception): +class Sml2MqttException(Exception): # noqa: N818 pass -class DeviceSetupFailed(Sml2MqttException): +class AllDevicesFailedError(Sml2MqttException): pass -class AllDevicesFailed(Sml2MqttException): +# ------------------------------------------------------------------------------------ +# Initial setup failed +# ------------------------------------------------------------------------------------ +class InitialSetupFailedError(Sml2MqttException): + pass + + +class DeviceSetupFailedError(InitialSetupFailedError): + pass + + +class InitialMqttConnectionFailedError(InitialSetupFailedError): + pass + + +# ------------------------------------------------------------------------------------ +# Config mapping errors +# ------------------------------------------------------------------------------------ +class Sml2MqttConfigMappingError(Sml2MqttException): + pass + + +class ObisIdForConfigurationMappingNotFoundError(Sml2MqttConfigMappingError): pass diff --git a/src/sml2mqtt/mqtt/__init__.py b/src/sml2mqtt/mqtt/__init__.py index 17d495c..b84870a 100644 --- a/src/sml2mqtt/mqtt/__init__.py +++ b/src/sml2mqtt/mqtt/__init__.py @@ -1,6 +1,6 @@ from .connect_delay import DynDelay -from .mqtt import connect, disconnect, publish +from .mqtt import cancel, publish, start, wait_for_connect, wait_for_disconnect # isort: split -from .mqtt_obj import BASE_TOPIC, MqttObj +from .mqtt_obj import BASE_TOPIC, MqttObj, patch_analyze, setup_base_topic diff --git a/src/sml2mqtt/mqtt/errors.py b/src/sml2mqtt/mqtt/errors.py new file mode 100644 index 0000000..8ad7fda --- /dev/null +++ b/src/sml2mqtt/mqtt/errors.py @@ -0,0 +1,10 @@ +class MqttError(Exception): + pass + + +class TopicFragmentExpectedError(Exception): + pass + + +class MqttConfigValuesMissingError(Exception): + pass diff --git a/src/sml2mqtt/mqtt/mqtt.py b/src/sml2mqtt/mqtt/mqtt.py index 383d435..6a4983f 100644 --- a/src/sml2mqtt/mqtt/mqtt.py +++ b/src/sml2mqtt/mqtt/mqtt.py @@ -1,124 +1,134 @@ -import time import traceback -from asyncio import create_task, Future -from typing import Optional, Union +from asyncio import CancelledError, create_task, Event, Queue, Task, TimeoutError, wait_for +from typing import Final, Optional, Union from asyncio_mqtt import Client, MqttError, Will import sml2mqtt from sml2mqtt.__log__ import log as _parent_logger - -# from sml2mqtt.config import CONFIG +from sml2mqtt.errors import InitialMqttConnectionFailedError from sml2mqtt.mqtt import DynDelay log = _parent_logger.getChild('mqtt') -TIME_BEFORE_RECONNECT = 15 -DELAY_CONNECT = DynDelay(0, 180) -MQTT: Optional[Client] = None -TASK_CONNECT: Optional[Future] = None +TASK: Optional[Task] = None +IS_CONNECTED: Optional[Event] = None + +def start(): + global TASK, IS_CONNECTED -async def disconnect(): - global TASK_CONNECT, MQTT + IS_CONNECTED = Event() - if TASK_CONNECT is not None: - TASK_CONNECT.cancel() - TASK_CONNECT = None + assert TASK is None + TASK = create_task(mqtt_task(), name='MQTT Task') - if MQTT is not None: - mqtt = MQTT - MQTT = None - if mqtt._client.is_connected(): - await mqtt.disconnect() +def cancel(): + global TASK + if TASK is not None: + TASK.cancel() + TASK = None -async def connect(): - global TASK_CONNECT - if TASK_CONNECT is None: - TASK_CONNECT = create_task(_connect_task()) +async def wait_for_connect(timeout: float): + if IS_CONNECTED is None: + return None -async def _connect_task(): - global TASK_CONNECT try: - await _connect_to_broker() - finally: - TASK_CONNECT = None + await wait_for(IS_CONNECTED.wait(), timeout) + except TimeoutError: + log.error('Initial mqtt connection failed!') + raise InitialMqttConnectionFailedError() from None + + return None + + +async def wait_for_disconnect(): + if TASK is None: + return None + + await TASK + + +QUEUE: Optional[Queue] = None -async def _connect_to_broker(): - global MQTT +async def mqtt_task(): + try: + await _mqtt_task() + finally: + log.debug('Task finished') + - # # We don't publish anything if we just analyze the data from the reader - # if sml2mqtt._args.ARGS.analyze: - # return None +async def _mqtt_task(): + global QUEUE + from .mqtt_obj import BASE_TOPIC config = sml2mqtt.config.CONFIG - while True: + cfg_connection = config.mqtt.connection + + delay = DynDelay(0, 300) + + payload_offline: Final = 'OFFLINE' + payload_online: Final = 'ONLINE' + + shutdown = False + + while not shutdown: + await delay.wait() + try: - async with DELAY_CONNECT: - # If we are already connected we try to disconnect before we reconnect - try: - if MQTT is not None: - if MQTT._client.is_connected(): - await MQTT.disconnect() - MQTT = None - except Exception as e: - log.error(f'Error while disconnecting: {e}') - - # since we just pass this into the mqtt wrapper we do not link it to the base topic - will_topic = sml2mqtt.mqtt.MqttObj( - config.mqtt.topic, config.mqtt.defaults.qos, config.mqtt.defaults.retain - ).update().create_child(config.mqtt.last_will) - will_topic.set_config(config.mqtt.last_will) - - MQTT = Client( - hostname=config.mqtt.connection.host, - port=config.mqtt.connection.port, - username=config.mqtt.connection.user if config.mqtt.connection.user else None, - password=config.mqtt.connection.password if config.mqtt.connection.password else None, - will=Will(will_topic.topic, payload='OFFLINE', qos=will_topic.qos, retain=will_topic.retain) - ) - - log.debug(f'Connecting to {config.mqtt.connection.host}:{config.mqtt.connection.port}') - await MQTT.connect() + # since we just pass this into the mqtt wrapper we do not link it to the base topic + will_topic = BASE_TOPIC.create_child( + topic_fragment=config.mqtt.last_will.topic).set_config(config.mqtt.last_will) + + client = Client( + hostname=cfg_connection.host, + port=cfg_connection.port, + username=cfg_connection.user if cfg_connection.user else None, + password=cfg_connection.password if cfg_connection.password else None, + will=Will(will_topic.topic, payload=payload_offline, qos=will_topic.qos, retain=will_topic.retain), + client_id=cfg_connection.client_id + ) + + log.debug(f'Connecting to {cfg_connection.host}:{cfg_connection.port}') + + async with client: log.debug('Success!') + delay.reset() + QUEUE = Queue() + IS_CONNECTED.set() - # signal that we are online - will_topic.publish('ONLINE') - break + try: + # signal that we are online + await client.publish(will_topic.topic, payload_online, will_topic.qos, will_topic.retain) + + # worker to publish things + while True: + topic, value, qos, retain = await QUEUE.get() + await client.publish(topic, value, qos, retain) + QUEUE.task_done() + except CancelledError: + # The last will testament only gets sent on abnormal disconnect + # Since we disconnect gracefully we have to manually sent the offline status + await client.publish(will_topic.topic, payload_offline, will_topic.qos, will_topic.retain) + shutdown = True except MqttError as e: + delay.increase() log.error(f'{e} ({e.__class__.__name__})') except Exception: + delay.increase() for line in traceback.format_exc().splitlines(): log.error(line) - return None + finally: + QUEUE = None + IS_CONNECTED.clear() -PUBS_FAILED_SINCE: Optional[float] = None - - -async def publish(topic: str, value: Union[int, float, str], qos: int, retain: bool): - global PUBS_FAILED_SINCE - - if MQTT is None or not MQTT._client.is_connected(): - await connect() - return None - - # publish message - try: - await MQTT.publish(topic, value, qos=qos, retain=retain) - PUBS_FAILED_SINCE = None - except MqttError as e: - log.error(f'Error while publishing to {topic}: {e} ({e.__class__.__name__})') - - # If we fail too often we try to reconnect - if PUBS_FAILED_SINCE is None: - PUBS_FAILED_SINCE = time.time() - else: - if time.time() - PUBS_FAILED_SINCE >= TIME_BEFORE_RECONNECT: - await connect() +def publish(topic: str, value: Union[int, float, str], qos: int, retain: bool): + if QUEUE is not None: + QUEUE.put_nowait((topic, value, qos, retain)) diff --git a/src/sml2mqtt/mqtt/mqtt_obj.py b/src/sml2mqtt/mqtt/mqtt_obj.py index 6f622d2..2932060 100644 --- a/src/sml2mqtt/mqtt/mqtt_obj.py +++ b/src/sml2mqtt/mqtt/mqtt_obj.py @@ -1,19 +1,17 @@ import dataclasses -from asyncio import create_task -from typing import List, Optional, Union +from typing import Any, Callable, Final, List, Optional, Union -from sml2mqtt import CMD_ARGS from sml2mqtt.__log__ import get_logger from sml2mqtt.config import OptionalMqttPublishConfig from sml2mqtt.mqtt import publish +from .errors import MqttConfigValuesMissingError, TopicFragmentExpectedError -class TopicFragmentExpected(Exception): - pass +pub_func: Callable[[str, Union[int, float, str], int, bool], Any] = publish -class MqttConfigValuesMissing(Exception): - pass +def publish_analyze(topic: str, value: Union[int, float, str], qos: int, retain: bool): + get_logger('mqtt.pub').info(f'{topic}: {value} (QOS: {qos}, retain: {retain})') @dataclasses.dataclass @@ -23,14 +21,11 @@ class MqttCfg: qos: Optional[int] = None retain: Optional[bool] = None - def is_full_config(self) -> bool: - if self.topic_fragment is None and self.topic_full is None: - return False - if self.qos is None: - return False - if self.retain is None: - return False - return True + def set_config(self, config: Optional[OptionalMqttPublishConfig]): + self.topic_full = config.full_topic + self.topic_fragment = config.topic + self.qos = config.qos + self.retain = config.retain class MqttObj: @@ -48,11 +43,7 @@ def __init__(self, topic_fragment: Optional[str] = None, qos: Optional[int] = No self.children: List[MqttObj] = [] def publish(self, value: Union[str, int, float, bytes]): - # do not publish when the analyze flag is set - if CMD_ARGS.analyze: - get_logger('mqtt.pub').info(f'{self.topic}: {value} (QOS: {self.qos}, retain: {self.retain})') - else: - create_task(publish(self.topic, value, self.qos, self.retain)) + pub_func(self.topic, value, self.qos, self.retain) def update(self) -> 'MqttObj': self._merge_values() @@ -62,24 +53,30 @@ def update(self) -> 'MqttObj': def _merge_values(self) -> 'MqttObj': + # no parent -> just set the config if self.parent is None: - if not self.cfg.is_full_config(): - raise MqttConfigValuesMissing() + assert self.cfg.topic_full is None + self.topic = self.cfg.topic_fragment # expect fragment only + self.qos = self.cfg.qos + self.retain = self.cfg.retain + if self.topic is None or self.qos is None or self.retain is None: + raise MqttConfigValuesMissingError() + return self + # effective topic if self.cfg.topic_full: self.topic = self.cfg.topic_full else: if not self.cfg.topic_fragment: - raise TopicFragmentExpected() - if self.parent is None: - self.topic = self.cfg.topic_fragment - else: - self.topic = f'{self.parent.topic}/{self.cfg.topic_fragment}' + raise TopicFragmentExpectedError() + self.topic = f'{self.parent.topic}/{self.cfg.topic_fragment}' + # effective QOS self.qos = self.cfg.qos if self.qos is None: self.qos = self.parent.qos + # effective retain self.retain = self.cfg.retain if self.retain is None: self.retain = self.parent.retain @@ -94,11 +91,7 @@ def set_config(self, cfg: Optional[OptionalMqttPublishConfig]) -> 'MqttObj': if cfg is None: return self - local = self.cfg - local.topic_full = cfg.full_topic - local.topic_fragment = cfg.topic - local.qos = cfg.qos - local.retain = cfg.retain + self.cfg.set_config(cfg) self.update() return self @@ -111,4 +104,17 @@ def create_child(self, topic_fragment: Optional[str] = None, qos: Optional[int] return child -BASE_TOPIC = MqttObj() +BASE_TOPIC: Final = MqttObj() + + +def setup_base_topic(topic: str, qos: int, retain: bool): + BASE_TOPIC.cfg.topic_fragment = topic + BASE_TOPIC.cfg.qos = qos + BASE_TOPIC.cfg.retain = retain + BASE_TOPIC.update() + + +def patch_analyze(): + global pub_func + + pub_func = publish_analyze diff --git a/src/sml2mqtt/process_value.py b/src/sml2mqtt/process_value.py deleted file mode 100644 index fec37dc..0000000 --- a/src/sml2mqtt/process_value.py +++ /dev/null @@ -1,63 +0,0 @@ -import logging -from typing import Dict - -from smllib.sml import SmlListEntry - -import sml2mqtt -from sml2mqtt.__args__ import CMD_ARGS -from sml2mqtt.__log__ import get_logger -from sml2mqtt.value import SmlValue - -VALUES: Dict[str, Dict[str, SmlValue]] = {} - - -values_log = get_logger('values') - - -def process_value(device: 'sml2mqtt.device.Device', obj: SmlListEntry, frame_values: Dict[str, SmlListEntry]): - try: - device_values = VALUES[device.device_id] - except KeyError: - VALUES[device.device_id] = device_values = {} - - try: - value = device_values[str(obj.obis)] - except KeyError: - device_values[str(obj.obis)] = value = create_sml_value(device, obj) - - value.set_value(obj, frame_values) - - -def create_sml_value(device: 'sml2mqtt.device.Device', obj: SmlListEntry) -> SmlValue: - obis = str(obj.obis) - device_id = device.device_id - log_level = logging.DEBUG if not CMD_ARGS.analyze else logging.INFO - - cfg = None - if sml2mqtt.CONFIG.devices is not None: - cfg = sml2mqtt.CONFIG.devices.get(device_id) - - def create_default_value(): - values_log.log(log_level, f'Creating default value handler for {device_id}/{obis}') - return SmlValue( - device_id, obis, device.mqtt_device.create_child(obis), - workarounds=[], - transformations=[], - filters=sml2mqtt.value.filter_from_config(None) - ) - - if cfg is None or cfg.values is None: - return create_default_value() - - value_cfg = cfg.values.get(obis) - if value_cfg is None: - return create_default_value() - - values_log.log(log_level, f'Creating value handler from config for {device_id}/{obis}') - - return SmlValue( - device_id, obis, device.mqtt_device.create_child(obis).set_config(value_cfg.mqtt), - workarounds=sml2mqtt.value.workaround_from_config(value_cfg.workarounds), - transformations=sml2mqtt.value.transform_from_config(value_cfg.transformations), - filters=sml2mqtt.value.filter_from_config(value_cfg.filters), - ) diff --git a/src/sml2mqtt/sml_value/__init__.py b/src/sml2mqtt/sml_value/__init__.py new file mode 100644 index 0000000..891458c --- /dev/null +++ b/src/sml2mqtt/sml_value/__init__.py @@ -0,0 +1,6 @@ +from . import __types__, filter + +# isort: split + +from sml2mqtt.sml_value.enum_builder import filter_from_config, transform_from_config, workaround_from_config +from sml2mqtt.sml_value.sml_value import SmlValue diff --git a/src/sml2mqtt/value/__types__.py b/src/sml2mqtt/sml_value/__types__.py similarity index 100% rename from src/sml2mqtt/value/__types__.py rename to src/sml2mqtt/sml_value/__types__.py diff --git a/src/sml2mqtt/value/enum_builder.py b/src/sml2mqtt/sml_value/enum_builder.py similarity index 82% rename from src/sml2mqtt/value/enum_builder.py rename to src/sml2mqtt/sml_value/enum_builder.py index 817e4a2..1f354ef 100644 --- a/src/sml2mqtt/value/enum_builder.py +++ b/src/sml2mqtt/sml_value/enum_builder.py @@ -3,10 +3,11 @@ import sml2mqtt from sml2mqtt.config.device import FilterOptionEnum, TransformOptionEnum, TYPE_SML_VALUE_FILTER_CFG, \ TYPE_SML_VALUE_TRANSFORM_CFG, TYPE_SML_VALUE_WORKAROUND_CFG, WorkaroundOptionEnum -from sml2mqtt.value.filter import ChangeFilter, DiffAbsFilter, DiffFilterBase, DiffPercFilter, FILTER_OBJ, RefreshEvery -from sml2mqtt.value.transformations import FactorTransformation, \ - OffsetTransformation, RoundTransformation, TRANSFORM_OBJ -from sml2mqtt.value.workarounds import NegativeOnEnergyMeterStatus, WORKAROUND_OBJ +from sml2mqtt.sml_value.filter import ChangeFilter, DiffAbsFilter, \ + DiffFilterBase, DiffPercFilter, FilterBase, RefreshEvery +from sml2mqtt.sml_value.transformations import FactorTransformation, \ + OffsetTransformation, RoundTransformation, TransformationBase +from sml2mqtt.sml_value.workarounds import NegativeOnEnergyMeterStatus, WorkaroundBase TYPE_A = TypeVar('TYPE_A') @@ -28,7 +29,7 @@ def _from_config(cfg: Union[TYPE_SML_VALUE_FILTER_CFG, TYPE_SML_VALUE_TRANSFORM_ return ret -def filter_from_config(cfg: TYPE_SML_VALUE_FILTER_CFG) -> List[FILTER_OBJ]: +def filter_from_config(cfg: TYPE_SML_VALUE_FILTER_CFG) -> List[FilterBase]: class_dict = { FilterOptionEnum.diff: DiffAbsFilter, FilterOptionEnum.perc: DiffPercFilter, @@ -52,7 +53,7 @@ def filter_from_config(cfg: TYPE_SML_VALUE_FILTER_CFG) -> List[FILTER_OBJ]: return filters -def transform_from_config(cfg: TYPE_SML_VALUE_TRANSFORM_CFG) -> List[TRANSFORM_OBJ]: +def transform_from_config(cfg: TYPE_SML_VALUE_TRANSFORM_CFG) -> List[TransformationBase]: class_dict = { TransformOptionEnum.factor: FactorTransformation, TransformOptionEnum.round: RoundTransformation, @@ -61,7 +62,7 @@ def transform_from_config(cfg: TYPE_SML_VALUE_TRANSFORM_CFG) -> List[TRANSFORM_O return _from_config(cfg, class_dict) -def workaround_from_config(cfg: TYPE_SML_VALUE_WORKAROUND_CFG) -> List[WORKAROUND_OBJ]: +def workaround_from_config(cfg: TYPE_SML_VALUE_WORKAROUND_CFG) -> List[WorkaroundBase]: class_dict = { WorkaroundOptionEnum.negative_on_energy_meter_status: NegativeOnEnergyMeterStatus, } diff --git a/src/sml2mqtt/sml_value/filter/__init__.py b/src/sml2mqtt/sml_value/filter/__init__.py new file mode 100644 index 0000000..8febfa5 --- /dev/null +++ b/src/sml2mqtt/sml_value/filter/__init__.py @@ -0,0 +1,7 @@ +from sml2mqtt.sml_value.filter.base import FilterBase + +# isort: split + +from sml2mqtt.sml_value.filter.change import ChangeFilter +from sml2mqtt.sml_value.filter.diff import DiffAbsFilter, DiffFilterBase, DiffPercFilter +from sml2mqtt.sml_value.filter.time import RefreshEvery diff --git a/src/sml2mqtt/value/filter/base.py b/src/sml2mqtt/sml_value/filter/base.py similarity index 57% rename from src/sml2mqtt/value/filter/base.py rename to src/sml2mqtt/sml_value/filter/base.py index c3fd74a..a01b8d4 100644 --- a/src/sml2mqtt/value/filter/base.py +++ b/src/sml2mqtt/sml_value/filter/base.py @@ -1,6 +1,4 @@ -from typing import TypeVar - -from sml2mqtt.value.__types__ import VALUE_TYPE +from sml2mqtt.sml_value.__types__ import VALUE_TYPE class FilterBase: @@ -9,6 +7,3 @@ def required(self, value: VALUE_TYPE) -> VALUE_TYPE: def done(self, value): raise NotImplementedError() - - -FILTER_OBJ = TypeVar('FILTER_OBJ', bound=FilterBase) diff --git a/src/sml2mqtt/value/filter/change.py b/src/sml2mqtt/sml_value/filter/change.py similarity index 100% rename from src/sml2mqtt/value/filter/change.py rename to src/sml2mqtt/sml_value/filter/change.py diff --git a/src/sml2mqtt/value/filter/diff.py b/src/sml2mqtt/sml_value/filter/diff.py similarity index 100% rename from src/sml2mqtt/value/filter/diff.py rename to src/sml2mqtt/sml_value/filter/diff.py diff --git a/src/sml2mqtt/value/filter/time.py b/src/sml2mqtt/sml_value/filter/time.py similarity index 100% rename from src/sml2mqtt/value/filter/time.py rename to src/sml2mqtt/sml_value/filter/time.py diff --git a/src/sml2mqtt/value/smlvalue.py b/src/sml2mqtt/sml_value/sml_value.py similarity index 86% rename from src/sml2mqtt/value/smlvalue.py rename to src/sml2mqtt/sml_value/sml_value.py index f7b363b..31d192d 100644 --- a/src/sml2mqtt/value/smlvalue.py +++ b/src/sml2mqtt/sml_value/sml_value.py @@ -3,28 +3,28 @@ from smllib.sml import SmlListEntry from sml2mqtt.mqtt import MqttObj -from sml2mqtt.value.filter import FILTER_OBJ -from sml2mqtt.value.transformations import TRANSFORM_OBJ -from sml2mqtt.value.workarounds import WORKAROUND_OBJ +from sml2mqtt.sml_value.filter import FilterBase +from sml2mqtt.sml_value.transformations import TransformationBase +from sml2mqtt.sml_value.workarounds import WorkaroundBase class SmlValue: def __init__(self, device: str, obis: str, mqtt: MqttObj, - workarounds: Iterable[WORKAROUND_OBJ], - transformations: Iterable[TRANSFORM_OBJ], - filters: Iterable[FILTER_OBJ]): + workarounds: Iterable[WorkaroundBase], + transformations: Iterable[TransformationBase], + filters: Iterable[FilterBase]): self.device_id: Final = device self.obis: Final = obis self.mqtt: Final = mqtt - self.sml_value: Optional[SmlListEntry] = None - self.last_value: Union[None, int, float, str] = None - self.workarounds: Final = workarounds self.transformations: Final = transformations self.filters: Final = filters + self.sml_value: Optional[SmlListEntry] = None + self.last_value: Union[None, int, float, str] = None + def set_value(self, sml_value: Optional[SmlListEntry], frame_values: Dict[str, SmlListEntry]): self.sml_value = sml_value diff --git a/src/sml2mqtt/sml_value/transformations/__init__.py b/src/sml2mqtt/sml_value/transformations/__init__.py new file mode 100644 index 0000000..cb5ccab --- /dev/null +++ b/src/sml2mqtt/sml_value/transformations/__init__.py @@ -0,0 +1,5 @@ +from sml2mqtt.sml_value.transformations.base import TransformationBase + +# isort: split + +from sml2mqtt.sml_value.transformations.math import FactorTransformation, OffsetTransformation, RoundTransformation diff --git a/src/sml2mqtt/sml_value/transformations/base.py b/src/sml2mqtt/sml_value/transformations/base.py new file mode 100644 index 0000000..f5d89fa --- /dev/null +++ b/src/sml2mqtt/sml_value/transformations/base.py @@ -0,0 +1,6 @@ +from sml2mqtt.sml_value.__types__ import VALUE_TYPE + + +class TransformationBase: + def process(self, value: VALUE_TYPE) -> VALUE_TYPE: + raise NotImplementedError() diff --git a/src/sml2mqtt/value/transformations/math.py b/src/sml2mqtt/sml_value/transformations/math.py similarity index 100% rename from src/sml2mqtt/value/transformations/math.py rename to src/sml2mqtt/sml_value/transformations/math.py diff --git a/src/sml2mqtt/sml_value/workarounds/__init__.py b/src/sml2mqtt/sml_value/workarounds/__init__.py new file mode 100644 index 0000000..41e9ebf --- /dev/null +++ b/src/sml2mqtt/sml_value/workarounds/__init__.py @@ -0,0 +1,5 @@ +from sml2mqtt.sml_value.workarounds.base import WorkaroundBase + +# isort: split + +from sml2mqtt.sml_value.workarounds.negative_on_energy_status import NegativeOnEnergyMeterStatus diff --git a/src/sml2mqtt/value/workarounds/base.py b/src/sml2mqtt/sml_value/workarounds/base.py similarity index 72% rename from src/sml2mqtt/value/workarounds/base.py rename to src/sml2mqtt/sml_value/workarounds/base.py index b9b32ab..544c15e 100644 --- a/src/sml2mqtt/value/workarounds/base.py +++ b/src/sml2mqtt/sml_value/workarounds/base.py @@ -1,8 +1,8 @@ -from typing import Dict, TypeVar +from typing import Dict from smllib.sml import SmlListEntry -from sml2mqtt.value.__types__ import WORKAROUND_TYPE +from sml2mqtt.sml_value.__types__ import WORKAROUND_TYPE class WorkaroundBase: @@ -16,6 +16,3 @@ def fix(self, value: SmlListEntry, frame_values: Dict[str, SmlListEntry]) -> Sml def __repr__(self): return f'<{self.__class__.__name__}>' - - -WORKAROUND_OBJ = TypeVar('WORKAROUND_OBJ', bound=WorkaroundBase) diff --git a/src/sml2mqtt/value/workarounds/negative_on_energy_status.py b/src/sml2mqtt/sml_value/workarounds/negative_on_energy_status.py similarity index 93% rename from src/sml2mqtt/value/workarounds/negative_on_energy_status.py rename to src/sml2mqtt/sml_value/workarounds/negative_on_energy_status.py index 5deff25..93efd8d 100644 --- a/src/sml2mqtt/value/workarounds/negative_on_energy_status.py +++ b/src/sml2mqtt/sml_value/workarounds/negative_on_energy_status.py @@ -16,7 +16,7 @@ def __init__(self, arg: WORKAROUND_TYPE): def fix(self, value: SmlListEntry, frame_values: Dict[str, SmlListEntry]) -> SmlListEntry: meter = frame_values.get(self.meter_obis) if meter is None: - raise ValueError(f'Configured meter obis "{self.meter_obis}" not found in current frame') + raise ValueError(f'Configured meter obis "{self.meter_obis:s}" not found in current frame') status = meter.status if not isinstance(status, int): @@ -25,3 +25,5 @@ def fix(self, value: SmlListEntry, frame_values: Dict[str, SmlListEntry]) -> Sml negative = status & 0x20 if negative: value.value *= -1 + + return value diff --git a/src/sml2mqtt/value/__init__.py b/src/sml2mqtt/value/__init__.py deleted file mode 100644 index 8897ea8..0000000 --- a/src/sml2mqtt/value/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from . import __types__, filter - -# isort: split - -import sml2mqtt.config - -# isort: split - -from sml2mqtt.value.enum_builder import filter_from_config, transform_from_config, workaround_from_config -from sml2mqtt.value.smlvalue import SmlValue diff --git a/src/sml2mqtt/value/filter/__init__.py b/src/sml2mqtt/value/filter/__init__.py deleted file mode 100644 index 3712307..0000000 --- a/src/sml2mqtt/value/filter/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from sml2mqtt.value.filter.base import FILTER_OBJ - -# isort: split - -from sml2mqtt.value.filter.change import ChangeFilter -from sml2mqtt.value.filter.diff import DiffAbsFilter, DiffFilterBase, DiffPercFilter -from sml2mqtt.value.filter.time import RefreshEvery diff --git a/src/sml2mqtt/value/refresh_marker.py b/src/sml2mqtt/value/refresh_marker.py deleted file mode 100644 index 9d4fdeb..0000000 --- a/src/sml2mqtt/value/refresh_marker.py +++ /dev/null @@ -1,26 +0,0 @@ -from time import monotonic -from typing import Any, Final - - -class RefreshMarker: - def __init__(self, refresh_time: float): - if refresh_time <= 0: - raise ValueError('Refresh time must be > 0') - - self._refresh_time: Final = refresh_time - self._last_refresh: float = monotonic() - - self._last_value: Any = None - - def required(self, value: Any) -> bool: - if value is not None and value != self._last_value: - return True - - if monotonic() - self._last_refresh >= self._refresh_time: - return True - - return False - - def done(self, value: Any): - self._last_refresh = monotonic() - self._last_value = value diff --git a/src/sml2mqtt/value/transformations/__init__.py b/src/sml2mqtt/value/transformations/__init__.py deleted file mode 100644 index 3c07f1e..0000000 --- a/src/sml2mqtt/value/transformations/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from sml2mqtt.value.transformations.base import TRANSFORM_OBJ - -# isort: split - -from sml2mqtt.value.transformations.math import FactorTransformation, OffsetTransformation, RoundTransformation diff --git a/src/sml2mqtt/value/transformations/base.py b/src/sml2mqtt/value/transformations/base.py deleted file mode 100644 index 1dd8815..0000000 --- a/src/sml2mqtt/value/transformations/base.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import TypeVar - -from sml2mqtt.value.__types__ import VALUE_TYPE - - -class TransformationBase: - def process(self, value: VALUE_TYPE) -> VALUE_TYPE: - raise NotImplementedError() - - -TRANSFORM_OBJ = TypeVar('TRANSFORM_OBJ', bound=TransformationBase) diff --git a/src/sml2mqtt/value/workarounds/__init__.py b/src/sml2mqtt/value/workarounds/__init__.py deleted file mode 100644 index 5eacc48..0000000 --- a/src/sml2mqtt/value/workarounds/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from sml2mqtt.value.workarounds.base import WORKAROUND_OBJ - -# isort: split - -from sml2mqtt.value.workarounds.negative_on_energy_status import NegativeOnEnergyMeterStatus diff --git a/tests/config/test_default.py b/tests/config/test_default.py index caa747d..36c6355 100644 --- a/tests/config/test_default.py +++ b/tests/config/test_default.py @@ -22,7 +22,7 @@ def test_default(): qos: 0 # Default value for QOS if no other QOS value in the config entry is set retain: false # Default value for retain if no other retain value in the config entry is set last will: - topic: status + topic: status # Topic fragment for building this topic with the parent topic general: Wh in kWh: true # Automatically convert Wh to kWh republish after: 120 # Republish automatically after this time (if no other filter configured) @@ -33,16 +33,16 @@ def test_default(): timeout: 3 # Seconds after which a timeout will be detected (default=3) devices: # Device configuration by ID or url DEVICE_ID_HEX: - mqtt: - topic: DEVICE_BASE_TOPIC - status: - topic: status + mqtt: # Optional MQTT configuration for this meter. + topic: DEVICE_BASE_TOPIC # Topic fragment for building this topic with the parent topic + status: # Optional MQTT status topic configuration for this meter + topic: status # Topic fragment for building this topic with the parent topic skip: # OBIS codes (HEX) of values that will not be published (optional) - OBIS values: # Special configurations for each of the values (optional) OBIS: mqtt: # Mqtt config for this entry (optional) - topic: OBIS + topic: OBIS # Topic fragment for building this topic with the parent topic workarounds: # Workarounds for the value (optional) - negative on energy meter status: true transformations: # Mathematical transformations for the value (optional) diff --git a/tests/conftest.py b/tests/conftest.py index 3e752cf..b3cda67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,10 +16,10 @@ def no_mqtt(monkeypatch): pub_list = [] - async def pub_func(topic: str, value, qos: int, retain: bool): + def pub_func(topic: str, value, qos: int, retain: bool): pub_list.append((topic, value, qos, retain)) - monkeypatch.setattr(sml2mqtt.mqtt.mqtt_obj, 'publish', pub_func) + monkeypatch.setattr(sml2mqtt.mqtt.mqtt_obj, 'pub_func', pub_func) yield pub_list @@ -58,3 +58,88 @@ def sml_frame_2(): b'1c8c00' yield SmlFrame(a2b_hex(data)) + + +@pytest.fixture +def sml_frame_1_analyze(): + msg = """ +Received Frame + -> b'760500531efa620062007263010176010105001bb4fe0b0a0149534b0005020de272620165001bb32e620163a71400760500531efb620062007263070177010b0a0149534b0005020de2070100620affff72620165001bb32e757707010060320101010101010449534b0177070100600100ff010101010b0a0149534b0005020de20177070100010800ff65001c010401621e52ff650026bea90177070100020800ff0101621e52ff62000177070100100700ff0101621b52005301100101016350ba00760500531efc6200620072630201710163ba1900' + + + transaction_id: 00531efa + group_no : 0 + abort_on_error: 0 + message_body + codepage : None + client_id : None + req_file_id: 001bb4fe + server_id : 0a0149534b0005020de2 + ref_time : 1815342 + sml_version: 1 + crc16 : 42772 + + transaction_id: 00531efb + group_no : 0 + abort_on_error: 0 + message_body + client_id : None + server_id : 0a0149534b0005020de2 + list_name : 0100620affff + act_sensor_time : 1815342 + val_list: + + obis : 010060320101 (1-0:96.50.1*1) + status : None + val_time : None + unit : None + scaler : None + value : ISK + value_signature: None + + obis : 0100600100ff (1-0:96.1.0*255) + status : None + val_time : None + unit : None + scaler : None + value : 0a0149534b0005020de2 + value_signature: None + + obis : 0100010800ff (1-0:1.8.0*255) + status : 1835268 + val_time : None + unit : 30 + scaler : -1 + value : 2539177 + value_signature: None + -> 253917.7Wh (Zählerstand Total) + + obis : 0100020800ff (1-0:2.8.0*255) + status : None + val_time : None + unit : 30 + scaler : -1 + value : 0 + value_signature: None + -> 0.0Wh (Wirkenergie Total) + + obis : 0100100700ff (1-0:16.7.0*255) + status : None + val_time : None + unit : 27 + scaler : 0 + value : 272 + value_signature: None + -> 272W (aktuelle Wirkleistung) + list_signature : None + act_gateway_time: None + crc16 : 20666 + + transaction_id: 00531efc + group_no : 0 + abort_on_error: 0 + message_body + global_signature: None + crc16 : 47641 +""" + return msg diff --git a/tests/device/conftest.py b/tests/device/conftest.py index 37e42b0..fb9ff7d 100644 --- a/tests/device/conftest.py +++ b/tests/device/conftest.py @@ -1,15 +1,18 @@ +from typing import Set, Union from unittest.mock import AsyncMock, Mock import pytest +from smllib import SmlFrame, SmlStreamReader import sml2mqtt.device.sml_device import sml2mqtt.device.sml_serial +from sml2mqtt import CMD_ARGS from sml2mqtt.config.config import PortSettings from sml2mqtt.device import Device, DeviceStatus -from sml2mqtt.mqtt import MqttObj +from sml2mqtt.mqtt import MqttObj, patch_analyze -@pytest.fixture +@pytest.fixture() def no_serial(monkeypatch): m = Mock() @@ -17,31 +20,67 @@ def no_serial(monkeypatch): monkeypatch.setattr(sml2mqtt.device, 'SmlSerial', m) monkeypatch.setattr(sml2mqtt.device.sml_serial, 'SmlSerial', m) + return m @pytest.fixture(autouse=True) def clean_devices(monkeypatch): monkeypatch.setattr(sml2mqtt.device.sml_device, 'ALL_DEVICES', {}) + return None -@pytest.fixture +class TestingStreamReader: + def __init__(self, reader: SmlStreamReader): + self.reader = reader + self.data = None + + def add(self, data: Union[SmlFrame, bytes]): + if isinstance(data, SmlFrame): + self.data = data + self.reader.clear() + else: + self.data = None + self.reader.add(data) + + def get_frame(self): + if self.data is None: + return self.reader.get_frame() + return self.data + + +class TestingDevice(Device): + + def __init__(self, url: str, timeout: float, skip_values: Set[str], mqtt_device: MqttObj): + super().__init__(url, timeout, skip_values, mqtt_device) + self.stream = TestingStreamReader(self.stream) + + self.testing_raise_on_status = True + + def set_status(self, new_status: DeviceStatus) -> bool: + if new_status is DeviceStatus.ERROR and self.testing_raise_on_status: + raise + return super().set_status(new_status) + + +@pytest.fixture() async def device(no_serial): device_url = 'device_url' mqtt_base = MqttObj('testing', 0, False).update() mqtt_device = mqtt_base.create_child(device_url) - obj = await Device.create(PortSettings(url=device_url), 1, set(), mqtt_device) + obj = await TestingDevice.create(PortSettings(url=device_url), 1, set(), mqtt_device) + + return obj + - # Wrapper so we see the traceback in the tests - def wrapper(func): - def raise_exception_on_error(status: DeviceStatus): - if status is status.ERROR: - raise - func(status) - return raise_exception_on_error +@pytest.fixture() +def arg_analyze(monkeypatch): + monkeypatch.setattr(CMD_ARGS, 'analyze', True) + patch_analyze() - assert hasattr(obj, 'set_status') - obj.set_status = wrapper(obj.set_status) + yield None - yield obj + module = sml2mqtt.mqtt.mqtt_obj + assert hasattr(module, 'pub_func') + module.pub_func = module.publish diff --git a/tests/device/frames/test_frame_1.py b/tests/device/frames/test_frame_1.py index 5b63215..5b6760d 100644 --- a/tests/device/frames/test_frame_1.py +++ b/tests/device/frames/test_frame_1.py @@ -1,122 +1,108 @@ -# flake8: noqa: E501 - import logging from smllib.reader import SmlFrame -from sml2mqtt import CMD_ARGS, CONFIG +from device.conftest import TestingDevice +from sml2mqtt import CONFIG from sml2mqtt.config.device import SmlDeviceConfig -from sml2mqtt.device import Device - -async def test_frame_no_id(device: Device, no_serial, caplog, sml_frame_1: SmlFrame, monkeypatch): - - monkeypatch.setitem(CONFIG.devices, 'device_url', SmlDeviceConfig( - mqtt={'topic': 'xxxx'}, skip={'010060320101', '0100600100ff'} - )) +async def test_frame_no_match_obis_id(device: TestingDevice, no_serial, caplog, monkeypatch, + sml_frame_1: SmlFrame, sml_frame_1_analyze, arg_analyze): caplog.set_level(logging.DEBUG) - monkeypatch.setattr(CMD_ARGS, 'analyze', True) - await device.process_frame(sml_frame_1) + device.testing_raise_on_status = False + device.serial_data_read(sml_frame_1) - msg = "\n".join(map(lambda x: x.msg, caplog.records)) + msg = "\n".join(x.msg for x in caplog.records) - assert msg == """ + assert msg == sml_frame_1_analyze + """ +Found none of the following obis ids in the sml frame: 0100000009ff Received Frame -> b'760500531efa620062007263010176010105001bb4fe0b0a0149534b0005020de272620165001bb32e620163a71400760500531efb620062007263070177010b0a0149534b0005020de2070100620affff72620165001bb32e757707010060320101010101010449534b0177070100600100ff010101010b0a0149534b0005020de20177070100010800ff65001c010401621e52ff650026bea90177070100020800ff0101621e52ff62000177070100100700ff0101621b52005301100101016350ba00760500531efc6200620072630201710163ba1900' - - transaction_id: 00531efa - group_no : 0 - abort_on_error: 0 - message_body - codepage : None - client_id : None - req_file_id: 001bb4fe - server_id : 0a0149534b0005020de2 - ref_time : 1815342 - sml_version: 1 - crc16 : 42772 - - transaction_id: 00531efb - group_no : 0 - abort_on_error: 0 - message_body - client_id : None - server_id : 0a0149534b0005020de2 - list_name : 0100620affff - act_sensor_time : 1815342 - val_list: - - obis : 010060320101 (1-0:96.50.1*1) - status : None - val_time : None - unit : None - scaler : None - value : ISK - value_signature: None - - obis : 0100600100ff (1-0:96.1.0*255) - status : None - val_time : None - unit : None - scaler : None - value : 0a0149534b0005020de2 - value_signature: None - - obis : 0100010800ff (1-0:1.8.0*255) - status : 1835268 - val_time : None - unit : 30 - scaler : -1 - value : 2539177 - value_signature: None - -> 253917.7Wh (Zählerstand Total) - - obis : 0100020800ff (1-0:2.8.0*255) - status : None - val_time : None - unit : 30 - scaler : -1 - value : 0 - value_signature: None - -> 0.0Wh (Wirkenergie Total) - - obis : 0100100700ff (1-0:16.7.0*255) - status : None - val_time : None - unit : 27 - scaler : 0 - value : 272 - value_signature: None - -> 272W (aktuelle Wirkleistung) - list_signature : None - act_gateway_time: None - crc16 : 20666 - - transaction_id: 00531efc - group_no : 0 - abort_on_error: 0 - message_body - global_signature: None - crc16 : 47641 - -Creating default value handler for device_url/0100010800ff -testing/xxxx/0100010800ff: 253.9177 (QOS: 0, retain: False) -Creating default value handler for device_url/0100100700ff -testing/xxxx/0100100700ff: 272 (QOS: 0, retain: False) +ERROR +testing/device_url/status: ERROR (QOS: 0, retain: False)""" + + +async def test_frame_no_config(device: TestingDevice, no_serial, caplog, monkeypatch, + sml_frame_1: SmlFrame, sml_frame_1_analyze, arg_analyze): + caplog.set_level(logging.DEBUG) + monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) + + device.testing_raise_on_status = False + device.serial_data_read(sml_frame_1) + + msg = "\n".join(x.msg for x in caplog.records) + + assert msg == sml_frame_1_analyze + """ +Found obis id 0100600100ff in the sml frame +No configuration found for 0a0149534b0005020de2 +Creating default value handler for 010060320101 +Creating default value handler for 0100010800ff +Creating default value handler for 0100100700ff +testing/0a0149534b0005020de2/010060320101: ISK (QOS: 0, retain: False) +testing/0a0149534b0005020de2/0100010800ff: 253.9177 (QOS: 0, retain: False) +testing/0a0149534b0005020de2/0100100700ff: 272 (QOS: 0, retain: False) +OK +testing/0a0149534b0005020de2/status: OK (QOS: 0, retain: False) + +testing/0a0149534b0005020de2/010060320101 (010060320101): + raw value: ISK + pub value: ISK + filters: + - + - + +testing/0a0149534b0005020de2/0100010800ff (0100010800ff): + raw value: 253.9177 + pub value: 253.9177 + filters: + - + - + +testing/0a0149534b0005020de2/0100100700ff (0100100700ff): + raw value: 272 + pub value: 272 + filters: + - + - + +SHUTDOWN +testing/0a0149534b0005020de2/status: SHUTDOWN (QOS: 0, retain: False)""" + + +async def test_frame_with_config(device: TestingDevice, no_serial, caplog, monkeypatch, + sml_frame_1: SmlFrame, sml_frame_1_analyze, arg_analyze): + caplog.set_level(logging.DEBUG) + + monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) + monkeypatch.setitem(CONFIG.devices, '0a0149534b0005020de2', SmlDeviceConfig( + skip=['010060320101'] + )) + + device.serial_data_read(sml_frame_1) + + msg = "\n".join(x.msg for x in caplog.records) + + assert msg == sml_frame_1_analyze + """ +Found obis id 0100600100ff in the sml frame +Configuration found for 0a0149534b0005020de2 +Creating default value handler for 0100010800ff +Creating default value handler for 0100100700ff +testing/0a0149534b0005020de2/0100010800ff: 253.9177 (QOS: 0, retain: False) +testing/0a0149534b0005020de2/0100100700ff: 272 (QOS: 0, retain: False) OK -testing/xxxx/status: OK (QOS: 0, retain: False) +testing/0a0149534b0005020de2/status: OK (QOS: 0, retain: False) -testing/xxxx/0100010800ff (0100010800ff): +testing/0a0149534b0005020de2/0100010800ff (0100010800ff): raw value: 253.9177 pub value: 253.9177 filters: - - -testing/xxxx/0100100700ff (0100100700ff): +testing/0a0149534b0005020de2/0100100700ff (0100100700ff): raw value: 272 pub value: 272 filters: @@ -124,4 +110,4 @@ async def test_frame_no_id(device: Device, no_serial, caplog, sml_frame_1: SmlFr - SHUTDOWN -testing/xxxx/status: SHUTDOWN (QOS: 0, retain: False)""" +testing/0a0149534b0005020de2/status: SHUTDOWN (QOS: 0, retain: False)""" diff --git a/tests/device/frames/test_frame_2.py b/tests/device/frames/test_frame_2.py index e3d5ad1..a2714aa 100644 --- a/tests/device/frames/test_frame_2.py +++ b/tests/device/frames/test_frame_2.py @@ -8,19 +8,19 @@ from sml2mqtt.device import Device -async def test_frame_no_shortcut(device: Device, no_serial, caplog, sml_frame_2: SmlFrame, monkeypatch, no_mqtt): +async def test_frame_2(device: Device, no_serial, caplog, sml_frame_2: SmlFrame, monkeypatch, no_mqtt): caplog.set_level(logging.DEBUG) - monkeypatch.setitem(CONFIG.devices, 'device_url', SmlDeviceConfig( + monkeypatch.setattr(CONFIG.general, 'device_id_obis', ['0100600100ff']) + monkeypatch.setitem(CONFIG.devices, '0a014c475a0003403b49', SmlDeviceConfig( mqtt={'topic': 'xxxx'} )) - await device.process_frame(sml_frame_2) + device.process_frame(sml_frame_2) await asyncio.sleep(0.01) assert no_mqtt == [ ('testing/xxxx/010060320101', 'LGZ', 0, False), - ('testing/xxxx/0100600100ff', '0a014c475a0003403b49', 0, False), ('testing/xxxx/0100010800ff', 5171.9237, 0, False), ('testing/xxxx/0100100700ff', 251, 0, False), ('testing/xxxx/status', 'OK', 0, False) diff --git a/tests/device/test_data.py b/tests/device/test_data.py index 045a899..ee20fa9 100644 --- a/tests/device/test_data.py +++ b/tests/device/test_data.py @@ -1,20 +1,27 @@ -# flake8: noqa: E501 -import asyncio import logging +from asyncio import Task from binascii import a2b_hex +from unittest.mock import Mock -from sml2mqtt import CMD_ARGS -from sml2mqtt.device import Device +from serial_asyncio import SerialTransport +from sml2mqtt.device import Device, SmlSerial -async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, monkeypatch): + +async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, arg_analyze): caplog.set_level(logging.DEBUG) - monkeypatch.setattr(CMD_ARGS, 'analyze', True) + # we want to test incoming data from the serial port + device.serial = SmlSerial() + device.serial.device = device + device.serial.transport = Mock(SerialTransport) + device.serial._task = Mock(Task) - await device.serial_data_read(a2b_hex(sml_data_1)) + chunk_size = 100 + for i in range(0, len(sml_data_1), chunk_size): + device.serial.data_received(a2b_hex(sml_data_1[i: i + chunk_size])) - msg = "\n".join(map(lambda x: x.msg, filter(lambda x: x.name == 'sml.mqtt.pub', caplog.records))) + msg = "\n".join(x.msg for x in filter(lambda x: x.name == 'sml.mqtt.pub', caplog.records)) assert msg == \ 'testing/00000000000000000000/0100010800ff: 450.09189911 (QOS: 0, retain: False)\n' \ @@ -28,7 +35,7 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, 'testing/00000000000000000000/status: OK (QOS: 0, retain: False)\n' \ 'testing/00000000000000000000/status: SHUTDOWN (QOS: 0, retain: False)' - msg = "\n".join(map(lambda x: x.msg, filter(lambda x: x.name == 'sml.device_url', caplog.records))) + msg = "\n".join(x.msg for x in filter(lambda x: x.name == 'sml.device_url', caplog.records)) assert msg == ''' Received Frame @@ -157,6 +164,16 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, global_signature: None crc16 : 9724 +Found obis id 0100000009ff in the sml frame +No configuration found for 00000000000000000000 +Creating default value handler for 0100010800ff +Creating default value handler for 0100010801ff +Creating default value handler for 0100010802ff +Creating default value handler for 0100020800ff +Creating default value handler for 0100100700ff +Creating default value handler for 0100240700ff +Creating default value handler for 0100380700ff +Creating default value handler for 01004c0700ff testing/00000000000000000000/0100010800ff (0100010800ff): raw value: 450.09189911 @@ -214,5 +231,3 @@ async def test_serial_data(device: Device, no_serial, caplog, sml_data_1: bytes, - - ''' - - await asyncio.sleep(0.1) diff --git a/tests/device/test_device.py b/tests/device/test_device.py new file mode 100644 index 0000000..c000f07 --- /dev/null +++ b/tests/device/test_device.py @@ -0,0 +1,27 @@ +import asyncio +from time import monotonic +from unittest.mock import Mock + +from sml2mqtt.device import Device, SmlSerial +from sml2mqtt.mqtt import MqttObj + + +async def test_device_await(device: Device, no_serial, caplog): + device = Device('test', 1, set(), MqttObj('testing', 0, False)) + device.serial = SmlSerial() + device.serial.url = 'test' + device.serial.transport = Mock() + device.serial.transport.is_closing = lambda: False + device.start() + + async def cancel(): + await asyncio.sleep(0.3) + device.stop() + + t = asyncio.create_task(cancel()) + start = monotonic() + await asyncio.wait_for(device, 1) + await t + assert monotonic() - start < 0.4 + + await asyncio.sleep(0.1) diff --git a/tests/device/test_watchdog.py b/tests/device/test_watchdog.py index be57bc1..d0853c8 100644 --- a/tests/device/test_watchdog.py +++ b/tests/device/test_watchdog.py @@ -1,7 +1,11 @@ import asyncio +from binascii import a2b_hex from unittest.mock import Mock +from sml2mqtt.config.config import PortSettings +from sml2mqtt.device import Device, DeviceStatus from sml2mqtt.device.watchdog import Watchdog +from sml2mqtt.mqtt import MqttObj async def test_watchdog_expire(): @@ -11,8 +15,12 @@ async def test_watchdog_expire(): w = Watchdog(0.1, m) w.start() - await asyncio.sleep(0.4) + + await asyncio.sleep(0.15) m.assert_called_once() + w.feed() + await asyncio.sleep(0.15) + assert m.call_count == 2 w.cancel() @@ -28,9 +36,9 @@ async def test_watchdog_no_expire(): w = Watchdog(0.1, m) w.start() - for _ in range(3): + for _ in range(4): w.feed() - await asyncio.sleep(0.07) + await asyncio.sleep(0.06) m.assert_not_called() @@ -39,3 +47,30 @@ async def test_watchdog_no_expire(): # Assert that the task is properly canceled await asyncio.sleep(0.05) assert w.task is None + + +async def test_watchdog_setup_and_feed(no_serial, sml_data_1): + device_url = 'watchdog_test' + + mqtt_base = MqttObj('testing', 0, False).update() + mqtt_device = mqtt_base.create_child(device_url) + + obj = await Device.create(PortSettings(url=device_url), 0.2, set(), mqtt_device) + obj.start() + assert obj.status == DeviceStatus.STARTUP + + await asyncio.sleep(0.3) + assert obj.status == DeviceStatus.MSG_TIMEOUT + + for _ in range(5): + await asyncio.sleep(0.15) + obj.serial_data_read(a2b_hex(sml_data_1)) + assert obj.status != DeviceStatus.MSG_TIMEOUT + + await asyncio.sleep(0.3) + assert obj.status == DeviceStatus.MSG_TIMEOUT + + async def cancel(): + obj.stop() + + await asyncio.gather(obj, cancel()) diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..09feee2 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,86 @@ +from inspect import getmembers, isclass +from pathlib import Path + +from easyconfig import yaml +from pydantic import BaseModel + +import sml2mqtt + + +def test_sample_yaml(pytestconfig): + file = pytestconfig.rootpath / 'docs' / 'configuration.rst' + + all_cfgs = [] + + lines = [] + add = False + indent = 0 + + for line in file.read_text().splitlines(): + line = line + stripped = line.strip() + + if add: + if not indent and stripped: + while line[indent] == ' ': + indent += 1 + + if stripped and line[0] != ' ': + all_cfgs.append(lines) + add = False + continue + + lines.append(line[indent:]) + + if stripped.startswith('.. code-block:: yaml') or stripped.startswith('.. code-block:: yml'): + add = True + lines = [] + + if add: + all_cfgs.append(lines) + + assert len(all_cfgs) == 2 + for cfg_lines in all_cfgs: + sample_cfg = '\n'.join(cfg_lines) + + map = yaml.yaml_rt.load(sample_cfg) + sml2mqtt.config.config.Settings(**map) + + +def test_config_documentation_complete(pytestconfig): + cfg_docs: Path = pytestconfig.rootpath / 'docs' / 'configuration.rst' + cfg_model_dir: Path = pytestconfig.rootpath / 'src' / 'sml2mqtt' / 'config' + assert cfg_model_dir.is_dir() + + documented_objs = set() + + # documented config + current_module = '' + for line in (x.strip().replace(' ', '') for x in cfg_docs.read_text().splitlines()): # type: str + if line.startswith('.. py:currentmodule::'): + current_module = line[21:].strip() + continue + + if line.startswith('.. autopydantic_model::'): + obj_name = line[23:].strip() + if current_module: + obj_name = f'{current_module}.{obj_name}' + assert obj_name not in documented_objs + documented_objs.add(obj_name) + + # get Config implementation from source + existing_objs = set() + for module_name in [f.stem for f in cfg_model_dir.glob('**/*.py')]: + module = getattr(sml2mqtt.config, module_name) + cfg_objs = [x[1] for x in getmembers(module, lambda x: isclass(x) and issubclass(x, BaseModel))] + cfg_names = { + f'{obj.__module__}.{obj.__qualname__}' for obj in cfg_objs if not obj.__module__.startswith('easyconfig.') + } + existing_objs.update(cfg_names) + + # we check this here to get the module with the error message + missing = cfg_names - documented_objs + assert not missing, module.__name__ + + # ensure that everything that is implemented is documented + assert existing_objs == documented_objs diff --git a/tests/test_readme.py b/tests/test_readme.py index 117f1e0..92984e5 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -14,5 +14,5 @@ def test_readme(pytestconfig): # First entry is the complete config for cfg_sample in yaml_parts: - obj = yaml_rt.load(yaml_parts[0]) # type: dict + obj = yaml_rt.load(cfg_sample) # type: dict Settings(**obj) diff --git a/tests/test_update_marker.py b/tests/test_update_marker.py deleted file mode 100644 index 4e4948f..0000000 --- a/tests/test_update_marker.py +++ /dev/null @@ -1,23 +0,0 @@ -import sml2mqtt.value.refresh_marker as marker_package - - -def test_rest_code(monkeypatch): - - timestamp = 0 - monkeypatch.setattr(marker_package, 'monotonic', lambda: timestamp) - - marker = marker_package.RefreshMarker(5) - - timestamp = 1 - assert marker.required(1) - assert marker.required(2) - marker.done(1) - assert not marker.required(1) - assert marker.required(2) - - timestamp = 6 - assert marker.required(1) - assert marker.required(2) - marker.done(2) - assert marker.required(1) - assert not marker.required(2) diff --git a/tests/values/filters/test_diff.py b/tests/values/filters/test_diff.py index 488c053..2714ad7 100644 --- a/tests/values/filters/test_diff.py +++ b/tests/values/filters/test_diff.py @@ -1,4 +1,4 @@ -from sml2mqtt.value.filter import DiffAbsFilter, DiffPercFilter +from sml2mqtt.sml_value.filter import DiffAbsFilter, DiffPercFilter def test_abs(): diff --git a/tests/values/transformations/test_math.py b/tests/values/transformations/test_math.py index 0d1fe21..590dcbd 100644 --- a/tests/values/transformations/test_math.py +++ b/tests/values/transformations/test_math.py @@ -2,9 +2,9 @@ from smllib.sml import SmlListEntry -from sml2mqtt.value import SmlValue -from sml2mqtt.value.filter import RefreshEvery -from sml2mqtt.value.transformations import RoundTransformation +from sml2mqtt.sml_value import SmlValue +from sml2mqtt.sml_value.filter import RefreshEvery +from sml2mqtt.sml_value.transformations import RoundTransformation def test_round(): diff --git a/tests/values/workarounds/test_negative_on_energy_meter.py b/tests/values/workarounds/test_negative_on_energy_meter.py index 692ba09..1bded77 100644 --- a/tests/values/workarounds/test_negative_on_energy_meter.py +++ b/tests/values/workarounds/test_negative_on_energy_meter.py @@ -2,7 +2,7 @@ from smllib.sml import SmlListEntry -from sml2mqtt.value.workarounds import NegativeOnEnergyMeterStatus +from sml2mqtt.sml_value.workarounds import NegativeOnEnergyMeterStatus def get_entries() -> Tuple[SmlListEntry, SmlListEntry]: diff --git a/tox.ini b/tox.ini index cf4cf7c..3618d8b 100644 --- a/tox.ini +++ b/tox.ini @@ -4,14 +4,16 @@ envlist = py38 py39 py310 - flake + py311 + docs [gh-actions] python = 3.8: py38 3.9: py39 - 3.10: py310, flake + 3.10: py310, docs + 3.11: py311 [testenv] deps = @@ -21,14 +23,19 @@ commands = python -m pytest -[testenv:flake] +[pytest] +asyncio_mode = auto + + +[testenv:docs] +description = invoke sphinx-build to build the HTML docs + deps = {[testenv]deps} - flake8 + -r{toxinidir}/docs/requirements.txt commands = - flake8 -v - + mkdir -p docs{/}_static + sphinx-build -d "{envtmpdir}{/}doctree" docs "{toxworkdir}{/}docs_out" --color -b html -E -W -n --keep-going -[pytest] -asyncio_mode = auto +allowlist_externals = mkdir diff --git a/whitelist.txt b/whitelist.txt new file mode 100644 index 0000000..388fd74 --- /dev/null +++ b/whitelist.txt @@ -0,0 +1,31 @@ +mqtt +qos + +# sml +sml +smllib +scaler +sml2mqtt + +# python packages +pydantic +easyconfig +unittest +binascii + +# pyserial +baudrate +stopbits +bytesize + +# pytest +autouse +caplog + +# pydantic +conint +constr +validator + +# own words +cfg