diff --git a/CHANGES.md b/CHANGES.md index 9917001..43e333f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ## Unreleased - Added transformer for MongoDB CDC to CrateDB SQL conversion +- Rename "Airrohr" decoder to "Sensor.Community" ## 2024/07/16 v0.0.1 - Added decoders for Airrohr, Tasmota, and TTS/TTN from Kotori DAQ diff --git a/README.md b/README.md index eb22042..2b6ff7f 100644 --- a/README.md +++ b/README.md @@ -36,21 +36,30 @@ pip install --upgrade commons-codec In order to learn how to use the library, please visit the [documentation], and explore the source code or its [examples]. -## License -The project uses the LGPLv3 license for the whole ensemble. However, individual -portions of the code base are vendorized from other Python packages, where -deviating licenses may apply. Please check for detailed license information -within the header sections of relevant files. -## Contributing +## Project Information + +### Acknowledgements +Kudos to the authors of all the many software components this library is +vendoring and building upon. + +### Contributing The `commons-codec` package is an open source project, and is [managed on GitHub](https://github.com/daq-tools/commons-codec). -We appreciate contributions of any kind. +The project is still in its infancy, and we appreciate contributions +of any kind. -## Etymology +### Etymology The [Apache Commons Codec] library was the inspiration for the name. Otherwise, both libraries' ingredients don't have anything in common, yet. +### License +The project uses the LGPLv3 license for the whole ensemble. However, individual +portions of the code base are vendorized from other Python packages, where +deviating licenses may apply. Please check for detailed license information +within the header sections of relevant files. + + [Apache Commons Codec]: https://commons.apache.org/proper/commons-codec/ [commons-codec]: https://pypi.org/project/commons-codec/ diff --git a/docs/backlog.md b/docs/backlog.md new file mode 100644 index 0000000..4ebf7c0 --- /dev/null +++ b/docs/backlog.md @@ -0,0 +1,14 @@ +# Backlog + +## Iteration +1 +- [x] Add Kotori decoders +- [x] Add DynamoDB and MongoDB CDC translators +- [x] Resolve dependencies: `bson` vs. `pymongo`? +- [ ] Documentation +- [x] Release v0.0.2 + +## Iteration +2 +- [ ] Performance: Batching! +- [ ] MongoDB: Implement stream resumption using `start_after` +- [ ] Feature: Filter by events, e.g. Ignore "delete" events? +- [ ] Integration Testing the "example" programs? diff --git a/pyproject.toml b/pyproject.toml index 27b6cce..0bd369b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,10 +22,12 @@ keywords = [ "encode", "i/o", "json", + "luftdaten.info", "map data", "marshall", "mongodb", "nested data", + "sensor.community", "serialize", "sql", "tasmota", diff --git a/src/commons_codec/decode/airrohr.py b/src/commons_codec/decode/sensor_community.py similarity index 88% rename from src/commons_codec/decode/airrohr.py rename to src/commons_codec/decode/sensor_community.py index 1b61bfd..ad53321 100644 --- a/src/commons_codec/decode/airrohr.py +++ b/src/commons_codec/decode/sensor_community.py @@ -6,13 +6,16 @@ from commons_codec.util.data import is_number -class AirrohrDecoder: +class SensorCommunity: """ - Decode JSON payloads in Airrohr format. + Decode JSON payloads in Sensor.Community format. + + Previously / Also: Airrohr, dusti.api, Luftdaten.info Documentation ============= - - https://getkotori.org/docs/integration/airrohr.html + - https://github.com/opendata-stuttgart/meta/wiki/APIs + - https://kotori.readthedocs.io/en/latest/integration/airrohr.html - https://community.hiveeyes.org/t/more-data-acquisition-payload-formats-for-kotori/1421/2 Example diff --git a/tests/decode/test_airrohr.py b/tests/decode/test_sensor_community.py similarity index 77% rename from tests/decode/test_airrohr.py rename to tests/decode/test_sensor_community.py index 59801ba..28f6b5e 100644 --- a/tests/decode/test_airrohr.py +++ b/tests/decode/test_sensor_community.py @@ -1,17 +1,14 @@ -# -*- coding: utf-8 -*- -# (c) 2020-2021 Andreas Motl +# Copyright (c) 2020-2024, The Kotori Developers and contributors. +# Distributed under the terms of the LGPLv3 license, see LICENSE. import logging import typing as t -from commons_codec.decode.airrohr import AirrohrDecoder +from commons_codec.decode.sensor_community import SensorCommunity from commons_codec.util.data import jd logger = logging.getLogger(__name__) -# https://community.hiveeyes.org/t/more-data-acquisition-payload-formats-for-kotori/1421/2 - - data_in = { "esp8266id": 12041741, "sensordatavalues": [ @@ -42,13 +39,12 @@ } -def test_decode_airrohr(): +def test_decode_sensor_community(): """ - Submit single reading in Airrohr JSON format to HTTP API - and proof it is stored in the InfluxDB database. + Verify decoding a single reading in Sensor.Community JSON format. """ - result = AirrohrDecoder.decode(jd(data_in)) + result = SensorCommunity.decode(jd(data_in)) assert result == data_out assert_type(result["SDS_P1"], float) assert_type(result["BME280_pressure"], float) @@ -59,6 +55,9 @@ def test_decode_airrohr(): def assert_type(value: t.Any, type_: t.Type): + """ + Assertion helper with better error reporting. + """ assert isinstance( value, type_ ), f"Value is of type '{type(value).__name__}', but should be '{type_.__name__}' instead" diff --git a/tests/decode/test_tasmota.py b/tests/decode/test_tasmota.py index a09a329..0e13f94 100644 --- a/tests/decode/test_tasmota.py +++ b/tests/decode/test_tasmota.py @@ -16,9 +16,7 @@ @pytest.mark.tasmota def test_tasmota_sonoff_sc(): """ - Publish a single SENSOR reading in Tasmota/JSON format - to MQTT broker, including a timestamp. - Proof that the reading is processed and stored correctly. + Verify decoding a reading from a "Sonoff SC" device in Tasmota JSON format. https://kotori.readthedocs.io/en/latest/integration/tasmota.html#submit """ @@ -45,9 +43,7 @@ def test_tasmota_sonoff_sc(): @pytest.mark.tasmota def test_tasmota_ds18b20(): """ - Publish another single SENSOR reading in Tasmota/JSON format - to MQTT broker, including a timestamp. - Proof that the reading is processed and stored correctly. + Verify decoding a reading from a "Wemos DS18B20" device in Tasmota JSON format. https://kotori.readthedocs.io/en/latest/integration/tasmota.html#submit """ @@ -68,8 +64,7 @@ def test_tasmota_ds18b20(): @pytest.mark.wemos def test_tasmota_wemos_dht22(): """ - Publish a reading from a Wemos multi sensor device. - Proof that the reading is processed and stored correctly. + Verify decoding a reading from a "Wemos DHT22" device in Tasmota JSON format. """ # Submit a single measurement. @@ -89,8 +84,7 @@ def test_tasmota_wemos_dht22(): @pytest.mark.wemos def test_tasmota_wemos_multi(): """ - Publish a reading from a Wemos multi sensor device. - Proof that the reading is processed and stored correctly. + Verify decoding a reading from a "Wemos multi sensor" device in Tasmota JSON format. """ # Submit a single measurement. @@ -121,9 +115,7 @@ def test_tasmota_wemos_multi(): @pytest.mark.tasmota def test_tasmota_state(): """ - Publish a single STATE reading in Tasmota/JSON format - to MQTT broker, including a timestamp. - Proof that the reading is processed and stored correctly. + Verify decoding a "state message" in Tasmota JSON format. https://kotori.readthedocs.io/en/latest/integration/tasmota.html#submit """ diff --git a/tests/decode/test_tts_ttn.py b/tests/decode/test_tts_ttn.py index 88eaa83..e1f0772 100644 --- a/tests/decode/test_tts_ttn.py +++ b/tests/decode/test_tts_ttn.py @@ -30,8 +30,9 @@ def test_decode_tts_ttn_full(tts_ttn_full): """ - Submit single reading in TTS/TTN webhook JSON format to HTTP API, - and verify it was correctly stored in the InfluxDB database. + Verify decoding a full message in TTS/TTN webhook JSON format. + + https://kotori.readthedocs.io/en/latest/integration/tts-ttn.html """ data_in = read_jsonfile(tts_ttn_full) @@ -60,8 +61,9 @@ def test_decode_tts_ttn_full(tts_ttn_full): def test_decode_tts_ttn_minimal(tts_ttn_minimal): """ - Submit single reading in TTS/TTN webhook JSON format to HTTP API, - and verify it was correctly stored in the InfluxDB database. + Verify decoding a minimal message in TTS/TTN webhook JSON format. + + https://kotori.readthedocs.io/en/latest/integration/tts-ttn.html """ data_in = read_jsonfile(tts_ttn_minimal)