From b64f17ffc9186af858fa18136008bd7b63e35525 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:35:57 -0700 Subject: [PATCH 01/31] Add new constructor methods --- harp/reader.py | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/harp/reader.py b/harp/reader.py index 6b3645c..a222c3a 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -13,6 +13,7 @@ from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.io import MessageType, read from harp.schema import read_schema +import requests @dataclass @@ -75,6 +76,117 @@ def __dir__(self) -> Iterable[str]: def __getattr__(self, __name: str) -> RegisterReader: return self.registers[__name] + @staticmethod + def from_file( + filepath: PathLike, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + device = read_schema(filepath, include_common_registers) + if base_path is None: + path = Path(filepath).absolute().resolve() + base_path = path.parent / device.device + else: + base_path = Path(base_path).absolute().resolve() / device.device + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_url( + url: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + timeout: int = 5) -> "DeviceReader": + + response = requests.get(url, timeout=timeout) + text = response.text + + device = read_schema(text, include_common_registers) + if base_path is None: + base_path = Path(device.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_str( + schema: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + device = read_schema(schema, include_common_registers) + if base_path is None: + base_path = Path(device.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_model( + model: Model, + base_path: Optional[PathLike] = None, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + if base_path is None: + base_path = Path(model.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + model, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in model.registers.keys() + } + return DeviceReader(model, reg_readers) + + @staticmethod + def from_dataset( + dataset: PathLike, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + path = Path(dataset).absolute().resolve() + is_dir = os.path.isdir(path) + if is_dir: + filepath = path / "device.yml" + return DeviceReader.from_file( + filepath=filepath, + base_path=path, + include_common_registers=include_common_registers, + epoch=epoch, + keep_type=keep_type) + else: + raise ValueError("The dataset must be a directory containing a device.yml file.") + def _compose_parser( f: Callable[[DataFrame], DataFrame], From ebd1956cd85889f2fe0e871fe1f2e151abf872ff Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:44:56 -0700 Subject: [PATCH 02/31] Deprecate function --- harp/reader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index a222c3a..f7e9eba 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -1,4 +1,6 @@ import os +import requests +from deprecated import deprecated from math import log2 from os import PathLike from pathlib import Path @@ -10,10 +12,10 @@ from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union from collections import UserDict from pandas._typing import Axes +from harp import __version__ from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.io import MessageType, read from harp.schema import read_schema -import requests @dataclass @@ -335,7 +337,7 @@ def parser(df: DataFrame): reader = partial(reader, columns=[name]) return RegisterReader(register, reader) - +@deprecated("This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead.") def create_reader( device: Union[str, PathLike, Model], include_common_registers: bool = True, From 2e51fc4748247c2aae218dd1b9f945af11b737b5 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:55:49 -0700 Subject: [PATCH 03/31] Document methods --- harp/reader.py | 173 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 149 insertions(+), 24 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index f7e9eba..91726a0 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -80,11 +80,37 @@ def __getattr__(self, __name: str) -> RegisterReader: @staticmethod def from_file( - filepath: PathLike, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + filepath: PathLike, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified schema yml file. + + Parameters + ---------- + filepath + A path to the device yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ device = read_schema(filepath, include_common_registers) if base_path is None: @@ -103,12 +129,39 @@ def from_file( @staticmethod def from_url( - url: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False, - timeout: int = 5) -> "DeviceReader": + url: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + timeout: int = 5, + ) -> "DeviceReader": + """Creates a device reader object from a url pointing to a device.yml file. + + Parameters + ---------- + url + The url pointing to the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + timeout + The number of seconds to wait for the server to send data before giving up. + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ response = requests.get(url, timeout=timeout) text = response.text @@ -129,11 +182,37 @@ def from_url( @staticmethod def from_str( - schema: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + schema: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a string containing a device.yml schema. + + Parameters + ---------- + schema + The string containing the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ device = read_schema(schema, include_common_registers) if base_path is None: @@ -151,10 +230,32 @@ def from_str( @staticmethod def from_model( - model: Model, - base_path: Optional[PathLike] = None, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + model: Model, + base_path: Optional[PathLike] = None, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a parsed device schema object. + + Parameters + ---------- + model + The parsed device schema object describing the device. + base_path + The path to attempt to resolve the location of data files. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ if base_path is None: base_path = Path(model.device).absolute().resolve() @@ -171,10 +272,34 @@ def from_model( @staticmethod def from_dataset( - dataset: PathLike, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + dataset: PathLike, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified dataset folder. + + Parameters + ---------- + dataset + A path to the dataset folder containing a device.yml schema describing the device. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ path = Path(dataset).absolute().resolve() is_dir = os.path.isdir(path) From 24b89773fdeace6fe4fa9ff01292c620eddb9b31 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:55:57 -0700 Subject: [PATCH 04/31] Linting --- harp/io.py | 3 ++- harp/model.py | 12 +++--------- harp/reader.py | 34 +++++++++++++++++++++------------- harp/schema.py | 6 ++++-- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/harp/io.py b/harp/io.py index 6c9630b..a37270e 100644 --- a/harp/io.py +++ b/harp/io.py @@ -2,9 +2,10 @@ from enum import IntEnum from os import PathLike from typing import Any, BinaryIO, Optional, Union -from pandas._typing import Axes + import numpy as np import pandas as pd +from pandas._typing import Axes REFERENCE_EPOCH = datetime(1904, 1, 1) """The reference epoch for UTC harp time.""" diff --git a/harp/model.py b/harp/model.py index 7c66f1d..f28a7be 100644 --- a/harp/model.py +++ b/harp/model.py @@ -6,16 +6,10 @@ from enum import Enum from typing import Dict, List, Optional, Union -from typing_extensions import Annotated -from pydantic import ( - BaseModel, - BeforeValidator, - ConfigDict, - Field, - RootModel, - field_serializer, -) +from pydantic import (BaseModel, BeforeValidator, ConfigDict, Field, RootModel, + field_serializer) +from typing_extensions import Annotated class PayloadType(str, Enum): diff --git a/harp/reader.py b/harp/reader.py index 91726a0..8f828f3 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -1,20 +1,23 @@ import os -import requests -from deprecated import deprecated +from collections import UserDict +from dataclasses import dataclass +from datetime import datetime +from functools import partial from math import log2 from os import PathLike from pathlib import Path -from datetime import datetime -from functools import partial -from dataclasses import dataclass +from typing import (Any, BinaryIO, Callable, Iterable, Mapping, Optional, + Protocol, Union) + +import requests +from deprecated import deprecated from numpy import dtype from pandas import DataFrame, Series -from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union -from collections import UserDict from pandas._typing import Axes + from harp import __version__ -from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.io import MessageType, read +from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.schema import read_schema @@ -31,8 +34,7 @@ def __call__( file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None, epoch: Optional[datetime] = None, keep_type: bool = False, - ) -> DataFrame: - ... + ) -> DataFrame: ... class RegisterReader: @@ -310,9 +312,12 @@ def from_dataset( base_path=path, include_common_registers=include_common_registers, epoch=epoch, - keep_type=keep_type) + keep_type=keep_type, + ) else: - raise ValueError("The dataset must be a directory containing a device.yml file.") + raise ValueError( + "The dataset must be a directory containing a device.yml file." + ) def _compose_parser( @@ -462,7 +467,10 @@ def parser(df: DataFrame): reader = partial(reader, columns=[name]) return RegisterReader(register, reader) -@deprecated("This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead.") + +@deprecated( + "This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead." +) def create_reader( device: Union[str, PathLike, Model], include_common_registers: bool = True, diff --git a/harp/schema.py b/harp/schema.py index 1bb1b0a..0787cc6 100644 --- a/harp/schema.py +++ b/harp/schema.py @@ -1,8 +1,10 @@ +from importlib import resources from os import PathLike from typing import TextIO, Union -from harp.model import Model, Registers + from pydantic_yaml import parse_yaml_raw_as -from importlib import resources + +from harp.model import Model, Registers def _read_common_registers() -> Registers: From a304318a27b8fb18a0f3075294bad2b5793271ae Mon Sep 17 00:00:00 2001 From: brunocruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Mon, 20 May 2024 16:27:09 -0700 Subject: [PATCH 05/31] Add isort and codespell linters and respective settings (#28) * Add `isort` and `codespell` linters and respective settings * Increase line length to 108 * Remove isort line length override --- pyproject.toml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cbd870e..53744e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,11 @@ classifiers = [ dev = [ "datamodel-code-generator", "pytest", - "black" + "black", + "isort", + "codespell" ] + jupyter = [ "ipykernel", "matplotlib" @@ -56,7 +59,7 @@ include = ["harp*"] [tool.setuptools_scm] [tool.black] -line-length = 88 +line-length = 108 target-version = ['py39'] include = '\.pyi?$' extend-exclude = ''' @@ -68,3 +71,10 @@ extend-exclude = ''' | reflex-generator ) ''' + +[tool.isort] +profile = 'black' + +[tool.codespell] +skip = '.git,*.pdf,*.svg' +ignore-words-list = 'nd' \ No newline at end of file From caaa6192b98b0c676e067053c663c483850f801c Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 21 May 2024 01:06:20 +0100 Subject: [PATCH 06/31] Add extended skip settings for isort --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 53744e9..7b4d5c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ extend-exclude = ''' [tool.isort] profile = 'black' +extend_skip = 'reflex-generator' [tool.codespell] skip = '.git,*.pdf,*.svg' From c84e835881b962af1aad34ba38cf6ada62c92a50 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 21 May 2024 01:12:21 +0100 Subject: [PATCH 07/31] Run black formatting --- harp/io.py | 16 ++++------------ harp/model.py | 20 +++++--------------- harp/reader.py | 11 +++-------- harp/schema.py | 8 ++------ tests/test_io.py | 9 ++------- 5 files changed, 16 insertions(+), 48 deletions(-) diff --git a/harp/io.py b/harp/io.py index 6c9630b..cb7a702 100644 --- a/harp/io.py +++ b/harp/io.py @@ -73,9 +73,7 @@ def read( """ data = np.fromfile(file, dtype=np.uint8) if len(data) == 0: - return pd.DataFrame( - columns=columns, index=pd.Index([], dtype=np.float64, name="Time") - ) + return pd.DataFrame(columns=columns, index=pd.Index([], dtype=np.float64, name="Time")) if address is not None and address != data[2]: raise ValueError(f"expected address {address} but got {data[2]}") @@ -86,13 +84,9 @@ def read( payloadtype = data[4] payloadoffset = 5 if payloadtype & 0x10 != 0: - seconds = np.ndarray( - nrows, dtype=np.uint32, buffer=data, offset=payloadoffset, strides=stride - ) + seconds = np.ndarray(nrows, dtype=np.uint32, buffer=data, offset=payloadoffset, strides=stride) payloadoffset += 4 - micros = np.ndarray( - nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride - ) + micros = np.ndarray(nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride) payloadoffset += 2 time = micros * _SECONDS_PER_TICK + seconds payloadtype = payloadtype & ~0x10 @@ -121,9 +115,7 @@ def read( result = pd.DataFrame(payload, index=index, columns=columns) if keep_type: - msgtype = np.ndarray( - nrows, dtype=np.uint8, buffer=data, offset=0, strides=stride - ) + msgtype = np.ndarray(nrows, dtype=np.uint8, buffer=data, offset=0, strides=stride) msgtype = pd.Categorical.from_codes(msgtype, categories=_messagetypes) # type: ignore result[MessageType.__name__] = msgtype return result diff --git a/harp/model.py b/harp/model.py index 7c66f1d..891eeb9 100644 --- a/harp/model.py +++ b/harp/model.py @@ -136,16 +136,12 @@ class Visibility(Enum): class Register(BaseModel): address: Annotated[ int, - Field( - le=255, description="Specifies the unique 8-bit address of the register." - ), + Field(le=255, description="Specifies the unique 8-bit address of the register."), ] type: Annotated[PayloadType, BeforeValidator(lambda v: PayloadType[v])] length: Annotated[ Optional[int], - Field( - ge=1, default=1, description="Specifies the length of the register payload." - ), + Field(ge=1, default=1, description="Specifies the length of the register payload."), ] access: Union[Access, List[Access]] = Field( ..., description="Specifies the expected use of the register." @@ -191,12 +187,6 @@ class Registers(BaseModel): class Model(Registers): device: str = Field(..., description="Specifies the name of the device.") - whoAmI: int = Field( - ..., description="Specifies the unique identifier for this device type." - ) - firmwareVersion: str = Field( - ..., description="Specifies the semantic version of the device firmware." - ) - hardwareTargets: str = Field( - ..., description="Specifies the semantic version of the device hardware." - ) + whoAmI: int = Field(..., description="Specifies the unique identifier for this device type.") + firmwareVersion: str = Field(..., description="Specifies the semantic version of the device firmware.") + hardwareTargets: str = Field(..., description="Specifies the semantic version of the device hardware.") diff --git a/harp/reader.py b/harp/reader.py index 6b3645c..e47f54e 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -28,8 +28,7 @@ def __call__( file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None, epoch: Optional[datetime] = None, keep_type: bool = False, - ) -> DataFrame: - ... + ) -> DataFrame: ... class RegisterReader: @@ -50,9 +49,7 @@ class RegisterMap(UserDict[str, RegisterReader]): def __init__(self, registers: Mapping[str, RegisterReader]) -> None: super().__init__(registers) - self._address_map = { - value.register.address: value for value in registers.values() - } + self._address_map = {value.register.address: value for value in registers.values()} def __getitem__(self, __key: Union[str, int]) -> RegisterReader: if isinstance(__key, int): @@ -266,9 +263,7 @@ def create_reader( base_path = path / device.device if is_dir else path.parent / device.device reg_readers = { - name: _create_register_parser( - device, name, _ReaderParams(base_path, epoch, keep_type) - ) + name: _create_register_parser(device, name, _ReaderParams(base_path, epoch, keep_type)) for name in device.registers.keys() } return DeviceReader(device, reg_readers) diff --git a/harp/schema.py b/harp/schema.py index 1bb1b0a..67531a9 100644 --- a/harp/schema.py +++ b/harp/schema.py @@ -11,9 +11,7 @@ def _read_common_registers() -> Registers: return parse_yaml_raw_as(Registers, fileIO.read()) -def read_schema( - file: Union[str, PathLike, TextIO], include_common_registers: bool = True -) -> Model: +def read_schema(file: Union[str, PathLike, TextIO], include_common_registers: bool = True) -> Model: """Read and parse a device schema from the specified file. Parameters @@ -40,9 +38,7 @@ def read_schema( schema.registers = dict(common.registers, **schema.registers) if common.bitMasks: schema.bitMasks = ( - common.bitMasks - if schema.bitMasks is None - else dict(common.bitMasks, **schema.bitMasks) + common.bitMasks if schema.bitMasks is None else dict(common.bitMasks, **schema.bitMasks) ) if common.groupMasks: schema.groupMasks = ( diff --git a/tests/test_io.py b/tests/test_io.py index 1c056d7..4459be0 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -26,9 +26,7 @@ expected_error=ValueError, ), DataFileParam(path="data/write_0.bin", expected_address=0, expected_rows=4), - DataFileParam( - path="data/write_0.bin", expected_address=0, expected_rows=4, keep_type=True - ), + DataFileParam(path="data/write_0.bin", expected_address=0, expected_rows=4, keep_type=True), ] @@ -45,10 +43,7 @@ def test_read(dataFile: DataFileParam): ) assert len(data) == dataFile.expected_rows if dataFile.keep_type: - assert ( - MessageType.__name__ in data.columns - and data[MessageType.__name__].dtype == "category" - ) + assert MessageType.__name__ in data.columns and data[MessageType.__name__].dtype == "category" if dataFile.expected_cols: for col in dataFile.expected_cols: From 41b5c7aff4457bdb8e75319e70b5aac89942d1f6 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 21 May 2024 01:20:36 +0100 Subject: [PATCH 08/31] Avoid importing typing_extensions --- harp/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/harp/model.py b/harp/model.py index 891eeb9..822f281 100644 --- a/harp/model.py +++ b/harp/model.py @@ -5,8 +5,7 @@ from __future__ import annotations from enum import Enum -from typing import Dict, List, Optional, Union -from typing_extensions import Annotated +from typing import Dict, List, Optional, Union, Annotated from pydantic import ( BaseModel, From df62f7580d3efe0c94d3cf61a02fd5e31c4e03ea Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 21 May 2024 01:21:54 +0100 Subject: [PATCH 09/31] Run isort formatting --- harp/io.py | 3 ++- harp/model.py | 2 +- harp/reader.py | 14 ++++++++------ harp/schema.py | 6 ++++-- tests/params.py | 6 ++++-- tests/test_io.py | 8 +++++--- tests/test_reader.py | 1 + tests/test_schema.py | 1 + 8 files changed, 26 insertions(+), 15 deletions(-) diff --git a/harp/io.py b/harp/io.py index cb7a702..b87c1f7 100644 --- a/harp/io.py +++ b/harp/io.py @@ -2,9 +2,10 @@ from enum import IntEnum from os import PathLike from typing import Any, BinaryIO, Optional, Union -from pandas._typing import Axes + import numpy as np import pandas as pd +from pandas._typing import Axes REFERENCE_EPOCH = datetime(1904, 1, 1) """The reference epoch for UTC harp time.""" diff --git a/harp/model.py b/harp/model.py index 822f281..b6f5031 100644 --- a/harp/model.py +++ b/harp/model.py @@ -5,7 +5,7 @@ from __future__ import annotations from enum import Enum -from typing import Dict, List, Optional, Union, Annotated +from typing import Annotated, Dict, List, Optional, Union from pydantic import ( BaseModel, diff --git a/harp/reader.py b/harp/reader.py index e47f54e..ea92e2e 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -1,17 +1,19 @@ import os +from collections import UserDict +from dataclasses import dataclass +from datetime import datetime +from functools import partial from math import log2 from os import PathLike from pathlib import Path -from datetime import datetime -from functools import partial -from dataclasses import dataclass +from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union + from numpy import dtype from pandas import DataFrame, Series -from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union -from collections import UserDict from pandas._typing import Axes -from harp.model import BitMask, GroupMask, Model, PayloadMember, Register + from harp.io import MessageType, read +from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.schema import read_schema diff --git a/harp/schema.py b/harp/schema.py index 67531a9..7abcdba 100644 --- a/harp/schema.py +++ b/harp/schema.py @@ -1,8 +1,10 @@ +from importlib import resources from os import PathLike from typing import TextIO, Union -from harp.model import Model, Registers + from pydantic_yaml import parse_yaml_raw_as -from importlib import resources + +from harp.model import Model, Registers def _read_common_registers() -> Registers: diff --git a/tests/params.py b/tests/params.py index 17335ff..9040a26 100644 --- a/tests/params.py +++ b/tests/params.py @@ -1,8 +1,10 @@ -import numpy as np +from dataclasses import dataclass from os import PathLike from pathlib import Path -from dataclasses import dataclass from typing import Iterable, Optional, Type, Union + +import numpy as np + from harp.model import Model datapath = Path(__file__).parent diff --git a/tests/test_io.py b/tests/test_io.py index 4459be0..25c5401 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,8 +1,10 @@ -import pytest -import numpy as np from contextlib import nullcontext + +import numpy as np +import pytest from pytest import mark -from harp.io import read, MessageType + +from harp.io import MessageType, read from tests.params import DataFileParam testdata = [ diff --git a/tests/test_reader.py b/tests/test_reader.py index 4053232..ba038f8 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,5 +1,6 @@ import numpy as np from pytest import mark + from harp.io import REFERENCE_EPOCH, MessageType from harp.reader import create_reader from tests.params import DeviceSchemaParam diff --git a/tests/test_schema.py b/tests/test_schema.py index 87dd003..160c540 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,4 +1,5 @@ from pytest import mark + from harp.schema import read_schema from tests.params import DeviceSchemaParam From 177c4aff8cfe7cad258d9a72cc3e95aaa4baa217 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 21 May 2024 01:26:14 +0100 Subject: [PATCH 10/31] Remove redundant file mode --- harp/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harp/schema.py b/harp/schema.py index 7abcdba..aa5d5e9 100644 --- a/harp/schema.py +++ b/harp/schema.py @@ -9,7 +9,7 @@ def _read_common_registers() -> Registers: file = resources.files(__package__) / "common.yml" - with file.open("rt") as fileIO: + with file.open("r") as fileIO: return parse_yaml_raw_as(Registers, fileIO.read()) From f684ed3a73b80ca94a84b5728a7d036d0f6f0cb7 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 22 Oct 2024 12:15:08 +0100 Subject: [PATCH 11/31] Avoid silent overflow for numpy compatibility --- harp/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harp/io.py b/harp/io.py index b87c1f7..374018d 100644 --- a/harp/io.py +++ b/harp/io.py @@ -90,7 +90,7 @@ def read( micros = np.ndarray(nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride) payloadoffset += 2 time = micros * _SECONDS_PER_TICK + seconds - payloadtype = payloadtype & ~0x10 + payloadtype = payloadtype & ~np.uint8(0x10) if epoch is not None: time = epoch + pd.to_timedelta(time, "s") # type: ignore index = pd.Series(time) From 750bb53d854eb9bcbbb1761b6dcc9d52ca84cac7 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Tue, 29 Oct 2024 10:26:05 +0000 Subject: [PATCH 12/31] Ensure stride uses default int type --- harp/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harp/io.py b/harp/io.py index 374018d..8c4bf96 100644 --- a/harp/io.py +++ b/harp/io.py @@ -80,7 +80,7 @@ def read( raise ValueError(f"expected address {address} but got {data[2]}") index = None - stride = data[1] + 2 + stride = int(data[1] + 2) nrows = len(data) // stride payloadtype = data[4] payloadoffset = 5 From 6ee0dc45e5148583fa211c8d5fd15e77a74e09fe Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 30 Oct 2024 11:23:04 +0000 Subject: [PATCH 13/31] Replace black and isort with ruff and pyright --- pyproject.toml | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7b4d5c3..5087448 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,9 +31,10 @@ classifiers = [ [project.optional-dependencies] dev = [ "datamodel-code-generator", + "pandas-stubs", "pytest", - "black", - "isort", + "pyright", + "ruff", "codespell" ] @@ -58,23 +59,20 @@ include = ["harp*"] [tool.setuptools_scm] -[tool.black] +[tool.ruff] line-length = 108 -target-version = ['py39'] -include = '\.pyi?$' -extend-exclude = ''' -# A regex preceded with ^/ will apply only to files and directories -# in the root of the project. -( - ^/LICENSE - ^/README.md - | reflex-generator -) -''' +target-version = "py39" +exclude = [ + "reflex-generator" +] -[tool.isort] -profile = 'black' -extend_skip = 'reflex-generator' +[tool.pyright] +venvPath = "." +venv = ".venv" +exclude = [ + ".venv/*", + "reflex-generator" +] [tool.codespell] skip = '.git,*.pdf,*.svg' From ec7db7bca8bc6e963a42c2e372c0f500201cfd94 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 30 Oct 2024 12:05:33 +0000 Subject: [PATCH 14/31] Add CI pipeline and code coverage --- .github/workflows/build.yml | 47 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 48 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..020c760 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,47 @@ +# Builds the python environment; linter and formatting via ruff; type annotations via pyright; +# tests via pytest; reports test coverage via pytest-cov. + +name: build +on: + push: + branches: ['*'] + pull_request: + workflow_dispatch: + +jobs: + build_run_tests: + name: Build env using pip and pyproject.toml on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + if: github.event.pull_request.draft == false + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [3.9, 3.11] + fail-fast: false + defaults: + run: + shell: ${{ matrix.os == 'windows-latest' && 'cmd' || 'bash' }} -l {0} # Adjust shell based on OS + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Create venv and install dependencies + run: | + python -m venv .venv + .venv/Scripts/activate || source .venv/bin/activate + pip install -e .[dev] + - name: Activate venv for later steps + run: | + echo "VIRTUAL_ENV=$(pwd)/.venv" >> $GITHUB_ENV + echo "$(pwd)/.venv/bin" >> $GITHUB_PATH # For Unix-like systems + echo "$(pwd)/.venv/Scripts" >> $GITHUB_PATH # For Windows + + - name: ruff + run: ruff check . + - name: pyright + run: pyright . + - name: pytest + run: pytest --cov harp \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5087448..d038094 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dev = [ "datamodel-code-generator", "pandas-stubs", "pytest", + "pytest-cov", "pyright", "ruff", "codespell" From 110250ea4cb520f209e98a83c241963bbed15548 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 30 Oct 2024 12:08:22 +0000 Subject: [PATCH 15/31] Apply ruff recommendations --- harp/__init__.py | 2 ++ harp/schema.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/harp/__init__.py b/harp/__init__.py index 33eef38..d3581a5 100644 --- a/harp/__init__.py +++ b/harp/__init__.py @@ -1,3 +1,5 @@ from harp.io import REFERENCE_EPOCH, MessageType, read from harp.reader import create_reader from harp.schema import read_schema + +__all__ = ["REFERENCE_EPOCH", "MessageType", "read", "create_reader", "read_schema"] diff --git a/harp/schema.py b/harp/schema.py index aa5d5e9..cee1532 100644 --- a/harp/schema.py +++ b/harp/schema.py @@ -35,7 +35,7 @@ def read_schema(file: Union[str, PathLike, TextIO], include_common_registers: bo return read_schema(fileIO) else: schema = parse_yaml_raw_as(Model, file.read()) - if not "WhoAmI" in schema.registers and include_common_registers: + if "WhoAmI" not in schema.registers and include_common_registers: common = _read_common_registers() schema.registers = dict(common.registers, **schema.registers) if common.bitMasks: From ff87f12b78b3cf37cc69283abbebf8764bab897f Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 30 Oct 2024 12:11:42 +0000 Subject: [PATCH 16/31] Apply pyright recommendations --- harp/reader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index ea92e2e..b1b4961 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -53,11 +53,11 @@ def __init__(self, registers: Mapping[str, RegisterReader]) -> None: super().__init__(registers) self._address_map = {value.register.address: value for value in registers.values()} - def __getitem__(self, __key: Union[str, int]) -> RegisterReader: - if isinstance(__key, int): - return self._address_map[__key] + def __getitem__(self, key: Union[str, int]) -> RegisterReader: + if isinstance(key, int): + return self._address_map[key] else: - return super().__getitem__(__key) + return super().__getitem__(key) class DeviceReader: From 3f7cb51670b5ee7fc36863a55165721a0122b334 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 30 Oct 2024 12:14:08 +0000 Subject: [PATCH 17/31] Update matrix job names --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 020c760..6ab47b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ on: jobs: build_run_tests: - name: Build env using pip and pyproject.toml on ${{ matrix.os }} + name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} if: github.event.pull_request.draft == false strategy: From 41d9038de4c666382916845bf6a03c9a0446c980 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 30 Oct 2024 13:29:14 +0000 Subject: [PATCH 18/31] Avoid using environment in CI pipeline --- .github/workflows/build.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ab47b1..a184f9f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,9 +18,6 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] python-version: [3.9, 3.11] fail-fast: false - defaults: - run: - shell: ${{ matrix.os == 'windows-latest' && 'cmd' || 'bash' }} -l {0} # Adjust shell based on OS steps: - name: Checkout uses: actions/checkout@v4 @@ -28,16 +25,8 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Create venv and install dependencies - run: | - python -m venv .venv - .venv/Scripts/activate || source .venv/bin/activate - pip install -e .[dev] - - name: Activate venv for later steps - run: | - echo "VIRTUAL_ENV=$(pwd)/.venv" >> $GITHUB_ENV - echo "$(pwd)/.venv/bin" >> $GITHUB_PATH # For Unix-like systems - echo "$(pwd)/.venv/Scripts" >> $GITHUB_PATH # For Windows + - name: Install dependencies + run: pip install -e .[dev] - name: ruff run: ruff check . From 849f30d0d12261b1e1198e9651fd0889a41314bb Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 30 Oct 2024 13:33:57 +0000 Subject: [PATCH 19/31] Cache pip dependencies --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a184f9f..115929b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: pip install -e .[dev] From 9c8bb7db7cdd86903d4d1b7063a1fe4cf9cdbebd Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:35:57 -0700 Subject: [PATCH 20/31] Add new constructor methods --- harp/reader.py | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/harp/reader.py b/harp/reader.py index b1b4961..8077705 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -15,6 +15,7 @@ from harp.io import MessageType, read from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.schema import read_schema +import requests @dataclass @@ -74,6 +75,117 @@ def __dir__(self) -> Iterable[str]: def __getattr__(self, __name: str) -> RegisterReader: return self.registers[__name] + @staticmethod + def from_file( + filepath: PathLike, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + device = read_schema(filepath, include_common_registers) + if base_path is None: + path = Path(filepath).absolute().resolve() + base_path = path.parent / device.device + else: + base_path = Path(base_path).absolute().resolve() / device.device + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_url( + url: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + timeout: int = 5) -> "DeviceReader": + + response = requests.get(url, timeout=timeout) + text = response.text + + device = read_schema(text, include_common_registers) + if base_path is None: + base_path = Path(device.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_str( + schema: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + device = read_schema(schema, include_common_registers) + if base_path is None: + base_path = Path(device.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_model( + model: Model, + base_path: Optional[PathLike] = None, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + if base_path is None: + base_path = Path(model.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + model, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in model.registers.keys() + } + return DeviceReader(model, reg_readers) + + @staticmethod + def from_dataset( + dataset: PathLike, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + path = Path(dataset).absolute().resolve() + is_dir = os.path.isdir(path) + if is_dir: + filepath = path / "device.yml" + return DeviceReader.from_file( + filepath=filepath, + base_path=path, + include_common_registers=include_common_registers, + epoch=epoch, + keep_type=keep_type) + else: + raise ValueError("The dataset must be a directory containing a device.yml file.") + def _compose_parser( f: Callable[[DataFrame], DataFrame], From a8909915515eaca0eb62f6d6d3642ef467c3349c Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:44:56 -0700 Subject: [PATCH 21/31] Deprecate function --- harp/reader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index 8077705..6f33aa1 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -15,7 +15,6 @@ from harp.io import MessageType, read from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.schema import read_schema -import requests @dataclass @@ -334,7 +333,7 @@ def parser(df: DataFrame): reader = partial(reader, columns=[name]) return RegisterReader(register, reader) - +@deprecated("This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead.") def create_reader( device: Union[str, PathLike, Model], include_common_registers: bool = True, From 1b3d4385c1ee354d897793ce5969fc956feb3f84 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:55:49 -0700 Subject: [PATCH 22/31] Document methods --- harp/reader.py | 173 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 149 insertions(+), 24 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index 6f33aa1..8981d8a 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -76,11 +76,37 @@ def __getattr__(self, __name: str) -> RegisterReader: @staticmethod def from_file( - filepath: PathLike, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + filepath: PathLike, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified schema yml file. + + Parameters + ---------- + filepath + A path to the device yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ device = read_schema(filepath, include_common_registers) if base_path is None: @@ -99,12 +125,39 @@ def from_file( @staticmethod def from_url( - url: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False, - timeout: int = 5) -> "DeviceReader": + url: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + timeout: int = 5, + ) -> "DeviceReader": + """Creates a device reader object from a url pointing to a device.yml file. + + Parameters + ---------- + url + The url pointing to the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + timeout + The number of seconds to wait for the server to send data before giving up. + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ response = requests.get(url, timeout=timeout) text = response.text @@ -125,11 +178,37 @@ def from_url( @staticmethod def from_str( - schema: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + schema: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a string containing a device.yml schema. + + Parameters + ---------- + schema + The string containing the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ device = read_schema(schema, include_common_registers) if base_path is None: @@ -147,10 +226,32 @@ def from_str( @staticmethod def from_model( - model: Model, - base_path: Optional[PathLike] = None, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + model: Model, + base_path: Optional[PathLike] = None, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a parsed device schema object. + + Parameters + ---------- + model + The parsed device schema object describing the device. + base_path + The path to attempt to resolve the location of data files. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ if base_path is None: base_path = Path(model.device).absolute().resolve() @@ -167,10 +268,34 @@ def from_model( @staticmethod def from_dataset( - dataset: PathLike, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + dataset: PathLike, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified dataset folder. + + Parameters + ---------- + dataset + A path to the dataset folder containing a device.yml schema describing the device. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ path = Path(dataset).absolute().resolve() is_dir = os.path.isdir(path) From a76c6f0ff7bd4b824f3ace9bd3f7785dfb667637 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:55:57 -0700 Subject: [PATCH 23/31] Linting --- harp/model.py | 11 +++-------- harp/reader.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/harp/model.py b/harp/model.py index b6f5031..5884b0e 100644 --- a/harp/model.py +++ b/harp/model.py @@ -7,14 +7,9 @@ from enum import Enum from typing import Annotated, Dict, List, Optional, Union -from pydantic import ( - BaseModel, - BeforeValidator, - ConfigDict, - Field, - RootModel, - field_serializer, -) +from pydantic import (BaseModel, BeforeValidator, ConfigDict, Field, RootModel, + field_serializer) +from typing_extensions import Annotated class PayloadType(str, Enum): diff --git a/harp/reader.py b/harp/reader.py index 8981d8a..bbeb2ac 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -306,9 +306,12 @@ def from_dataset( base_path=path, include_common_registers=include_common_registers, epoch=epoch, - keep_type=keep_type) + keep_type=keep_type, + ) else: - raise ValueError("The dataset must be a directory containing a device.yml file.") + raise ValueError( + "The dataset must be a directory containing a device.yml file." + ) def _compose_parser( @@ -458,7 +461,10 @@ def parser(df: DataFrame): reader = partial(reader, columns=[name]) return RegisterReader(register, reader) -@deprecated("This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead.") + +@deprecated( + "This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead." +) def create_reader( device: Union[str, PathLike, Model], include_common_registers: bool = True, From 828cd3bac0a452ab4f98652776289ade6b2809be Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:10:36 +0000 Subject: [PATCH 24/31] Fix rebasing --- harp/model.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/harp/model.py b/harp/model.py index 1010fd1..09f30c2 100644 --- a/harp/model.py +++ b/harp/model.py @@ -7,13 +7,7 @@ from enum import Enum from typing import Annotated, Dict, List, Optional, Union -from pydantic import (BaseModel, BeforeValidator, ConfigDict, Field, RootModel, - field_serializer) -from typing import Dict, List, Optional, Union - -from pydantic import (BaseModel, BeforeValidator, ConfigDict, Field, RootModel, - field_serializer) -from typing_extensions import Annotated +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, RootModel, field_serializer class PayloadType(str, Enum): From 13a68626983a075ba130ffd7f74835f9afd1f349 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:10:51 +0000 Subject: [PATCH 25/31] Add deprecated decorator --- harp/__init__.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/harp/__init__.py b/harp/__init__.py index d3581a5..e7e7877 100644 --- a/harp/__init__.py +++ b/harp/__init__.py @@ -1,5 +1,26 @@ from harp.io import REFERENCE_EPOCH, MessageType, read from harp.reader import create_reader from harp.schema import read_schema +import warnings +import functools + __all__ = ["REFERENCE_EPOCH", "MessageType", "read", "create_reader", "read_schema"] + + +def deprecated(message): + # This decorator is only available from the stdlib warnings module in Python 3.13 + # Making it available here for compatibility with older versions + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + warnings.warn( + f"Call to deprecated function {func.__name__}: {message}", + category=DeprecationWarning, + stacklevel=1, + ) + return func(*args, **kwargs) + + return wrapper + + return decorator From 3cefa8e2b785698645f273f8df23e918d56461b4 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:11:32 +0000 Subject: [PATCH 26/31] Fix rebasing --- harp/reader.py | 243 +------------------------------------------------ 1 file changed, 1 insertion(+), 242 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index 46ccd83..1a7c773 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -313,248 +313,7 @@ def from_dataset( keep_type=keep_type, ) else: - raise ValueError( - "The dataset must be a directory containing a device.yml file." - ) - - @staticmethod - def from_file( - filepath: PathLike, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False, - ) -> "DeviceReader": - """Creates a device reader object from the specified schema yml file. - - Parameters - ---------- - filepath - A path to the device yml schema describing the device. - base_path - The path to attempt to resolve the location of data files. - include_common_registers - Specifies whether to include the set of Harp common registers in the - parsed device schema object. If a parsed device schema object is provided, - this parameter is ignored. - epoch - The default reference datetime at which time zero begins. If specified, - the data frames returned by each register reader will have a datetime index. - keep_type - Specifies whether to include a column with the message type by default. - - Returns - ------- - A device reader object which can be used to read binary data for each - register or to access metadata about each register. Individual registers - can be accessed using dot notation using the name of the register as the - key. - """ - - device = read_schema(filepath, include_common_registers) - if base_path is None: - path = Path(filepath).absolute().resolve() - base_path = path.parent / device.device - else: - base_path = Path(base_path).absolute().resolve() / device.device - - reg_readers = { - name: _create_register_parser( - device, name, _ReaderParams(base_path, epoch, keep_type) - ) - for name in device.registers.keys() - } - return DeviceReader(device, reg_readers) - - @staticmethod - def from_url( - url: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False, - timeout: int = 5, - ) -> "DeviceReader": - """Creates a device reader object from a url pointing to a device.yml file. - - Parameters - ---------- - url - The url pointing to the device.yml schema describing the device. - base_path - The path to attempt to resolve the location of data files. - include_common_registers - Specifies whether to include the set of Harp common registers in the - parsed device schema object. If a parsed device schema object is provided, - this parameter is ignored. - epoch - The default reference datetime at which time zero begins. If specified, - the data frames returned by each register reader will have a datetime index. - keep_type - Specifies whether to include a column with the message type by default. - timeout - The number of seconds to wait for the server to send data before giving up. - Returns - ------- - A device reader object which can be used to read binary data for each - register or to access metadata about each register. Individual registers - can be accessed using dot notation using the name of the register as the - key. - """ - - response = requests.get(url, timeout=timeout) - text = response.text - - device = read_schema(text, include_common_registers) - if base_path is None: - base_path = Path(device.device).absolute().resolve() - else: - base_path = Path(base_path).absolute().resolve() - - reg_readers = { - name: _create_register_parser( - device, name, _ReaderParams(base_path, epoch, keep_type) - ) - for name in device.registers.keys() - } - return DeviceReader(device, reg_readers) - - @staticmethod - def from_str( - schema: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False, - ) -> "DeviceReader": - """Creates a device reader object from a string containing a device.yml schema. - - Parameters - ---------- - schema - The string containing the device.yml schema describing the device. - base_path - The path to attempt to resolve the location of data files. - include_common_registers - Specifies whether to include the set of Harp common registers in the - parsed device schema object. If a parsed device schema object is provided, - this parameter is ignored. - epoch - The default reference datetime at which time zero begins. If specified, - the data frames returned by each register reader will have a datetime index. - keep_type - Specifies whether to include a column with the message type by default. - - Returns - ------- - A device reader object which can be used to read binary data for each - register or to access metadata about each register. Individual registers - can be accessed using dot notation using the name of the register as the - key. - """ - - device = read_schema(schema, include_common_registers) - if base_path is None: - base_path = Path(device.device).absolute().resolve() - else: - base_path = Path(base_path).absolute().resolve() - - reg_readers = { - name: _create_register_parser( - device, name, _ReaderParams(base_path, epoch, keep_type) - ) - for name in device.registers.keys() - } - return DeviceReader(device, reg_readers) - - @staticmethod - def from_model( - model: Model, - base_path: Optional[PathLike] = None, - epoch: Optional[datetime] = None, - keep_type: bool = False, - ) -> "DeviceReader": - """Creates a device reader object from a parsed device schema object. - - Parameters - ---------- - model - The parsed device schema object describing the device. - base_path - The path to attempt to resolve the location of data files. - epoch - The default reference datetime at which time zero begins. If specified, - the data frames returned by each register reader will have a datetime index. - keep_type - Specifies whether to include a column with the message type by default. - - Returns - ------- - A device reader object which can be used to read binary data for each - register or to access metadata about each register. Individual registers - can be accessed using dot notation using the name of the register as the - key. - """ - - if base_path is None: - base_path = Path(model.device).absolute().resolve() - else: - base_path = Path(base_path).absolute().resolve() - - reg_readers = { - name: _create_register_parser( - model, name, _ReaderParams(base_path, epoch, keep_type) - ) - for name in model.registers.keys() - } - return DeviceReader(model, reg_readers) - - @staticmethod - def from_dataset( - dataset: PathLike, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False, - ) -> "DeviceReader": - """Creates a device reader object from the specified dataset folder. - - Parameters - ---------- - dataset - A path to the dataset folder containing a device.yml schema describing the device. - include_common_registers - Specifies whether to include the set of Harp common registers in the - parsed device schema object. If a parsed device schema object is provided, - this parameter is ignored. - epoch - The default reference datetime at which time zero begins. If specified, - the data frames returned by each register reader will have a datetime index. - keep_type - Specifies whether to include a column with the message type by default. - - Returns - ------- - A device reader object which can be used to read binary data for each - register or to access metadata about each register. Individual registers - can be accessed using dot notation using the name of the register as the - key. - """ - - path = Path(dataset).absolute().resolve() - is_dir = os.path.isdir(path) - if is_dir: - filepath = path / "device.yml" - return DeviceReader.from_file( - filepath=filepath, - base_path=path, - include_common_registers=include_common_registers, - epoch=epoch, - keep_type=keep_type, - ) - else: - raise ValueError( - "The dataset must be a directory containing a device.yml file." - ) + raise ValueError("The dataset must be a directory containing a device.yml file.") def _compose_parser( From 89870d9c953ec525409fa9b3213043b7384f0462 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:12:17 +0000 Subject: [PATCH 27/31] Favor library's deprecated decorator --- harp/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harp/reader.py b/harp/reader.py index 1a7c773..aae311b 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -10,7 +10,7 @@ Protocol, Union) import requests -from deprecated import deprecated +from harp import deprecated from numpy import dtype from pandas import DataFrame, Series from pandas._typing import Axes From 83397d146641ec8871a72bdfb549959f3f63251b Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:12:32 +0000 Subject: [PATCH 28/31] Linting --- harp/reader.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index aae311b..ebc0d28 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -6,8 +6,7 @@ from math import log2 from os import PathLike from pathlib import Path -from typing import (Any, BinaryIO, Callable, Iterable, Mapping, Optional, - Protocol, Union) +from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union import requests from harp import deprecated @@ -15,7 +14,6 @@ from pandas import DataFrame, Series from pandas._typing import Axes -from harp import __version__ from harp.io import MessageType, read from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.schema import read_schema @@ -120,9 +118,7 @@ def from_file( base_path = Path(base_path).absolute().resolve() / device.device reg_readers = { - name: _create_register_parser( - device, name, _ReaderParams(base_path, epoch, keep_type) - ) + name: _create_register_parser(device, name, _ReaderParams(base_path, epoch, keep_type)) for name in device.registers.keys() } return DeviceReader(device, reg_readers) @@ -173,9 +169,7 @@ def from_url( base_path = Path(base_path).absolute().resolve() reg_readers = { - name: _create_register_parser( - device, name, _ReaderParams(base_path, epoch, keep_type) - ) + name: _create_register_parser(device, name, _ReaderParams(base_path, epoch, keep_type)) for name in device.registers.keys() } return DeviceReader(device, reg_readers) @@ -221,9 +215,7 @@ def from_str( base_path = Path(base_path).absolute().resolve() reg_readers = { - name: _create_register_parser( - device, name, _ReaderParams(base_path, epoch, keep_type) - ) + name: _create_register_parser(device, name, _ReaderParams(base_path, epoch, keep_type)) for name in device.registers.keys() } return DeviceReader(device, reg_readers) @@ -263,9 +255,7 @@ def from_model( base_path = Path(base_path).absolute().resolve() reg_readers = { - name: _create_register_parser( - model, name, _ReaderParams(base_path, epoch, keep_type) - ) + name: _create_register_parser(model, name, _ReaderParams(base_path, epoch, keep_type)) for name in model.registers.keys() } return DeviceReader(model, reg_readers) From 48dadeb7dfbaa930739a611302c5d73a96976a66 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:16:46 +0000 Subject: [PATCH 29/31] Add requests as dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d038094..8833988 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,8 @@ license = {text = "MIT License"} dependencies = [ "pydantic-yaml", - "pandas" + "pandas", + "requests" ] classifiers = [ From 4c39bd89b2349b075b0b20607b1ff9fb6b6ea696 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:22:58 +0000 Subject: [PATCH 30/31] Move decorator to separate private helper module to prevent circular dependencies --- harp/__init__.py | 20 -------------------- harp/_helpers.py | 20 ++++++++++++++++++++ harp/reader.py | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 harp/_helpers.py diff --git a/harp/__init__.py b/harp/__init__.py index e7e7877..962e5fe 100644 --- a/harp/__init__.py +++ b/harp/__init__.py @@ -1,26 +1,6 @@ from harp.io import REFERENCE_EPOCH, MessageType, read from harp.reader import create_reader from harp.schema import read_schema -import warnings -import functools __all__ = ["REFERENCE_EPOCH", "MessageType", "read", "create_reader", "read_schema"] - - -def deprecated(message): - # This decorator is only available from the stdlib warnings module in Python 3.13 - # Making it available here for compatibility with older versions - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"Call to deprecated function {func.__name__}: {message}", - category=DeprecationWarning, - stacklevel=1, - ) - return func(*args, **kwargs) - - return wrapper - - return decorator diff --git a/harp/_helpers.py b/harp/_helpers.py new file mode 100644 index 0000000..0dbec61 --- /dev/null +++ b/harp/_helpers.py @@ -0,0 +1,20 @@ +import warnings +import functools + + +def deprecated(message): + # This decorator is only available from the stdlib warnings module in Python 3.13 + # Making it available here for compatibility with older versions + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + warnings.warn( + f"Call to deprecated function {func.__name__}: {message}", + category=DeprecationWarning, + stacklevel=1, + ) + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/harp/reader.py b/harp/reader.py index ebc0d28..e3fec6c 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -9,7 +9,7 @@ from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union import requests -from harp import deprecated +from harp._helpers import deprecated from numpy import dtype from pandas import DataFrame, Series from pandas._typing import Axes From 132cb738883a2510c102f0084c7805a6c3213c86 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:24:46 +0000 Subject: [PATCH 31/31] Ignore warnings emitted by deprecated decorator --- tests/test_reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_reader.py b/tests/test_reader.py index ba038f8..842ce63 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -20,6 +20,7 @@ @mark.parametrize("schemaFile", testdata) +@mark.filterwarnings("ignore:Call to deprecated") def test_create_reader(schemaFile: DeviceSchemaParam): reader = create_reader(schemaFile.path, epoch=REFERENCE_EPOCH) schemaFile.assert_schema(reader.device)