Skip to content

Commit

Permalink
Merge pull request #196 from PowerGridModel/feature/pgm-serializer
Browse files Browse the repository at this point in the history
Bump power grid model to 1.6.x
  • Loading branch information
TonyXiang8787 authored Oct 6, 2023
2 parents 4245735 + ed81435 commit 085bd91
Show file tree
Hide file tree
Showing 10 changed files with 500 additions and 163 deletions.
7 changes: 4 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ repos:
hooks:
- id: reuse
- repo: https://github.com/pycqa/isort
rev: 5.10.1
rev: 5.12.0
hooks:
- id: isort
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.991
rev: v1.2.0
hooks:
- id: mypy
additional_dependencies: [numpy, pandas]
Expand All @@ -25,6 +25,7 @@ repos:
- id: pylint
name: pylint
entry: pylint
files: ^src/.+\.py$
language: system
types: [ python ]
args: [ "--rcfile=pyproject.toml" ]
Expand All @@ -35,4 +36,4 @@ repos:
language: system
pass_filenames: false
always_run: true
args: [ "--cov-fail-under=95" ]
args: [ "--cov-fail-under=99" ]
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ dependencies = [
"numpy>=1.20",
"openpyxl",
"pandas",
"power_grid_model>=1.4, <1.6",
"power_grid_model>=1.6",
"pyyaml",
"structlog",
"tqdm",
Expand Down Expand Up @@ -76,7 +76,7 @@ power_grid_model_io = ["config/**/*.yaml"]

[tool.pytest.ini_options]
testpaths = ["tests/unit"]
addopts = ["--cov=power_grid_model_io", "--cov-report=term", "--cov-report=html:cov_html", "--cov-fail-under=100"]
addopts = ["--cov=power_grid_model_io", "--cov-report=term", "--cov-report=html:cov_html", "--cov-fail-under=99"]

[tool.black]
line-length = 120
Expand Down
21 changes: 11 additions & 10 deletions set_pypi_version.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ def set_version(pkg_dir: Path):


def get_pypi_latest():
r = requests.get("https://pypi.org/pypi/power-grid-model-io/json")
data = r.json()
request = requests.get("https://pypi.org/pypi/power-grid-model-io/json")
data = request.json()
version: str = data["info"]["version"]
return (int(x) for x in version.split("."))

Expand All @@ -57,16 +57,17 @@ def get_new_version(major, minor, latest_major, latest_minor, latest_patch):
if (major > latest_major) or ((major == latest_major) and minor > latest_minor):
# brand-new version with patch zero
return f"{major}.{minor}.0"
elif major == latest_major and minor == latest_minor:

if major == latest_major and minor == latest_minor:
# current version, increment path
return f"{major}.{minor}.{latest_patch + 1}"
else:
# does not allow building older version
raise ValueError(
"Invalid version number!\n"
f"latest version: {latest_major}.{latest_minor}.{latest_patch}\n"
f"to be built version: {major}.{minor}\n"
)

# does not allow building older version
raise ValueError(
"Invalid version number!\n"
f"latest version: {latest_major}.{latest_minor}.{latest_patch}\n"
f"to be built version: {major}.{minor}\n"
)


if __name__ == "__main__":
Expand Down
100 changes: 72 additions & 28 deletions src/power_grid_model_io/converters/pgm_json_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,16 @@
Power Grid Model 'Converter': Load and store power grid model data in the native PGM JSON format.
"""

import json
import warnings
from pathlib import Path
from typing import Optional, Union, cast
from typing import Any, Dict, List, Optional, Union

import numpy as np
from power_grid_model.data_types import BatchDataset, ComponentList, Dataset, SingleDataset, SinglePythonDataset
from power_grid_model.utils import (
convert_batch_dataset_to_batch_list,
convert_list_to_batch_data,
initialize_array,
is_nan,
)
from power_grid_model import initialize_array
from power_grid_model._utils import is_nan
from power_grid_model.data_types import ComponentList, Dataset, SingleDataset, SinglePythonDataset
from power_grid_model.utils import json_deserialize, json_serialize

from power_grid_model_io.converters.base_converter import BaseConverter
from power_grid_model_io.data_stores.json_file_store import JsonFileStore
Expand Down Expand Up @@ -65,14 +64,23 @@ def _parse_data(self, data: StructuredData, data_type: str, extra_info: Optional
"""
self._log.debug(f"Loading PGM {data_type} data")
if isinstance(data, list):
parsed_data = [
self._parse_dataset(data=dataset, data_type=data_type, extra_info=extra_info) for dataset in data
]
return convert_list_to_batch_data(parsed_data)
if not isinstance(data, dict):
raise TypeError("Raw data should be either a list or a dictionary!")
return self._parse_dataset(data=data, data_type=data_type, extra_info=extra_info)

result = json_deserialize(
json.dumps(
{
"attributes": {},
"data": data,
"is_batch": isinstance(data, list),
"type": data_type,
"version": "1.0",
}
)
)

if extra_info is not None:
self._extract_extra_info(original_data=data, deserialized_data=result, extra_info=extra_info)

return result

def _parse_dataset(
self, data: SinglePythonDataset, data_type: str, extra_info: Optional[ExtraInfo]
Expand Down Expand Up @@ -161,21 +169,17 @@ def _serialize_data(self, data: Dataset, extra_info: Optional[ExtraInfo]) -> Str
the function returns a structured dataset
"""
# Check if the dataset is a single dataset or batch dataset
# It is batch dataset if it is 2D array or a indptr/data structure
result = json.loads(json_serialize(data))["data"]

# If it is a batch, convert the batch data to a list of batches, then convert each batch individually.
if self._is_batch(data=data):
if extra_info is not None:
if extra_info is not None:
if self._is_batch(data=data):
self._log.warning("Extra info is not supported for batch data export")
# We have established that this is batch data, so let's tell the type checker that this is a BatchDataset
data = cast(BatchDataset, data)
list_data = convert_batch_dataset_to_batch_list(data)
return [self._serialize_dataset(data=x) for x in list_data]
else:
for component_data in result.values():
for component in component_data:
component.update(extra_info.get(component["id"], {}))

# We have established that this is not batch data, so let's tell the type checker that this is a SingleDataset
data = cast(SingleDataset, data)
return self._serialize_dataset(data=data, extra_info=extra_info)
return result

@staticmethod
def _is_batch(data: Dataset) -> bool:
Expand Down Expand Up @@ -245,3 +249,43 @@ def _serialize_dataset(data: SingleDataset, extra_info: Optional[ExtraInfo] = No
]
for component, objects in data.items()
}

def _extract_extra_info(
self, original_data: StructuredData, deserialized_data: SingleDataset, extra_info: ExtraInfo
) -> None:
if not isinstance(original_data, dict):
warnings.warn("Extracting extra info is not supported for batch data.")
return

reserialized_data = self._serialize_data(data=deserialized_data, extra_info=extra_info)
if len(original_data) != len(reserialized_data) or not isinstance(reserialized_data, dict):
warnings.warn("The extra info cannot be determined.")
return

for component, component_data in original_data.items():
for entry in component_data:
self._extract_extra_component_info(component, entry, reserialized_data, extra_info)

def _extract_extra_component_info(
self, component: str, attributes: Dict[str, Any], reserialized_data: SingleDataset, extra_info: ExtraInfo
):
entry_id = attributes["id"]
reserialized_entry = self._get_first_by(reserialized_data[component], "id", entry_id)
if reserialized_entry is None:
warnings.warn(f"The extra info cannot be determined for component '{component}' with ID {entry_id}")
return

for attribute, value in attributes.items():
if attribute not in reserialized_entry:
if entry_id not in extra_info:
extra_info[entry_id] = {}

extra_info[entry_id][attribute] = value

@staticmethod
def _get_first_by(data: List[Dict[str, Any]], field: str, value: Any) -> Optional[Dict[str, Any]]:
for entry in data:
if entry[field] == value:
return entry

return None
Loading

0 comments on commit 085bd91

Please sign in to comment.