Merge branch 'dev' into feat/store-matrices-as-hdf5

antarest/core/model.py

@@ -13,9 +13,6 @@
 import enum
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-import typing_extensions as te
-from pydantic import StringConstraints
-
 from antarest.core.serialization import AntaresBaseModel
 
 if TYPE_CHECKING:
@@ -25,7 +22,6 @@
 JSON = Dict[str, Any]
 ELEMENT = Union[str, int, float, bool, bytes]
 SUB_JSON = Union[ELEMENT, JSON, List[Any], None]
-LowerCaseStr = te.Annotated[str, StringConstraints(to_lower=True)]
 
 
 class PublicMode(enum.StrEnum):

antarest/launcher/extensions/adequacy_patch/extension.py

@@ -24,7 +24,7 @@
 from antarest.core.utils.utils import assert_this
 from antarest.launcher.extensions.interface import ILauncherExtension
 from antarest.study.service import StudyService
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 
 logger = logging.getLogger(__name__)

antarest/study/business/area_management.py

@@ -32,8 +32,7 @@
     ThermalAreasProperties,
     UIProperties,
 )
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
-from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, DistrictSet
+from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, DistrictSet, transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.model.command.create_area import CreateArea

antarest/study/business/areas/renewable_management.py

@@ -22,13 +22,12 @@
 from antarest.study.business.enum_ignore_case import EnumIgnoreCase
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.renewable import (
     RenewableConfig,
     RenewableConfigType,
     RenewableProperties,
     create_renewable_config,
-    create_renewable_properties,
 )
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
@@ -275,6 +274,7 @@ def update_cluster(
         Raises:
             RenewableClusterNotFound: If the cluster to update is not found.
         """
+
         study_version = StudyVersion.parse(study.version)
         file_study = self._get_file_study(study)
         path = _CLUSTER_PATH.format(area_id=area_id, cluster_id=cluster_id)
@@ -284,19 +284,19 @@ def update_cluster(
         except KeyError:
             raise RenewableClusterNotFound(path, cluster_id) from None
         else:
-            old_properties = create_renewable_properties(study_version, **values)
+            old_config = create_renewable_config(study_version, **values)
 
         # use Python values to synchronize Config and Form values
         new_values = cluster_data.model_dump(by_alias=False, exclude_none=True)
-        new_properties = old_properties.copy(exclude={"id"}, update=new_values)
+        new_config = old_config.copy(exclude={"id"}, update=new_values)
+        new_data = new_config.model_dump(mode="json", by_alias=True, exclude={"id"})
 
         # create the dict containing the new values using aliases
         data: t.Dict[str, t.Any] = {}
-        for updated_field, updated_value in new_values.items():
-            if updated_field in old_properties.model_fields:
-                field_info = old_properties.model_fields[updated_field]
-                field_name = field_info.alias if field_info.alias else updated_field
-                data[field_name] = updated_value
+        for field_name, field in new_config.model_fields.items():
+            if field_name in new_values:
+                name = field.alias if field.alias else field_name
+                data[name] = new_data[name]
 
         # create the update config commands with the modified data
         command_context = self.storage_service.variant_study_service.command_factory.command_context
@@ -308,7 +308,7 @@ def update_cluster(
         ]
         execute_or_add_commands(study, file_study, commands, self.storage_service)
 
-        values = new_properties.model_dump(by_alias=False)
+        values = new_config.model_dump(by_alias=False)
         return RenewableClusterOutput(**values, id=cluster_id)
 
     def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[str]) -> None:
@@ -357,8 +357,9 @@ def duplicate_cluster(
         Raises:
             DuplicateRenewableCluster: If a cluster with the new name already exists in the area.
         """
-        new_id = transform_name_to_id(new_cluster_name)
-        if any(new_id == cluster.id for cluster in self.get_clusters(study, area_id)):
+        new_id = transform_name_to_id(new_cluster_name, lower=False)
+        lower_new_id = new_id.lower()
+        if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)):
             raise DuplicateRenewableCluster(area_id, new_id)
 
         # Cluster duplication
@@ -370,8 +371,9 @@ def duplicate_cluster(
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
-        source_path = f"input/renewables/series/{area_id}/{source_id}/series"
-        new_path = f"input/renewables/series/{area_id}/{new_id}/series"
+        lower_source_id = source_id.lower()
+        source_path = f"input/renewables/series/{area_id}/{lower_source_id}/series"
+        new_path = f"input/renewables/series/{area_id}/{lower_new_id}/series"
 
         # Prepare and execute commands
         storage_service = self.storage_service.get_storage(study)

antarest/study/business/areas/st_storage_management.py

@@ -33,7 +33,7 @@
 from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import STUDY_VERSION_8_8, Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.st_storage import (
     STStorage880Config,
     STStorage880Properties,
@@ -305,7 +305,7 @@ def _make_create_cluster_cmd(
     ) -> CreateSTStorage:
         command = CreateSTStorage(
             area_id=area_id,
-            parameters=cluster.model_dump(mode="json", by_alias=True, exclude={"id"}),
+            parameters=cluster,
             command_context=self.storage_service.variant_study_service.command_factory.command_context,
             study_version=study_version,
         )
@@ -551,7 +551,8 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_clus
             ClusterAlreadyExists: If a cluster with the new name already exists in the area.
         """
         new_id = transform_name_to_id(new_cluster_name)
-        if any(new_id == storage.id for storage in self.get_storages(study, area_id)):
+        lower_new_id = new_id.lower()
+        if any(lower_new_id == storage.id.lower() for storage in self.get_storages(study, area_id)):
             raise DuplicateSTStorage(area_id, new_id)
 
         # Cluster duplication
@@ -570,13 +571,16 @@ def duplicate_cluster(self, study: Study, area_id: str, source_id: str, new_clus
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
+        lower_source_id = source_id.lower()
         # noinspection SpellCheckingInspection
         ts_names = ["pmax_injection", "pmax_withdrawal", "lower_rule_curve", "upper_rule_curve", "inflows"]
         source_paths = [
-            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=source_id, ts_name=ts_name) for ts_name in ts_names
+            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_source_id, ts_name=ts_name)
+            for ts_name in ts_names
         ]
         new_paths = [
-            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=new_id, ts_name=ts_name) for ts_name in ts_names
+            _STORAGE_SERIES_PATH.format(area_id=area_id, storage_id=lower_new_id, ts_name=ts_name)
+            for ts_name in ts_names
         ]
 
         # Prepare and execute commands

antarest/study/business/areas/thermal_management.py

@@ -28,13 +28,12 @@
 from antarest.study.business.all_optional_meta import all_optional_model, camel_case_model
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import STUDY_VERSION_8_7, Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.config.thermal import (
     Thermal870Config,
     Thermal870Properties,
     ThermalConfigType,
     create_thermal_config,
-    create_thermal_properties,
 )
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
@@ -349,6 +348,7 @@ def update_cluster(
             ThermalClusterNotFound: If the provided `cluster_id` does not match the ID of the cluster
             in the provided cluster_data.
         """
+
         study_version = StudyVersion.parse(study.version)
         file_study = self._get_file_study(study)
         path = _CLUSTER_PATH.format(area_id=area_id, cluster_id=cluster_id)
@@ -357,19 +357,19 @@ def update_cluster(
         except KeyError:
             raise ThermalClusterNotFound(path, cluster_id) from None
         else:
-            old_properties = create_thermal_properties(study_version, **values)
+            old_config = create_thermal_config(study_version, **values)
 
         # Use Python values to synchronize Config and Form values
         new_values = cluster_data.model_dump(mode="json", by_alias=False, exclude_none=True)
-        new_properties = old_properties.copy(exclude={"id"}, update=new_values)
+        new_config = old_config.copy(exclude={"id"}, update=new_values)
+        new_data = new_config.model_dump(mode="json", by_alias=True, exclude={"id"})
 
         # create the dict containing the new values using aliases
         data: t.Dict[str, t.Any] = {}
-        for updated_field, updated_value in new_values.items():
-            if updated_field in old_properties.model_fields:
-                field_info = old_properties.model_fields[updated_field]
-                field_name = field_info.alias if field_info.alias else updated_field
-                data[field_name] = updated_value
+        for field_name, field in new_config.model_fields.items():
+            if field_name in new_values:
+                name = field.alias if field.alias else field_name
+                data[name] = new_data[name]
 
         # create the update config commands with the modified data
         command_context = self.storage_service.variant_study_service.command_factory.command_context
@@ -381,7 +381,7 @@ def update_cluster(
         ]
         execute_or_add_commands(study, file_study, commands, self.storage_service)
 
-        values = {**new_properties.model_dump(mode="json", by_alias=False), "id": cluster_id}
+        values = {**new_config.model_dump(mode="json", by_alias=False), "id": cluster_id}
         return ThermalClusterOutput.model_validate(values)
 
     def delete_clusters(self, study: Study, area_id: str, cluster_ids: t.Sequence[str]) -> None:
@@ -431,8 +431,9 @@ def duplicate_cluster(
         Raises:
             ClusterAlreadyExists: If a cluster with the new name already exists in the area.
         """
-        new_id = transform_name_to_id(new_cluster_name)
-        if any(new_id == cluster.id for cluster in self.get_clusters(study, area_id)):
+        new_id = transform_name_to_id(new_cluster_name, lower=False)
+        lower_new_id = new_id.lower()
+        if any(lower_new_id == cluster.id.lower() for cluster in self.get_clusters(study, area_id)):
             raise DuplicateThermalCluster(area_id, new_id)
 
         # Cluster duplication
@@ -444,22 +445,23 @@ def duplicate_cluster(
         create_cluster_cmd = self._make_create_cluster_cmd(area_id, new_config, study_version)
 
         # Matrix edition
+        lower_source_id = source_id.lower()
         source_paths = [
-            f"input/thermal/series/{area_id}/{source_id}/series",
-            f"input/thermal/prepro/{area_id}/{source_id}/modulation",
-            f"input/thermal/prepro/{area_id}/{source_id}/data",
+            f"input/thermal/series/{area_id}/{lower_source_id}/series",
+            f"input/thermal/prepro/{area_id}/{lower_source_id}/modulation",
+            f"input/thermal/prepro/{area_id}/{lower_source_id}/data",
         ]
         new_paths = [
-            f"input/thermal/series/{area_id}/{new_id}/series",
-            f"input/thermal/prepro/{area_id}/{new_id}/modulation",
-            f"input/thermal/prepro/{area_id}/{new_id}/data",
+            f"input/thermal/series/{area_id}/{lower_new_id}/series",
+            f"input/thermal/prepro/{area_id}/{lower_new_id}/modulation",
+            f"input/thermal/prepro/{area_id}/{lower_new_id}/data",
         ]
         study_version = StudyVersion.parse(study.version)
         if study_version >= STUDY_VERSION_8_7:
-            source_paths.append(f"input/thermal/series/{area_id}/{source_id}/CO2Cost")
-            source_paths.append(f"input/thermal/series/{area_id}/{source_id}/fuelCost")
-            new_paths.append(f"input/thermal/series/{area_id}/{new_id}/CO2Cost")
-            new_paths.append(f"input/thermal/series/{area_id}/{new_id}/fuelCost")
+            source_paths.append(f"input/thermal/series/{area_id}/{lower_source_id}/CO2Cost")
+            source_paths.append(f"input/thermal/series/{area_id}/{lower_source_id}/fuelCost")
+            new_paths.append(f"input/thermal/series/{area_id}/{lower_new_id}/CO2Cost")
+            new_paths.append(f"input/thermal/series/{area_id}/{lower_new_id}/fuelCost")
 
         # Prepare and execute commands
         commands: t.List[t.Union[CreateCluster, ReplaceMatrix]] = [create_cluster_cmd]
@@ -477,7 +479,8 @@ def duplicate_cluster(
         return ThermalClusterOutput(**new_config.model_dump(mode="json", by_alias=False))
 
     def validate_series(self, study: Study, area_id: str, cluster_id: str) -> bool:
-        thermal_cluster_path = Path(f"input/thermal/series/{area_id}/{cluster_id.lower()}")
+        lower_cluster_id = cluster_id.lower()
+        thermal_cluster_path = Path(f"input/thermal/series/{area_id}/{lower_cluster_id}")
         series_path = [thermal_cluster_path / "series"]
         if StudyVersion.parse(study.version) >= STUDY_VERSION_8_7:
             series_path.append(thermal_cluster_path / "CO2Cost")

antarest/study/business/binding_constraint_management.py

@@ -30,7 +30,7 @@
     MatrixWidthMismatchError,
     WrongMatrixHeightError,
 )
-from antarest.core.model import JSON, LowerCaseStr
+from antarest.core.model import JSON
 from antarest.core.requests import CaseInsensitiveDict
 from antarest.core.serialization import AntaresBaseModel
 from antarest.core.utils.string import to_camel_case
@@ -44,7 +44,7 @@
     BindingConstraintFrequency,
     BindingConstraintOperator,
 )
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.business.matrix_constants.binding_constraint.series_after_v87 import (
@@ -340,7 +340,7 @@ class ConstraintOutput830(ConstraintOutputBase):
 
 
 class ConstraintOutput870(ConstraintOutput830):
-    group: LowerCaseStr = DEFAULT_GROUP
+    group: str = DEFAULT_GROUP
 
 
 # WARNING: Do not change the order of the following line, it is used to determine

antarest/study/business/district_manager.py

@@ -16,7 +16,7 @@
 from antarest.core.serialization import AntaresBaseModel
 from antarest.study.business.utils import execute_or_add_commands
 from antarest.study.model import Study
-from antarest.study.storage.rawstudy.model.filesystem.config.field_validators import transform_name_to_id
+from antarest.study.storage.rawstudy.model.filesystem.config.model import transform_name_to_id
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.variantstudy.model.command.create_district import CreateDistrict, DistrictBaseFilter
 from antarest.study.storage.variantstudy.model.command.remove_district import RemoveDistrict

antarest/study/business/model/link_model.py

@@ -148,6 +148,7 @@ class LinkBaseDTO(AntaresBaseModel):
     transmission_capacities: TransmissionCapacity = TransmissionCapacity.ENABLED
     asset_type: AssetType = AssetType.AC
     display_comments: bool = True
+    comments: str = ""
     colorr: int = Field(default=DEFAULT_COLOR, ge=0, le=255)
     colorb: int = Field(default=DEFAULT_COLOR, ge=0, le=255)
     colorg: int = Field(default=DEFAULT_COLOR, ge=0, le=255)
@@ -196,6 +197,7 @@ class LinkInternal(AntaresBaseModel):
     transmission_capacities: TransmissionCapacity = TransmissionCapacity.ENABLED
     asset_type: AssetType = AssetType.AC
     display_comments: bool = True
+    comments: str = ""
     colorr: int = Field(default=DEFAULT_COLOR, ge=0, le=255)
     colorb: int = Field(default=DEFAULT_COLOR, ge=0, le=255)
     colorg: int = Field(default=DEFAULT_COLOR, ge=0, le=255)

antarest/study/business/table_mode_management.py

@@ -219,9 +219,7 @@ def update_table_data(
             thermals_by_areas = collections.defaultdict(dict)
             for key, values in data.items():
                 area_id, cluster_id = key.split(" / ")
-                # Thermal clusters ids were not lowered at the time.
-                # So to ensure this endpoint still works with old scripts we have to lower the id at first.
-                thermals_by_areas[area_id][cluster_id.lower()] = ThermalClusterInput(**values)
+                thermals_by_areas[area_id][cluster_id] = ThermalClusterInput(**values)
             thermals_map = self._thermal_manager.update_thermals_props(study, thermals_by_areas)
             data = {
                 f"{area_id} / {cluster_id}": cluster.model_dump(by_alias=True, exclude={"id", "name"})
@@ -234,8 +232,7 @@ def update_table_data(
             renewables_by_areas = collections.defaultdict(dict)
             for key, values in data.items():
                 area_id, cluster_id = key.split(" / ")
-                # Same reason as for thermal clusters
-                renewables_by_areas[area_id][cluster_id.lower()] = RenewableClusterInput(**values)
+                renewables_by_areas[area_id][cluster_id] = RenewableClusterInput(**values)
             renewables_map = self._renewable_manager.update_renewables_props(study, renewables_by_areas)
             data = {
                 f"{area_id} / {cluster_id}": cluster.model_dump(by_alias=True, exclude={"id", "name"})

antarest/study/service.py

@@ -1600,6 +1600,7 @@ def _create_edit_study_command(
                         # Can happen with data with only one column. In this case, we don't care about the delimiter.
                         delimiter = "\t"
                     df = pd.read_csv(io.BytesIO(data), delimiter=delimiter, header=None).replace(",", ".", regex=True)
+                    df = df.dropna(axis=1, how="all")  # We want to remove columns full of NaN at the import
                     matrix = df.to_numpy(dtype=np.float64)
                 matrix = matrix.reshape((1, 0)) if matrix.size == 0 else matrix
                 return ReplaceMatrix(