From 3688d915f1c6f2d514dd05165859ece11c3b2b84 Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Wed, 11 Sep 2024 16:50:25 +0200 Subject: [PATCH 1/8] Test interlis update --- .../interlis/gui/editors/damage_channel.py | 17 -- ...nimal-dataset-SIA405-ABWASSER-modified.xtf | 149 ++++++++++++++++++ .../teksi_wastewater/tests/test_interlis.py | 23 +++ 3 files changed, 172 insertions(+), 17 deletions(-) delete mode 100644 plugin/teksi_wastewater/interlis/gui/editors/damage_channel.py create mode 100644 plugin/teksi_wastewater/tests/data/minimal-dataset-SIA405-ABWASSER-modified.xtf diff --git a/plugin/teksi_wastewater/interlis/gui/editors/damage_channel.py b/plugin/teksi_wastewater/interlis/gui/editors/damage_channel.py deleted file mode 100644 index 0bdcf0e98..000000000 --- a/plugin/teksi_wastewater/interlis/gui/editors/damage_channel.py +++ /dev/null @@ -1,17 +0,0 @@ -from .base import Editor - - -class DamageChannelEditor(Editor): - class_name = "damage_channel" - - # DISABLED FOR NOW AS PER https://github.com/TWW/tww2ili/issues/8 - # this would allow to deselect BCD inspections (start of inspection) by default, - # as they don't provide valuable information. Once consensus is reached, we - # can remove this Editor altogether. - # def initially_checked(self): - # """ - # Determines if the item must be initially checked. To be overriden by subclasses. - # """ - # if self.obj.channel_damage_code__REL.value_en == "BCD": - # return False - # return True diff --git a/plugin/teksi_wastewater/tests/data/minimal-dataset-SIA405-ABWASSER-modified.xtf b/plugin/teksi_wastewater/tests/data/minimal-dataset-SIA405-ABWASSER-modified.xtf new file mode 100644 index 000000000..b0a8c5dea --- /dev/null +++ b/plugin/teksi_wastewater/tests/data/minimal-dataset-SIA405-ABWASSER-modified.xtf @@ -0,0 +1,149 @@ + + + + + + + + + + + + dataset generated by xtfout version 1.5, (c) infoGrips GmbH 2005-2022 + + + + + + 20240104 + + + 2024 + ch000000RE000001 + in_Betrieb + + + PAA.Sammelkanal + Freispiegelleitung + + + 20240104 + + + 2024 + Test1 + in_Betrieb + + + 600 + 800 + Kontroll_Einsteigschacht + + + 20240104 + + + 2024 + Test2 + in_Betrieb + + + Kontroll_Einsteigschacht + + + 20240104 + + + Kreisprofil + 1.00 + Kreisprofil + + + 20240104 + + + Test1-Test2F + 448.100 + + 2745874.4451267494.562 + + + + + 20240104 + + + Test1-Test2T + 447.510 + + 2745935.9321267519.317 + + + + + 20240104 + + + Test1 + + + 2745874.4451267494.562 + + 448.000 + + + 20240104 + + + Test2 + + + 2745935.9321267519.317 + + 447.510 + + + 20240911 + + + rp_from_level modified + Test1-Test2 + + 66.29 + 300 + Kunststoff_Polyvinilchlorid + + + 2745874.4451267494.562 + 2745935.9321267519.317 + + + + + + + + 20240104 + + + Test1 + + 451.000 + + 2745874.4451267494.562 + + + + 20240104 + + + Test2 + + 450.500 + + 2745935.9321267519.317 + + + + + diff --git a/plugin/teksi_wastewater/tests/test_interlis.py b/plugin/teksi_wastewater/tests/test_interlis.py index 64a367228..a75facbc2 100644 --- a/plugin/teksi_wastewater/tests/test_interlis.py +++ b/plugin/teksi_wastewater/tests/test_interlis.py @@ -27,6 +27,7 @@ MINIMAL_DATASET_DSS = "minimal-dataset-DSS.xtf" MINIMAL_DATASET_ORGANISATION_ARBON_ONLY = "minimal-dataset-organisation-arbon-only.xtf" MINIMAL_DATASET_SIA405_ABWASSER = "minimal-dataset-SIA405-ABWASSER.xtf" +MINIMAL_DATASET_SIA405_ABWASSER_MODIFIED = "minimal-dataset-SIA405-ABWASSER-modified.xtf" MINIMAL_DATASET_KEK_MANHOLE_DAMAGE = "minimal-dataset-VSA-KEK-manhole-damage.xtf" TEST_DATASET_DSS = "test-dataset-DSS.xtf" TEST_DATASET_ORGANISATIONS = "test-dataset-organisations.xtf" @@ -101,6 +102,12 @@ def test_minimal_import_export(self): ) self.assertIsNotNone(result) + result = DatabaseUtils.fetchone( + "SELECT remark FROM tww_od.wastewater_networkelement WHERE obj_id='ch000000RE000001';" + ) + self.assertIsNotNone(result) + self.assertEqual(result[0], "rp_from_level added") + # Import minimal dss xtf_file_input = self._get_data_filename(MINIMAL_DATASET_DSS) interlisImporterExporter = InterlisImporterExporter() @@ -173,6 +180,22 @@ def test_minimal_import_export(self): ) self.assertIsNotNone(interlis_object) + # Import modified minimal sia405 (test update) + xtf_file_input = self._get_data_filename(MINIMAL_DATASET_SIA405_ABWASSER_MODIFIED) + interlisImporterExporter = InterlisImporterExporter() + interlisImporterExporter.interlis_import(xtf_file_input=xtf_file_input) + + result = DatabaseUtils.fetchone( + "SELECT obj_id FROM tww_od.reach WHERE obj_id='ch000000RE000001';" + ) + self.assertIsNotNone(result) + + result = DatabaseUtils.fetchone( + "SELECT remark FROM tww_od.wastewater_networkelement WHERE obj_id='ch000000RE000001';" + ) + self.assertIsNotNone(result) + self.assertEqual(result[0], "rp_from_level modified") + def test_dss_dataset_import_export(self): # Import organisation xtf_file_input = self._get_data_filename(TEST_DATASET_ORGANISATIONS) From 5fe0d287bb1848e66dbf7972467667836071e820 Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Thu, 12 Sep 2024 00:16:54 +0200 Subject: [PATCH 2/8] Update working but not for geometries --- .../interlis/gui/editors/base.py | 38 +++++++++++-------- .../gui/interlis_import_selection_dialog.py | 10 ++--- ...nterlis_importer_to_intermediate_schema.py | 36 +++++++++++++++--- 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/plugin/teksi_wastewater/interlis/gui/editors/base.py b/plugin/teksi_wastewater/interlis/gui/editors/base.py index bed275a44..f84ceb3e5 100644 --- a/plugin/teksi_wastewater/interlis/gui/editors/base.py +++ b/plugin/teksi_wastewater/interlis/gui/editors/base.py @@ -55,33 +55,38 @@ def __init__(self, main_dialog, session, obj): self.session = session self.obj = obj + self._tree_widget_item = None + self.preprocess() self.update_state() @property - def listitem(self): + def tree_widget_item(self): """ - The editor's listitem (created on the fly if needed) + The editor's QTreeWidgetItem (created on the fly if needed) """ - if not hasattr(self, "_listitem"): - self._listitem = QTreeWidgetItem() - self._listitem.setCheckState( + if self._tree_widget_item is None: + self.update_tree_widget_item() + + return self._tree_widget_item + + def update_tree_widget_item(self): + if self._tree_widget_item is None: + self._tree_widget_item = QTreeWidgetItem() + self._tree_widget_item.setCheckState( 0, Qt.Checked if self.initially_checked() else Qt.Unchecked ) - self.update_listitem() - return self._listitem - def update_listitem(self): disp_id = str( getattr(self.obj, "obj_id", getattr(self.obj, "value_en", "?")) ) # some elements may not have obj_id, such as value_lists - self.listitem.setText(0, getattr(self.obj, "identifier", disp_id)) - self.listitem.setToolTip(0, disp_id) + self._tree_widget_item.setText(0, getattr(self.obj, "identifier", disp_id)) + self._tree_widget_item.setToolTip(0, disp_id) - self.listitem.setText(1, self.status) + self._tree_widget_item.setText(1, self.status) - self.listitem.setText(2, self.validity) + self._tree_widget_item.setText(2, self.validity) if self.status == Editor.EXISTING: color = "lightgray" elif self.validity == Editor.INVALID: @@ -92,7 +97,7 @@ def update_listitem(self): color = "lightgreen" else: color = "lightgray" - self.listitem.setBackground(2, QBrush(QColor(color))) + self._tree_widget_item.setBackground(2, QBrush(QColor(color))) @property def widget(self): @@ -133,12 +138,15 @@ def update_state(self): self.status = Editor.NEW elif obj_inspect.deleted: self.status = Editor.DELETED - elif obj_inspect.modified: - self.status = Editor.MODIFIED elif obj_inspect.persistent: self.status = Editor.EXISTING else: self.status = Editor.UNKNOWN + + # For modified use the session is_modified method (slower but more correct) + if self.session.is_modified(self.obj): + self.status = Editor.MODIFIED + self.validate() def validate(self): diff --git a/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py b/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py index 10950936e..220bbadd2 100644 --- a/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py +++ b/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py @@ -73,8 +73,8 @@ def update_tree(self): ) self.treeWidget.addTopLevelItem(self.category_items[cls]) - editor.update_listitem() - self.category_items[cls].addChild(editor.listitem) + editor.update_tree_widget_item() + self.category_items[cls].addChild(editor.tree_widget_item) if editor.validity != Editor.VALID: self.treeWidget.expandItem(self.category_items[cls]) @@ -136,7 +136,7 @@ def current_item_changed(self, current_item, previous_item): Calls refresh_widget_for_obj for the currently selected object """ for editor in self.editors.values(): - if editor.listitem == current_item: + if editor.tree_widget_item == current_item: self.refresh_editor(editor) break else: @@ -154,7 +154,7 @@ def refresh_editor(self, editor): editor.update_state() # Update the list item - editor.update_listitem() + editor.update_tree_widget_item() # Update generic widget contents self.debugTextEdit.clear() @@ -226,6 +226,6 @@ def rollback_session(self): def get_obj_from_listitem(self, listitem): for obj, editor in self.editors.items(): - if editor.listitem == listitem: + if editor.tree_widget_item == listitem: return obj return None diff --git a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py index 273ad9494..a235973ff 100644 --- a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py +++ b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py @@ -1,3 +1,5 @@ +from datetime import datetime + from geoalchemy2.functions import ST_Force3D from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_dirty @@ -430,12 +432,32 @@ def create_or_update(self, cls, **kwargs): # We try to get the instance from the session/database obj_id = kwargs.get("obj_id", None) if obj_id: - instance = self.session_tww.query(cls).get(kwargs.get("obj_id", None)) + instance = self.session_tww.get(cls, obj_id) if instance: - # We found it -> update - instance.__dict__.update(kwargs) - flag_dirty(instance) # we flag it as dirty so it stays in the session + flag_dirty( + instance + ) # we flag it as dirty so it stays in the session. This is a workaround trick + # needed bcause the session is not meant to be used as a cache: https://docs.sqlalchemy.org/en/20/orm/session_basics.html#is-the-session-a-cache + + # Update modified values + for key, value in kwargs.items(): + if key == "last_modification": + value = datetime.combine(value, datetime.min.time()) + + instanceAttribute = getattr(instance, key, None) + if "geometry" in key: + print(f"new: {value}") + print(f"old: {instanceAttribute}") + continue + + if instanceAttribute != value: + print( + f"cls {cls} - key {key} - value {value} - getattr {getattr(instance, key, None)}" + ) + + # Setattr in the background updates the session state and make it possible to use "is_modified" afterwards + setattr(instance, key, value) else: # We didn't find it -> create instance = cls(**kwargs) @@ -462,7 +484,11 @@ def wastewater_structure_common(self, row): self.model_classes_tww_od.wastewater_structure_accessibility, row.zugaenglichkeit ), "contract_section": row.baulos, - "detail_geometry3d_geometry": ST_Force3D(row.detailgeometrie), + "detail_geometry3d_geometry": ( + row.detailgeometrie + if row.detailgeometrie is None + else ST_Force3D(row.detailgeometrie) + ), # TODO : NOT MAPPED VSA-DSS 3D # "elevation_determination": self.get_vl_code( # self.model_classes_tww_od.wastewater_structure_elevation_determination, row.hoehenbestimmung From 9d30c411f16bad3d94612a5e960547c60bb195e0 Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Tue, 17 Sep 2024 20:43:38 +0200 Subject: [PATCH 3/8] For geometries as welll --- .../interlis_importer_to_intermediate_schema.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py index a235973ff..ec93ef774 100644 --- a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py +++ b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py @@ -446,16 +446,10 @@ def create_or_update(self, cls, **kwargs): value = datetime.combine(value, datetime.min.time()) instanceAttribute = getattr(instance, key, None) - if "geometry" in key: - print(f"new: {value}") - print(f"old: {instanceAttribute}") - continue + if value is not None and "_geometry" in key: + value = self.session_tww.scalar(value) if instanceAttribute != value: - print( - f"cls {cls} - key {key} - value {value} - getattr {getattr(instance, key, None)}" - ) - # Setattr in the background updates the session state and make it possible to use "is_modified" afterwards setattr(instance, key, value) else: From 541a34be3614061508d69f38d7f6c75dbb58e641 Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Tue, 17 Sep 2024 23:29:40 +0200 Subject: [PATCH 4/8] Pimp up import dialog --- .../interlis/gui/editors/base.py | 17 ++- .../gui/interlis_import_selection_dialog.py | 132 ++++++++++++++---- .../gui/interlis_import_selection_dialog.ui | 74 ++++++---- 3 files changed, 163 insertions(+), 60 deletions(-) diff --git a/plugin/teksi_wastewater/interlis/gui/editors/base.py b/plugin/teksi_wastewater/interlis/gui/editors/base.py index f84ceb3e5..c3cc0432f 100644 --- a/plugin/teksi_wastewater/interlis/gui/editors/base.py +++ b/plugin/teksi_wastewater/interlis/gui/editors/base.py @@ -75,18 +75,21 @@ def update_tree_widget_item(self): if self._tree_widget_item is None: self._tree_widget_item = QTreeWidgetItem() self._tree_widget_item.setCheckState( - 0, Qt.Checked if self.initially_checked() else Qt.Unchecked + self.main_dialog.Columns.NAME, + Qt.Checked if self.initially_checked() else Qt.Unchecked, ) disp_id = str( getattr(self.obj, "obj_id", getattr(self.obj, "value_en", "?")) ) # some elements may not have obj_id, such as value_lists - self._tree_widget_item.setText(0, getattr(self.obj, "identifier", disp_id)) - self._tree_widget_item.setToolTip(0, disp_id) + self._tree_widget_item.setText( + self.main_dialog.Columns.NAME, getattr(self.obj, "identifier", disp_id) + ) + self._tree_widget_item.setToolTip(self.main_dialog.Columns.NAME, disp_id) - self._tree_widget_item.setText(1, self.status) + self._tree_widget_item.setText(self.main_dialog.Columns.STATE, self.status) - self._tree_widget_item.setText(2, self.validity) + self._tree_widget_item.setText(self.main_dialog.Columns.VALIDITY, self.validity) if self.status == Editor.EXISTING: color = "lightgray" elif self.validity == Editor.INVALID: @@ -97,7 +100,9 @@ def update_tree_widget_item(self): color = "lightgreen" else: color = "lightgray" - self._tree_widget_item.setBackground(2, QBrush(QColor(color))) + self._tree_widget_item.setBackground( + self.main_dialog.Columns.VALIDITY, QBrush(QColor(color)) + ) @property def widget(self): diff --git a/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py b/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py index 220bbadd2..a42686ab4 100644 --- a/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py +++ b/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.py @@ -1,15 +1,17 @@ import os import sys from collections import defaultdict +from enum import IntEnum from qgis.core import Qgis from qgis.PyQt.QtCore import Qt -from qgis.PyQt.QtGui import QFont +from qgis.PyQt.QtGui import QBrush, QColor, QFont from qgis.PyQt.QtWidgets import QDialog, QHeaderView, QMessageBox, QTreeWidgetItem from qgis.PyQt.uic import loadUi from qgis.utils import iface from sqlalchemy import inspect from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import get_history from ...utils.qt_utils import OverrideCursor from .editors.base import Editor @@ -19,6 +21,17 @@ class InterlisImportSelectionDialog(QDialog): + + class Columns(IntEnum): + NAME = 0 + STATE = 1 + VALIDITY = 2 + + class ColumnsDebug(IntEnum): + KEY = 0 + VALUE = 1 + VALUE_OLD = 2 + def __init__(self, parent=None): super().__init__(parent) loadUi( @@ -28,9 +41,17 @@ def __init__(self, parent=None): self.accepted.connect(self.commit_session) self.rejected.connect(self.rollback_session) - header = self.treeWidget.header() - header.setSectionResizeMode(QHeaderView.ResizeToContents) - header.setSectionResizeMode(0, QHeaderView.Stretch) + self.treeWidget.header().setSectionResizeMode(QHeaderView.ResizeToContents) + self.treeWidget.header().setSectionResizeMode(self.Columns.NAME, QHeaderView.Stretch) + + self.debugTreeWidget.header().setSectionResizeMode(QHeaderView.ResizeToContents) + self.debugTreeWidget.header().setSectionResizeMode( + self.ColumnsDebug.KEY, QHeaderView.Interactive + ) + self.debugTreeWidget.header().setSectionResizeMode( + self.ColumnsDebug.VALUE, QHeaderView.Interactive + ) + self.debugTreeWidget.hideColumn(self.ColumnsDebug.VALUE_OLD) def init_with_session(self, session: Session): """ @@ -66,22 +87,27 @@ def update_tree(self): continue if cls not in self.category_items: - self.category_items[cls].setText(0, cls.__name__) - self.category_items[cls].setCheckState(0, Qt.Checked) + self.category_items[cls].setText(self.Columns.NAME, cls.__name__) + self.category_items[cls].setCheckState(self.Columns.NAME, Qt.Checked) self.category_items[cls].setFont( - 0, QFont(QFont().defaultFamily(), weight=QFont.Weight.Bold) + self.Columns.NAME, QFont(QFont().defaultFamily(), weight=QFont.Weight.Bold) ) self.treeWidget.addTopLevelItem(self.category_items[cls]) editor.update_tree_widget_item() self.category_items[cls].addChild(editor.tree_widget_item) + if editor.status != Editor.EXISTING: + self.category_items[cls].setText(self.Columns.STATE, "*") + if editor.validity != Editor.VALID: self.treeWidget.expandItem(self.category_items[cls]) # Show counts for cls, category_item in self.category_items.items(): - category_item.setText(0, f"{cls.__name__} ({category_item.childCount()})") + category_item.setText( + self.Columns.NAME, f"{cls.__name__} ({category_item.childCount()})" + ) def item_changed(self, item, column): """ @@ -90,7 +116,7 @@ def item_changed(self, item, column): (propagation to parent/children is disabled for now) """ - checked = item.checkState(0) == Qt.Checked + checked = item.checkState(self.Columns.NAME) == Qt.Checked # add or remove object from session obj = self.get_obj_from_listitem(item) @@ -100,13 +126,13 @@ def item_changed(self, item, column): else: self.session.expunge(obj) - checked_state = item.checkState(0) + checked_state = item.checkState(self.Columns.NAME) if checked_state == Qt.PartiallyChecked: return # propagate to children for child in [item.child(i) for i in range(item.childCount())]: - child.setCheckState(0, checked_state) + child.setCheckState(self.Columns.NAME, checked_state) # propagate to parent parent = item.parent() @@ -114,22 +140,22 @@ def item_changed(self, item, column): has_checked = False has_unchecked = False for sibling in [parent.child(i) for i in range(parent.childCount())]: - if sibling.checkState(0) == Qt.Checked: + if sibling.checkState(self.Columns.NAME) == Qt.Checked: has_checked = True - if sibling.checkState(0) == Qt.Unchecked: + if sibling.checkState(self.Columns.NAME) == Qt.Unchecked: has_unchecked = True if has_checked and has_unchecked: break if has_checked and has_unchecked: - parent.setCheckState(0, Qt.PartiallyChecked) + parent.setCheckState(self.Columns.NAME, Qt.PartiallyChecked) elif has_checked: - parent.setCheckState(0, Qt.Checked) + parent.setCheckState(self.Columns.NAME, Qt.Checked) elif has_unchecked: - parent.setCheckState(0, Qt.Unchecked) + parent.setCheckState(self.Columns.NAME, Qt.Unchecked) else: # no children at all !! - parent.setCheckState(0, Qt.PartiallyChecked) + parent.setCheckState(self.Columns.NAME, Qt.PartiallyChecked) def current_item_changed(self, current_item, previous_item): """ @@ -140,7 +166,7 @@ def current_item_changed(self, current_item, previous_item): self.refresh_editor(editor) break else: - self.debugTextEdit.clear() + self.debugTreeWidget.clear() self.validityLabel.clear() current_widget = self.stackedWidget.currentWidget() if current_widget: @@ -157,16 +183,55 @@ def refresh_editor(self, editor): editor.update_tree_widget_item() # Update generic widget contents - self.debugTextEdit.clear() self.validityLabel.clear() + self.debugTreeWidget.clear() + + if editor.status == Editor.MODIFIED: + self.debugTreeWidget.showColumn(self.ColumnsDebug.VALUE_OLD) + else: + self.debugTreeWidget.hideColumn(self.ColumnsDebug.VALUE_OLD) + + # Show all attributes in the debug text edit + treeWidgetItemAttributes = QTreeWidgetItem() + treeWidgetItemAttributes.setText(self.ColumnsDebug.KEY, "Attributes") + for attribute in inspect(editor.obj).mapper.column_attrs: + treeWidgetItemAttribute = QTreeWidgetItem() + treeWidgetItemAttribute.setText(self.ColumnsDebug.KEY, attribute.key) + val = getattr(editor.obj, attribute.key) + if val is not None: + treeWidgetItemAttribute.setText(self.ColumnsDebug.VALUE, f"{val}") + + if editor.status == Editor.MODIFIED: + + history = get_history(editor.obj, attribute.key) + + value_old = None + if history.unchanged != (): + value_old = history.unchanged[0] + + if history.deleted != (): + value_old = history.deleted[0] + + if value_old is not None: + treeWidgetItemAttribute.setText(self.ColumnsDebug.VALUE_OLD, f"{value_old}") + + if val != value_old: + brush = QBrush(QColor("orange")) + treeWidgetItemAttribute.setBackground(self.ColumnsDebug.KEY, brush) + treeWidgetItemAttribute.setBackground(self.ColumnsDebug.VALUE, brush) + treeWidgetItemAttribute.setBackground(self.ColumnsDebug.VALUE_OLD, brush) + + treeWidgetItemAttributes.addChild(treeWidgetItemAttribute) + + self.debugTreeWidget.addTopLevelItem(treeWidgetItemAttributes) + self.debugTreeWidget.expandItem(treeWidgetItemAttributes) + + # Debug + treeWidgetItemDebug = QTreeWidgetItem() + treeWidgetItemDebug.setText(self.ColumnsDebug.KEY, "Debug") - # Show all attributes in the debug text edit - self.debugTextEdit.append("-- ATTRIBUTES --") - for c in inspect(editor.obj).mapper.column_attrs: - val = getattr(editor.obj, c.key) - self.debugTextEdit.append(f"{c.key}: {val}") # Show sqlalchemy state in the debug text edit - self.debugTextEdit.append("-- SQLALCHEMY STATUS --") + sqlAlchemyStates = [] for status_name in [ "transient", "pending", @@ -177,9 +242,20 @@ def refresh_editor(self, editor): "expired", ]: if getattr(inspect(editor.obj), status_name): - self.debugTextEdit.append(f"{status_name} ") - self.debugTextEdit.append("-- DEBUG --") - self.debugTextEdit.append(repr(editor.obj)) + sqlAlchemyStates.append(f"{status_name}") + + treeWidgetItemSqlAlchemyState = QTreeWidgetItem() + treeWidgetItemSqlAlchemyState.setText(self.ColumnsDebug.KEY, "Sql Alchemy status") + treeWidgetItemSqlAlchemyState.setText(self.ColumnsDebug.VALUE, ", ".join(sqlAlchemyStates)) + treeWidgetItemDebug.addChild(treeWidgetItemSqlAlchemyState) + + treeWidgetItemEditorObject = QTreeWidgetItem() + treeWidgetItemEditorObject.setText(self.ColumnsDebug.KEY, "Editor object") + treeWidgetItemEditorObject.setText(self.ColumnsDebug.VALUE, repr(editor.obj)) + treeWidgetItemDebug.addChild(treeWidgetItemEditorObject) + + self.debugTreeWidget.addTopLevelItem(treeWidgetItemDebug) + self.debugTreeWidget.resizeColumnToContents(self.ColumnsDebug.KEY) # Show the validity label self.validityLabel.setText(editor.message) diff --git a/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.ui b/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.ui index f3145e78b..6a543a2d8 100644 --- a/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.ui +++ b/plugin/teksi_wastewater/interlis/gui/interlis_import_selection_dialog.ui @@ -25,9 +25,9 @@ TWW interlis import - + - + Qt::Horizontal @@ -38,25 +38,22 @@ 0 - - false - false - name + Name - state + State - validity + Validity @@ -149,7 +146,7 @@ - + @@ -167,24 +164,49 @@ - - - - - - - - - Raw data (debug) - - - true + + + Qt::Vertical - - - - - + + + + + + + Raw data + + + true + + + + + + false + + + + Key + + + + + Value + + + + + Old value + + + <html><head/><body><p>The value that is currently written in the database. If accepted, it will be overwritten by the &quot;Value&quot; column.</p></body></html> + + + + + + From 67625cb4dc9301f810196f6e656d1dbd45e7caa3 Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Fri, 20 Sep 2024 09:56:21 +0200 Subject: [PATCH 5/8] Compute WKB earlier --- .../interlis_importer_to_intermediate_schema.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py index ec93ef774..f424b089a 100644 --- a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py +++ b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py @@ -446,9 +446,6 @@ def create_or_update(self, cls, **kwargs): value = datetime.combine(value, datetime.min.time()) instanceAttribute = getattr(instance, key, None) - if value is not None and "_geometry" in key: - value = self.session_tww.scalar(value) - if instanceAttribute != value: # Setattr in the background updates the session state and make it possible to use "is_modified" afterwards setattr(instance, key, value) @@ -481,7 +478,7 @@ def wastewater_structure_common(self, row): "detail_geometry3d_geometry": ( row.detailgeometrie if row.detailgeometrie is None - else ST_Force3D(row.detailgeometrie) + else self.session_tww.scalar(ST_Force3D(row.detailgeometrie)) ), # TODO : NOT MAPPED VSA-DSS 3D # "elevation_determination": self.get_vl_code( @@ -1917,7 +1914,7 @@ def _import_haltungspunkt(self): ), position_of_connection=row.lage_anschluss, remark=row.bemerkung, - situation3d_geometry=ST_Force3D(row.lage), + situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage)), ) self.session_tww.add(reach_point) print(".", end="") @@ -1949,7 +1946,7 @@ def _import_abwasserknoten(self): # fk_hydr_geometry=row.REPLACE_ME, # TODO : NOT MAPPED backflow_level_current=row.rueckstaukote_ist, bottom_level=row.sohlenkote, - situation3d_geometry=ST_Force3D(row.lage), + situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage)), ) self.session_tww.add(wastewater_node) print(".", end="") @@ -1979,7 +1976,7 @@ def _import_haltung(self): ), length_effective=row.laengeeffektiv, material=self.get_vl_code(self.model_classes_tww_vl.reach_material, row.material), - progression3d_geometry=ST_Force3D(row.verlauf), + progression3d_geometry=self.session_tww.scalar(ST_Force3D(row.verlauf)), reliner_material=self.get_vl_code( self.model_classes_tww_od.reach_reliner_material, row.reliner_material ), @@ -2059,7 +2056,7 @@ def _import_deckel(self): positional_accuracy=self.get_vl_code( self.model_classes_tww_od.cover_positional_accuracy, row.lagegenauigkeit ), - situation3d_geometry=ST_Force3D(row.lage), + situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage)), sludge_bucket=self.get_vl_code( self.model_classes_tww_od.cover_sludge_bucket, row.schlammeimer ), From ea279d04c0f08c475240d40d2df96a0b366e2b60 Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Thu, 26 Sep 2024 22:15:18 +0200 Subject: [PATCH 6/8] Special handling for more time and float columns Check for already existing re_... instances --- ...nterlis_importer_to_intermediate_schema.py | 104 ++++++++++++++---- 1 file changed, 82 insertions(+), 22 deletions(-) diff --git a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py index f424b089a..0751184bc 100644 --- a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py +++ b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import date, datetime from geoalchemy2.functions import ST_Force3D from sqlalchemy.orm import Session @@ -440,12 +440,35 @@ def create_or_update(self, cls, **kwargs): ) # we flag it as dirty so it stays in the session. This is a workaround trick # needed bcause the session is not meant to be used as a cache: https://docs.sqlalchemy.org/en/20/orm/session_basics.html#is-the-session-a-cache - # Update modified values + # Update dates times (different resolution Interlis / TWW) + date_time_keys = [ + "last_modification", + "time_point", + "date_last_examen", + "renovation_date", + "date_entry", + "time", + "date_mutation", + ] + + # Double fields that needs special comparison (imported as text from interlis) + double_value_keys = ["value", "x", "y"] + for key, value in kwargs.items(): - if key == "last_modification": + if key in date_time_keys and isinstance(value, date): value = datetime.combine(value, datetime.min.time()) instanceAttribute = getattr(instance, key, None) + + if key in double_value_keys: + try: + value = float(value) + instanceAttribute = float(instanceAttribute) + except Exception: + logger.warning( + f"Values of column '{key}' are not convertible to float: interlis='{value}', old='{instanceAttribute}'" + ) + if instanceAttribute != value: # Setattr in the background updates the session state and make it possible to use "is_modified" afterwards setattr(instance, key, value) @@ -2157,10 +2180,26 @@ def _import_untersuchung(self): # The day ili2pg works, we probably need to double-check whether the referenced wastewater structure exists prior # to creating this association. # Soft matching based on from/to_point_identifier will be done in the GUI data checking process. - exam_to_wastewater_structure = self.create_or_update( - self.model_classes_tww_od.re_maintenance_event_wastewater_structure, - fk_wastewater_structure=row.abwasserbauwerkref, - fk_maintenance_event=row.t_ili_tid, + + exam_to_wastewater_structure = ( + self.session_tww.query( + self.model_classes_tww_od.re_maintenance_event_wastewater_structure + ) + .filter_by( + fk_wastewater_structure=row.abwasserbauwerkref, + fk_maintenance_event=row.t_ili_tid, + ) + .first() + ) + if exam_to_wastewater_structure is not None: + # Already existing -> do nothing + continue + + exam_to_wastewater_structure = ( + self.model_classes_tww_od.re_maintenance_event_wastewater_structure( + fk_wastewater_structure=row.abwasserbauwerkref, + fk_maintenance_event=row.t_ili_tid, + ) ) self.session_tww.add(exam_to_wastewater_structure) @@ -2277,16 +2316,29 @@ def _import_erhaltungsereignis_abwasserbauwerkassoc(self): for row in self.session_interlis.query( self.model_classes_interlis.erhaltungsereignis_abwasserbauwerkassoc ): - - re_maintenance_event_wastewater_structure = self.create_or_update( - self.model_classes_tww_od.re_maintenance_event_wastewater_structure, - # this class does not inherit base_commmon - # **self.base_common(row), - # --- re_maintenance_event_wastewater_structure --- - fk_maintenance_event=self.get_pk( - row.erhaltungsereignis_abwasserbauwerkassocref__REL - ), - fk_wastewater_structure=self.get_pk(row.abwasserbauwerkref__REL), + re_maintenance_event_wastewater_structure = ( + self.session_tww.query( + self.model_classes_tww_od.re_maintenance_event_wastewater_structure + ) + .filter_by( + fk_wastewater_structure=self.get_pk(row.abwasserbauwerkref__REL), + fk_maintenance_event=self.get_pk( + row.erhaltungsereignis_abwasserbauwerkassocref__REL + ), + ) + .first() + ) + if re_maintenance_event_wastewater_structure is not None: + # Already existing -> do nothing + continue + + re_maintenance_event_wastewater_structure = ( + self.model_classes_tww_od.re_maintenance_event_wastewater_structure( + fk_maintenance_event=self.get_pk( + row.erhaltungsereignis_abwasserbauwerkassocref__REL + ), + fk_wastewater_structure=self.get_pk(row.abwasserbauwerkref__REL), + ) ) self.session_tww.add(re_maintenance_event_wastewater_structure) @@ -2296,11 +2348,19 @@ def _import_gebaeudegruppe_entsorgungassoc(self): for row in self.session_interlis.query( self.model_classes_interlis.gebaeudegruppe_entsorgungassoc ): - re_building_group_disposal = self.create_or_update( - self.model_classes_tww_od.re_building_group_disposal, - # this class does not inherit base_commmon - # **self.base_common(row), - # --- re_building_group_disposal --- + re_building_group_disposal = ( + self.session_tww.query(self.model_classes_tww_od.re_building_group_disposal) + .filter_by( + fk_building_group=self.get_pk(row.gebaeudegruppe_entsorgungassocref__REL), + fk_disposal=self.get_pk(row.entsorgungref__REL), + ) + .first() + ) + if re_building_group_disposal is not None: + # Already existing -> do nothing + continue + + re_building_group_disposal = self.model_classes_tww_od.re_building_group_disposal( fk_building_group=self.get_pk(row.gebaeudegruppe_entsorgungassocref__REL), fk_disposal=self.get_pk(row.entsorgungref__REL), ) From d3fda6474482f4542a3647d9c7da364aa95bf5de Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Sun, 29 Sep 2024 20:57:16 +0200 Subject: [PATCH 7/8] Use 2.5D quote for geometry on Interlis import --- ...nterlis_importer_to_intermediate_schema.py | 6 ++--- .../teksi_wastewater/tests/test_interlis.py | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py index 0751184bc..6be50c49f 100644 --- a/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py +++ b/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_importer_to_intermediate_schema.py @@ -1937,7 +1937,7 @@ def _import_haltungspunkt(self): ), position_of_connection=row.lage_anschluss, remark=row.bemerkung, - situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage)), + situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage, row.kote)), ) self.session_tww.add(reach_point) print(".", end="") @@ -1969,7 +1969,7 @@ def _import_abwasserknoten(self): # fk_hydr_geometry=row.REPLACE_ME, # TODO : NOT MAPPED backflow_level_current=row.rueckstaukote_ist, bottom_level=row.sohlenkote, - situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage)), + situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage, row.sohlenkote)), ) self.session_tww.add(wastewater_node) print(".", end="") @@ -2079,7 +2079,7 @@ def _import_deckel(self): positional_accuracy=self.get_vl_code( self.model_classes_tww_od.cover_positional_accuracy, row.lagegenauigkeit ), - situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage)), + situation3d_geometry=self.session_tww.scalar(ST_Force3D(row.lage, row.kote)), sludge_bucket=self.get_vl_code( self.model_classes_tww_od.cover_sludge_bucket, row.schlammeimer ), diff --git a/plugin/teksi_wastewater/tests/test_interlis.py b/plugin/teksi_wastewater/tests/test_interlis.py index a75facbc2..d03ea699b 100644 --- a/plugin/teksi_wastewater/tests/test_interlis.py +++ b/plugin/teksi_wastewater/tests/test_interlis.py @@ -108,6 +108,12 @@ def test_minimal_import_export(self): self.assertIsNotNone(result) self.assertEqual(result[0], "rp_from_level added") + result = DatabaseUtils.fetchone( + "SELECT bottom_level FROM tww_od.wastewater_node WHERE obj_id='ch000000WN000001';" + ) + self.assertIsNotNone(result) + self.assertEqual(result[0], 448.0) + # Import minimal dss xtf_file_input = self._get_data_filename(MINIMAL_DATASET_DSS) interlisImporterExporter = InterlisImporterExporter() @@ -118,6 +124,13 @@ def test_minimal_import_export(self): ) self.assertIsNotNone(result) + # Check that we don't loose Z information on second import + result = DatabaseUtils.fetchone( + "SELECT bottom_level FROM tww_od.wastewater_node WHERE obj_id='ch000000WN000001';" + ) + self.assertIsNotNone(result) + self.assertEqual(result[0], 448.0) + # Import minimal kek xtf_file_input = self._get_data_filename(MINIMAL_DATASET_KEK_MANHOLE_DAMAGE) interlisImporterExporter = InterlisImporterExporter() @@ -129,6 +142,13 @@ def test_minimal_import_export(self): self.assertIsNotNone(result) self.assertEqual(result[0], "fk11abk6EX000002") + # Check that we don't loose Z information on second import + result = DatabaseUtils.fetchone( + "SELECT bottom_level FROM tww_od.wastewater_node WHERE obj_id='ch000000WN000001';" + ) + self.assertIsNotNone(result) + self.assertEqual(result[0], 448.0) + # Export minimal sia405 export_xtf_file = self._get_output_filename("export_minimal_dataset_sia405") interlisImporterExporter.interlis_export( @@ -196,6 +216,13 @@ def test_minimal_import_export(self): self.assertIsNotNone(result) self.assertEqual(result[0], "rp_from_level modified") + # Check that we don't loose Z information on second import + result = DatabaseUtils.fetchone( + "SELECT bottom_level FROM tww_od.wastewater_node WHERE obj_id='ch000000WN000001';" + ) + self.assertIsNotNone(result) + self.assertEqual(result[0], 448.0) + def test_dss_dataset_import_export(self): # Import organisation xtf_file_input = self._get_data_filename(TEST_DATASET_ORGANISATIONS) From 6c468082a37a7bfad9489b2890b8cb4425443dbc Mon Sep 17 00:00:00 2001 From: Damiano Lombardi Date: Sun, 29 Sep 2024 21:13:28 +0200 Subject: [PATCH 8/8] Repristinate correct showing of status NEW --- plugin/teksi_wastewater/interlis/gui/editors/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugin/teksi_wastewater/interlis/gui/editors/base.py b/plugin/teksi_wastewater/interlis/gui/editors/base.py index c3cc0432f..441ecaea8 100644 --- a/plugin/teksi_wastewater/interlis/gui/editors/base.py +++ b/plugin/teksi_wastewater/interlis/gui/editors/base.py @@ -149,7 +149,11 @@ def update_state(self): self.status = Editor.UNKNOWN # For modified use the session is_modified method (slower but more correct) - if self.session.is_modified(self.obj): + if ( + self.status != Editor.NEW + and self.status != Editor.DELETED + and self.session.is_modified(self.obj) + ): self.status = Editor.MODIFIED self.validate()