From 9b8bc59f06b3d4dc9998c92496eca763f36a75d5 Mon Sep 17 00:00:00 2001
From: signedav <david@opengis.ch>
Date: Wed, 27 Apr 2022 09:10:35 +0200
Subject: [PATCH 1/2] parse itf files for models to be listen in the model
 import dialog. escape parsing on xtf when it goes over 100 lines without any
 results

---
 QgisModelBaker/utils/gui_utils.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/QgisModelBaker/utils/gui_utils.py b/QgisModelBaker/utils/gui_utils.py
index d3c72a2a6..f5c6b099a 100644
--- a/QgisModelBaker/utils/gui_utils.py
+++ b/QgisModelBaker/utils/gui_utils.py
@@ -1,3 +1,4 @@
+import mmap
 import os
 import pathlib
 import re
@@ -490,9 +491,10 @@ def refresh_model(self, source_model, db_connector=None, silent=False):
 
     def _transfer_file_models(self, xtf_file_path):
         """
+        Get model names from an ITF file does a regex parse with mmap (to avoid long parsing time).
         Get model names from an XTF file. Since XTF can be very large, we follow this strategy:
         1. Parse line by line.
-            1.a. Compare parsed line with the regular expression to get the Header Section.
+            1.a. Compare parsed line with the regular expression to get the Header Section. (escape after 100 lines)
             1.b. If found, stop parsing the XTF file and go to 2. If not found, append the new line to parsed lines and go
                 to next line.
         2. Give the Header Section to an XML parser and extract models. Note that we don't give the full XTF file to the XML
@@ -501,18 +503,33 @@ def _transfer_file_models(self, xtf_file_path):
         :return: List of model names from the XTF
         """
         models = []
+
+        # parse models from ITF
+        with open(xtf_file_path) as f:
+            s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+            matches = re.findall(rb"MODL (.*)", s)
+            if matches:
+                for match in matches:
+                    model = {}
+                    model["name"] = match.rstrip().decode()
+                    models.append(model)
+                return models
+
+        # parse models from XTF
         start_string = "<HEADERSECTION"
         end_string = "</HEADERSECTION>"
         text_found = ""
         with open(xtf_file_path, "r") as f:
             lines = ""
-            for line in f:
+            for line_number, line in enumerate(f):
                 lines += line
                 start_pos = lines.find(start_string)
                 end_pos = lines.find(end_string)
                 if end_pos > start_pos:
                     text_found = lines[start_pos : end_pos + len(end_string)]
                     break
+                if line_number > 100:
+                    break
 
         if text_found:
             try:

From e2e665408e2fac6fff44e60ff57fe62383809d6f Mon Sep 17 00:00:00 2001
From: signedav <david@opengis.ch>
Date: Thu, 28 Apr 2022 08:15:47 +0200
Subject: [PATCH 2/2] naming of transferfile is datafile instead of xtf file
 since it could be itf as well

---
 QgisModelBaker/utils/gui_utils.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/QgisModelBaker/utils/gui_utils.py b/QgisModelBaker/utils/gui_utils.py
index f5c6b099a..1ef68867b 100644
--- a/QgisModelBaker/utils/gui_utils.py
+++ b/QgisModelBaker/utils/gui_utils.py
@@ -449,10 +449,10 @@ def refresh_model(self, source_model, db_connector=None, silent=False):
             filtered_source_model_index = filtered_source_model.index(
                 r, SourceModel.Columns.SOURCE
             )
-            xtf_file_path = filtered_source_model_index.data(
+            data_file_path = filtered_source_model_index.data(
                 int(SourceModel.Roles.PATH)
             )
-            models = self._transfer_file_models(xtf_file_path)
+            models = self._transfer_file_models(data_file_path)
             for model in models:
                 if model["name"]:
                     enabled = model["name"] not in db_modelnames
@@ -483,13 +483,13 @@ def refresh_model(self, source_model, db_connector=None, silent=False):
                                 " (inactive because it already exists in the database)"
                                 if not enabled
                                 else "",
-                                xtf_file_path,
+                                data_file_path,
                             )
                         )
 
         return self.rowCount()
 
-    def _transfer_file_models(self, xtf_file_path):
+    def _transfer_file_models(self, data_file_path):
         """
         Get model names from an ITF file does a regex parse with mmap (to avoid long parsing time).
         Get model names from an XTF file. Since XTF can be very large, we follow this strategy:
@@ -500,12 +500,12 @@ def _transfer_file_models(self, xtf_file_path):
         2. Give the Header Section to an XML parser and extract models. Note that we don't give the full XTF file to the XML
         parser because it will read it completely, which may be not optimal.
         :param xtf_path: Path to an XTF file
-        :return: List of model names from the XTF
+        :return: List of model names from the datafile
         """
         models = []
 
         # parse models from ITF
-        with open(xtf_file_path) as f:
+        with open(data_file_path) as f:
             s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
             matches = re.findall(rb"MODL (.*)", s)
             if matches:
@@ -519,7 +519,7 @@ def _transfer_file_models(self, xtf_file_path):
         start_string = "<HEADERSECTION"
         end_string = "</HEADERSECTION>"
         text_found = ""
-        with open(xtf_file_path, "r") as f:
+        with open(data_file_path, "r") as f:
             lines = ""
             for line_number, line in enumerate(f):
                 lines += line
@@ -549,7 +549,7 @@ def _transfer_file_models(self, xtf_file_path):
                 self.print_info.emit(
                     self.tr(
                         "Could not parse transferfile file `{file}` ({exception})".format(
-                            file=xtf_file_path, exception=str(e)
+                            file=data_file_path, exception=str(e)
                         )
                     )
                 )