Merge pull request #687 from opengisch/itfmodelscan

Parse ITF files for models to be listen in the model import dialog

QgisModelBaker/utils/gui_utils.py

@@ -1,3 +1,4 @@
+import mmap
 import os
 import pathlib
 import re
@@ -448,10 +449,10 @@ def refresh_model(self, source_model, db_connector=None, silent=False):
             filtered_source_model_index = filtered_source_model.index(
                 r, SourceModel.Columns.SOURCE
             )
-            xtf_file_path = filtered_source_model_index.data(
+            data_file_path = filtered_source_model_index.data(
                 int(SourceModel.Roles.PATH)
             )
-            models = self._transfer_file_models(xtf_file_path)
+            models = self._transfer_file_models(data_file_path)
             for model in models:
                 if model["name"]:
                     enabled = model["name"] not in db_modelnames
@@ -482,37 +483,53 @@ def refresh_model(self, source_model, db_connector=None, silent=False):
                                 " (inactive because it already exists in the database)"
                                 if not enabled
                                 else "",
-                                xtf_file_path,
+                                data_file_path,
                             )
                         )
 
         return self.rowCount()
 
-    def _transfer_file_models(self, xtf_file_path):
+    def _transfer_file_models(self, data_file_path):
         """
+        Get model names from an ITF file does a regex parse with mmap (to avoid long parsing time).
         Get model names from an XTF file. Since XTF can be very large, we follow this strategy:
         1. Parse line by line.
-            1.a. Compare parsed line with the regular expression to get the Header Section.
+            1.a. Compare parsed line with the regular expression to get the Header Section. (escape after 100 lines)
             1.b. If found, stop parsing the XTF file and go to 2. If not found, append the new line to parsed lines and go
                 to next line.
         2. Give the Header Section to an XML parser and extract models. Note that we don't give the full XTF file to the XML
         parser because it will read it completely, which may be not optimal.
         :param xtf_path: Path to an XTF file
-        :return: List of model names from the XTF
+        :return: List of model names from the datafile
         """
         models = []
+
+        # parse models from ITF
+        with open(data_file_path) as f:
+            s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+            matches = re.findall(rb"MODL (.*)", s)
+            if matches:
+                for match in matches:
+                    model = {}
+                    model["name"] = match.rstrip().decode()
+                    models.append(model)
+                return models
+
+        # parse models from XTF
         start_string = "<HEADERSECTION"
         end_string = "</HEADERSECTION>"
         text_found = ""
-        with open(xtf_file_path, "r") as f:
+        with open(data_file_path, "r") as f:
             lines = ""
-            for line in f:
+            for line_number, line in enumerate(f):
                 lines += line
                 start_pos = lines.find(start_string)
                 end_pos = lines.find(end_string)
                 if end_pos > start_pos:
                     text_found = lines[start_pos : end_pos + len(end_string)]
                     break
+                if line_number > 100:
+                    break
 
         if text_found:
             try:
@@ -532,7 +549,7 @@ def _transfer_file_models(self, xtf_file_path):
                 self.print_info.emit(
                     self.tr(
                         "Could not parse transferfile file `{file}` ({exception})".format(
-                            file=xtf_file_path, exception=str(e)
+                            file=data_file_path, exception=str(e)
                         )
                     )
                 )