Merge pull request #160 from espressif/fix/parallel_index

Fix/parallel index

idf_build_apps/app.py

@@ -74,7 +74,6 @@ class App(BaseModel):
     NAME_PLACEHOLDER: t.ClassVar[str] = '@n'  # replace it with self.name
     FULL_NAME_PLACEHOLDER: t.ClassVar[str] = '@f'  # replace it with escaped self.app_dir
     IDF_VERSION_PLACEHOLDER: t.ClassVar[str] = '@v'  # replace it with the IDF version
-    PARALLEL_INDEX_PLACEHOLDER: t.ClassVar[str] = '@p'  # replace it with the parallel index
     INDEX_PLACEHOLDER: t.ClassVar[str] = '@i'  # replace it with the build index (while build_apps)
 
     SDKCONFIG_LINE_REGEX: t.ClassVar[t.Pattern] = re.compile(r'^([^=]+)=\"?([^\"\n]*)\"?\n*$')
@@ -115,7 +114,6 @@ class App(BaseModel):
     copy_sdkconfig: bool = False
 
     # build_apps() related
-    parallel_index: t.Optional[int] = None  # used for expand
     index: t.Optional[int] = None
 
     # build status related
@@ -244,8 +242,6 @@ def _expand(self, path):
 
         if self.index is not None:
             path = path.replace(self.INDEX_PLACEHOLDER, str(self.index))
-        if self.parallel_index:
-            path = path.replace(self.PARALLEL_INDEX_PLACEHOLDER, str(self.parallel_index))
         path = path.replace(
             self.IDF_VERSION_PLACEHOLDER, f'{IDF_VERSION_MAJOR}_{IDF_VERSION_MINOR}_{IDF_VERSION_PATCH}'
         )

idf_build_apps/args.py

@@ -760,22 +760,24 @@ class BuildArguments(FindBuildArguments):
         default=None,
         metadata=asdict(
             FieldMetadata(
-                description='[INTERNAL CI USE] record size json filepath of the built apps to the specified file. '
+                description='Record size json filepath of the built apps to the specified file. '
                 'Each line is a json string. Can expand placeholders @p',
             )
         ),
     )
+    _collect_size_info: t.Optional[str] = field(init=False, repr=False, default=None)
     collect_app_info: t.Optional[str] = field(
         default=None,
         metadata=asdict(
             FieldMetadata(
-                description='[INTERNAL CI USE] record serialized app model of the built apps to the specified file. '
+                description='Record serialized app model of the built apps to the specified file. '
                 'Each line is a json string. Can expand placeholders @p',
             )
         ),
     )
+    _collect_app_info: t.Optional[str] = field(init=False, repr=False, default=None)
     ignore_warning_str: InitVar[t.Optional[t.List[str]]] = _Field.UNSET
-    ignore_warning_strings: t.Optional[t.List[str]] = field(
+    ignore_warning_strs: t.Optional[t.List[str]] = field(
         default=None,
         metadata=asdict(
             FieldMetadata(
@@ -816,6 +818,10 @@ class BuildArguments(FindBuildArguments):
             )
         ),
     )
+    _junitxml: t.Optional[str] = field(init=False, repr=False, default=None)
+
+    # used for expanding placeholders
+    PARALLEL_INDEX_PLACEHOLDER: t.ClassVar[str] = '@p'  # replace it with the parallel index
 
     def __post_init__(  # type: ignore
         self,
@@ -841,20 +847,71 @@ def __post_init__(  # type: ignore
             config_rules_str=config_rules_str,
         )
 
-        self.set_deprecated_field('ignore_warning_strings', 'ignore_warning_str', ignore_warning_str)
+        self.set_deprecated_field('ignore_warning_strs', 'ignore_warning_str', ignore_warning_str)
         self.set_deprecated_field('ignore_warning_files', 'ignore_warning_file', ignore_warning_file)
 
-        self.ignore_warning_strings = to_list(self.ignore_warning_strings) or []
+        self.ignore_warning_strs = to_list(self.ignore_warning_strs) or []
 
         ignore_warnings_regexes = []
-        if self.ignore_warning_strings:
-            for s in self.ignore_warning_strings:
+        if self.ignore_warning_strs:
+            for s in self.ignore_warning_strs:
                 ignore_warnings_regexes.append(re.compile(s))
         if self.ignore_warning_files:
             for s in self.ignore_warning_files:
                 ignore_warnings_regexes.append(re.compile(s.strip()))
         App.IGNORE_WARNS_REGEXES = ignore_warnings_regexes
 
+        if not isinstance(BuildArguments.collect_size_info, property):
+            self._collect_size_info = self.collect_size_info
+            BuildArguments.collect_size_info = property(  # type: ignore
+                BuildArguments._get_collect_size_info,
+                BuildArguments._set_collect_size_info,
+            )
+
+        if not isinstance(BuildArguments.collect_app_info, property):
+            self._collect_app_info = self.collect_app_info
+            BuildArguments.collect_app_info = property(  # type: ignore
+                BuildArguments._get_collect_app_info,
+                BuildArguments._set_collect_app_info,
+            )
+
+        if not isinstance(BuildArguments.junitxml, property):
+            self._junitxml = self.junitxml
+            BuildArguments.junitxml = property(  # type: ignore
+                BuildArguments._get_junitxml,
+                BuildArguments._set_junitxml,
+            )
+
+    def _get_collect_size_info(self) -> t.Optional[str]:
+        return (
+            self._collect_size_info.replace(self.PARALLEL_INDEX_PLACEHOLDER, str(self.parallel_index))
+            if self._collect_size_info
+            else None
+        )
+
+    def _set_collect_size_info(self, k: str) -> None:
+        self._collect_size_info = k
+
+    def _get_collect_app_info(self) -> t.Optional[str]:
+        return (
+            self._collect_app_info.replace(self.PARALLEL_INDEX_PLACEHOLDER, str(self.parallel_index))
+            if self._collect_app_info
+            else None
+        )
+
+    def _set_collect_app_info(self, k: str) -> None:
+        self._collect_app_info = k
+
+    def _get_junitxml(self) -> t.Optional[str]:
+        return (
+            self._junitxml.replace(self.PARALLEL_INDEX_PLACEHOLDER, str(self.parallel_index))
+            if self._junitxml
+            else None
+        )
+
+    def _set_junitxml(self, k: str) -> None:
+        self._junitxml = k
+
 
 @dataclass
 class DumpManifestShaArguments(GlobalArguments):

idf_build_apps/main.py

@@ -125,9 +125,6 @@ def build_apps(
     start, stop = get_parallel_start_stop(len(apps), build_arguments.parallel_count, build_arguments.parallel_index)
     LOGGER.info('Total %s apps. running build for app %s-%s', len(apps), start, stop)
 
-    for app in apps[start - 1 : stop]:  # we use 1-based
-        app.parallel_index = build_arguments.parallel_index
-
     # cleanup collect files if exists at this early-stage
     for f in (build_arguments.collect_app_info, build_arguments.collect_size_info, build_arguments.junitxml):
         if f and os.path.isfile(f):

tests/test_args.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-from idf_build_apps.args import DependencyDrivenBuildArguments, FindArguments, FindBuildArguments
+from idf_build_apps.args import BuildArguments, DependencyDrivenBuildArguments, FindArguments, FindBuildArguments
 from idf_build_apps.config import IDF_BUILD_APPS_TOML_FN
 
 
@@ -64,3 +64,19 @@ def test_apply_config_with_deprecated_names(tmp_path, capsys):
 def test_empty_argument():
     args = FindArguments()
     assert args.config_rules is None
+
+
+def test_build_args_expansion():
+    args = BuildArguments(parallel_index=2)
+
+    args.collect_app_info = '@p.txt'
+    assert args.collect_app_info == '2.txt'
+
+    args.parallel_index = 3
+    assert args.collect_app_info == '3.txt'
+
+    args.junitxml = 'x_@p.txt'
+    assert args.junitxml == 'x_3.txt'
+
+    args.collect_size_info = '@p_@p.txt'
+    assert args.collect_size_info == '3_3.txt'