From 38d6715e1be2ae7466d12526350b0351828fb85e Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Mon, 18 Nov 2024 03:48:10 +0200 Subject: [PATCH 1/5] fix: unify :zip path normalization --- package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.py b/package.py index dfdbb65b..0cee1eb0 100644 --- a/package.py +++ b/package.py @@ -770,8 +770,9 @@ def commands_step(path, commands): _path = os.path.normpath(os.path.join(path, _path)) step("zip:embedded", _path, prefix) elif len(c) == 2: - prefix = None _, _path = c + prefix = None + _path = os.path.normpath(_path) step("zip:embedded", _path, prefix) elif len(c) == 1: prefix = None @@ -862,6 +863,7 @@ def commands_step(path, commands): tmp_dir=claim.get("npm_tmp_dir"), ) if path: + path = os.path.normpath(path) step("zip", path, prefix) if patterns: # Take patterns into account when computing hash From aa2cb10b4fe9908655c69e1bc77af1f0257e12e8 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Mon, 18 Nov 2024 05:45:49 +0200 Subject: [PATCH 2/5] fix: make `source_path` blocks independent The `source_path` blocks don't impact anymore to each other by workdir changes --- package.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/package.py b/package.py index 0cee1eb0..938dd3cb 100644 --- a/package.py +++ b/package.py @@ -776,7 +776,8 @@ def commands_step(path, commands): step("zip:embedded", _path, prefix) elif len(c) == 1: prefix = None - step("zip:embedded", path, prefix) + _path = None + step("zip:embedded", _path, prefix) else: raise ValueError( ":zip invalid call signature, use: " @@ -788,6 +789,8 @@ def commands_step(path, commands): step("sh", path, "\n".join(batch)) batch.clear() + step("reset:workdir") + for claim in claims: if isinstance(claim, str): path = claim @@ -884,6 +887,8 @@ def commands_step(path, commands): return build_plan def execute(self, build_plan, zip_stream, query): + tf_work_dir = os.getcwd() + zs = zip_stream sh_work_dir = None pf = None @@ -893,10 +898,14 @@ def execute(self, build_plan, zip_stream, query): if cmd.startswith("zip"): ts = 0 if cmd == "zip:embedded" else None source_path, prefix = action[1:] - if sh_work_dir: - if source_path != sh_work_dir: - if not os.path.isfile(source_path): - source_path = sh_work_dir + if not sh_work_dir: + sh_work_dir = tf_work_dir + log.info("WORKDIR: %s", sh_work_dir) + if source_path: + if not os.path.isabs(source_path): + source_path = os.path.join(sh_work_dir, source_path) + else: + source_path = sh_work_dir if os.path.isdir(source_path): if pf: self._zip_write_with_filter( @@ -944,10 +953,17 @@ def execute(self, build_plan, zip_stream, query): elif cmd == "sh": with tempfile.NamedTemporaryFile(mode="w+t", delete=True) as temp_file: path, script = action[1:] - # NOTE: Execute `pwd` to determine the subprocess shell's working directory after having executed all other commands. + + if not path: + path = tf_work_dir + if not os.path.isabs(path): + path = os.path.join(tf_work_dir, path) + script = "\n".join( ( script, + # NOTE: Execute `pwd` to determine the subprocess shell's + # working directory after having executed all other commands. "retcode=$?", f"pwd >{temp_file.name}", "exit $retcode", @@ -968,7 +984,7 @@ def execute(self, build_plan, zip_stream, query): # NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop sh_work_dir = temp_file.read().strip() - log.info("WD: %s", sh_work_dir) + log.info("WORKDIR: %s", sh_work_dir) call_stdout, call_stderr = p.communicate() exit_code = p.returncode @@ -981,6 +997,8 @@ def execute(self, build_plan, zip_stream, query): call_stderr.decode("utf-8").strip(), ) ) + elif cmd == "reset:workdir": + sh_work_dir = tf_work_dir elif cmd == "set:filter": patterns = action[1] pf = ZipContentFilter(args=self._args) From a4fd2e1ab16c4b382263ed6b274e0ac32a0e5526 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Mon, 18 Nov 2024 06:17:25 +0200 Subject: [PATCH 3/5] feat: allow to don't specify `path` in `source_path` and use explicit :zip commands --- package.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/package.py b/package.py index 938dd3cb..0da66706 100644 --- a/package.py +++ b/package.py @@ -676,8 +676,11 @@ def plan(self, source_path, query): source_paths = [] build_plan = [] - step = lambda *x: build_plan.append(x) - hash = source_paths.append + def step(*x): + build_plan.append(x) + + def hash(path): + source_paths.append(path) def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): command = runtime @@ -753,13 +756,6 @@ def commands_step(path, commands): if c.startswith(":zip"): if path: hash(path) - else: - # If path doesn't defined for a block with - # commands it will be set to Terraform's - # current working directory - # NB: cwd may vary when using Terraform 0.14+ like: - # `terraform -chdir=...` - path = query.paths.cwd if batch: step("sh", path, "\n".join(batch)) batch.clear() From fe72310e2483baebf2c8a564f33b807371a3f937 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Mon, 18 Nov 2024 06:51:47 +0200 Subject: [PATCH 4/5] fix: improve sh exec step debug logging --- package.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/package.py b/package.py index 0da66706..56cf9bb9 100644 --- a/package.py +++ b/package.py @@ -572,6 +572,10 @@ def compile(self, patterns): rules.append((None, r)) self._rules = rules + def reset(self): + self._log.debug("reset filter patterns") + self._rules = None + def filter(self, path, prefix=None): path = os.path.normpath(path) if prefix: @@ -883,6 +887,8 @@ def commands_step(path, commands): return build_plan def execute(self, build_plan, zip_stream, query): + sh_log = logging.getLogger("sh") + tf_work_dir = os.getcwd() zs = zip_stream @@ -896,7 +902,7 @@ def execute(self, build_plan, zip_stream, query): source_path, prefix = action[1:] if not sh_work_dir: sh_work_dir = tf_work_dir - log.info("WORKDIR: %s", sh_work_dir) + log.debug("WORKDIR: %s", sh_work_dir) if source_path: if not os.path.isabs(source_path): source_path = os.path.join(sh_work_dir, source_path) @@ -955,6 +961,11 @@ def execute(self, build_plan, zip_stream, query): if not os.path.isabs(path): path = os.path.join(tf_work_dir, path) + if log.isEnabledFor(DEBUG2): + log.debug("exec shell script ...") + for line in script.splitlines(): + sh_log.debug(line) + script = "\n".join( ( script, @@ -974,17 +985,9 @@ def execute(self, build_plan, zip_stream, query): cwd=path, ) - p.wait() - temp_file.seek(0) - - # NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop - sh_work_dir = temp_file.read().strip() - - log.info("WORKDIR: %s", sh_work_dir) - call_stdout, call_stderr = p.communicate() exit_code = p.returncode - log.info("exit_code: %s", exit_code) + log.debug("exit_code: %s", exit_code) if exit_code != 0: raise RuntimeError( "Script did not run successfully, exit code {}: {} - {}".format( @@ -993,13 +996,21 @@ def execute(self, build_plan, zip_stream, query): call_stderr.decode("utf-8").strip(), ) ) + + temp_file.seek(0) + # NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop + sh_work_dir = temp_file.read().strip() + log.debug("WORKDIR: %s", sh_work_dir) + elif cmd == "reset:workdir": sh_work_dir = tf_work_dir + log.debug("WORKDIR: %s", sh_work_dir) elif cmd == "set:filter": patterns = action[1] pf = ZipContentFilter(args=self._args) pf.compile(patterns) elif cmd == "clear:filter": + pf.reset() pf = None @staticmethod From 0224ce00bf1f84da4cf25c04899e1a7a8149f969 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Mon, 18 Nov 2024 07:07:17 +0200 Subject: [PATCH 5/5] fix: pass tests --- package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/package.py b/package.py index 56cf9bb9..8cbeba30 100644 --- a/package.py +++ b/package.py @@ -905,7 +905,9 @@ def execute(self, build_plan, zip_stream, query): log.debug("WORKDIR: %s", sh_work_dir) if source_path: if not os.path.isabs(source_path): - source_path = os.path.join(sh_work_dir, source_path) + source_path = os.path.normpath( + os.path.join(sh_work_dir, source_path) + ) else: source_path = sh_work_dir if os.path.isdir(source_path): @@ -959,7 +961,7 @@ def execute(self, build_plan, zip_stream, query): if not path: path = tf_work_dir if not os.path.isabs(path): - path = os.path.join(tf_work_dir, path) + path = os.path.normpath(os.path.join(tf_work_dir, path)) if log.isEnabledFor(DEBUG2): log.debug("exec shell script ...")