Skip to content

Commit

Permalink
Forward stderr from failing pex process in exception (#114)
Browse files Browse the repository at this point in the history
* Forward stderr from failing pex process in exception

This will allow to easily know the reason of pew build error when the user
only gets the python exception without an easy access to error logs (e.g.
in a notebook)
  • Loading branch information
jcuquemelle authored Nov 3, 2023
1 parent bc14247 commit 73269f0
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 7 deletions.
16 changes: 10 additions & 6 deletions cluster_pack/packaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ class PexTooLargeError(RuntimeError):
pass


class PexCreationError(RuntimeError):
pass


class PythonEnvDescription(NamedTuple):
path_to_archive: str
interpreter_cmd: str
Expand Down Expand Up @@ -181,15 +185,15 @@ def pack_in_pex(requirements: List[str],
call = subprocess.run(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
call.check_returncode()

if not allow_large_pex and os.path.getsize(output + tmp_ext) > 2 * 1024 * 1024 * 1024:
raise PexTooLargeError("The generate pex is larger than 2Gb and won't be executable"
" by python; Please set the 'allow_large_pex' "
"flag in upload_env")

except CalledProcessError as err:
_logger.exception('Cannot create pex')
_logger.exception(err.stderr.decode("ascii"))
raise
raise PexCreationError(err.stderr.decode("ascii"))

if not allow_large_pex and os.path.getsize(output + tmp_ext) > 2 * 1024 * 1024 * 1024:
raise PexTooLargeError("The generate pex is larger than 2Gb and won't be executable"
" by python; Please set the 'allow_large_pex' "
"flag in upload_env")

if allow_large_pex:
shutil.make_archive(output, 'zip', output + tmp_ext)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ def test_format_pex_requirements():
pex_inherit_path="false")
pex_info = PexInfo.from_pex(f"{tempdir}/out.pex")
cleaned_requirements = uploader._format_pex_requirements(pex_info)
pip_version = 'pip==21.3.1' if sys.version_info.minor == 6 else 'pip==23.3'
pip_version = 'pip==21.3.1' if sys.version_info.minor == 6 else 'pip==23.3.1'
assert [pip_version, 'pipdeptree==2.0.0', 'six==1.15.0'] == cleaned_requirements


Expand Down

0 comments on commit 73269f0

Please sign in to comment.