Control test /tmp disk usage. (#2561)

pytest.ini

@@ -0,0 +1,11 @@
+[pytest]
+# This leaves behind the minimum number of prior test run tmp files (just 1 prior run). Setting
+# this to 0 appears to just turn culling off (or perhaps it selects the default of 3). Either way,
+# we choose 1 as a backstop against our primary line of tmp bloat defense, setting a custom
+# `--basetemp` in our test runner script. When this is done, that custom `--basetemp` is always
+# nuked in full prior to test start up.
+tmp_path_retention_count = 1
+
+# N.B.: This does not work with a fair number of our tests under xdist, but it would be good to
+# enable.
+# tmp_path_retention_policy = failed

testing/bin/run_tests.py

@@ -5,11 +5,13 @@
 from __future__ import absolute_import
 
 import atexit
+import getpass
 import logging
 import os
 import re
 import subprocess
 import sys
+import tempfile
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
 
 import coloredlogs
@@ -151,7 +153,19 @@ def main():
     else:
         logger.info("No test control environment variables set.")
 
-    args = [sys.executable, "-m", "pytest", "-n", "auto"]
+    args = [
+        sys.executable,
+        "-m",
+        "pytest",
+        # We set up out own --basetemp (similar to the default), just to leverage the fact that this
+        # gets the prior run's temp files cleaned up just before start. This appears to be the only
+        # way to get this behavior, since the `tmp_path_retention_count` setting only seems to work
+        # for values of 1 or more, which leaves 1 or more prior run temp files lying around.
+        "--basetemp",
+        os.path.join(tempfile.gettempdir(), "pytest-of-{user}-pex".format(user=getpass.getuser())),
+        "-n",
+        "auto",
+    ]
     if options.it:
         args.extend(["tests/integration", "-p", "testing.pytest_shard"])
     else:

tests/integration/test_issue_2203.py

@@ -4,6 +4,7 @@
 import os.path
 import stat
 import subprocess
+from collections import OrderedDict
 
 from pex.common import safe_rmtree
 from pex.typing import TYPE_CHECKING
@@ -48,9 +49,15 @@ def assert_pex_works():
     assert_pex_works()
 
     write_mask = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
+    orig_mode_by_path = OrderedDict()  # type: OrderedDict[str, int]
     for root, dirs, files in os.walk(venv.site_packages_dir, topdown=False):
         for path in files + dirs:
             abs_path = os.path.join(root, path)
-            os.chmod(abs_path, os.stat(abs_path).st_mode & ~write_mask)
-
-    assert_pex_works()
+            orig_mode = os.stat(abs_path).st_mode
+            orig_mode_by_path[abs_path] = orig_mode
+            os.chmod(abs_path, orig_mode & ~write_mask)
+    try:
+        assert_pex_works()
+    finally:
+        for path, mode in orig_mode_by_path.items():
+            os.chmod(path, mode)

tests/test_pex_builder.py

@@ -536,11 +536,12 @@ def write_zipapp(path):
     file_too_big = os.path.join(str(tmpdir), "file_too_big.py")
     with open(file_too_big, "w") as fp:
         fp.write("\n")
+    accum = file_too_big + ".accum"
     for _ in range(32):
-        accum = file_too_big + ".accum"
         os.rename(file_too_big, accum)
         with open(file_too_big, "wb") as dest:
             subprocess.check_call(args=["cat", accum, accum], stdout=dest.fileno())
+    os.unlink(accum)
     assert os.path.getsize(file_too_big) > 0xFFFFFFFF, (
         "ZIP64 must be used if file sizes are bigger than 0xFFFFFFFF per "
         "https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT 4.3.9.2"
@@ -549,11 +550,13 @@ def write_zipapp(path):
     zipapp_too_big = os.path.join(str(tmpdir), "too-big.pyz")
     with write_zipapp(zipapp_too_big) as zf:
         zf.write(file_too_big, os.path.basename(file_too_big))
+    os.unlink(file_too_big)
     assert_perform_check_zip64_handling(
         zipapp_too_big,
         # N.B.: PyPy < 3.8 hangs when trying to run a zip64 app with a too-large file entry.
         test_run=not IS_PYPY or sys.version_info[:2] >= (3, 8),
     )
+    os.unlink(zipapp_too_big)
 
     zipapp_too_many_entries = os.path.join(str(tmpdir), "too-many-entries.pyz")
     with write_zipapp(zipapp_too_many_entries) as zf:

tests/test_zip_utils.py

@@ -37,6 +37,7 @@ def test_zip64_fail_fast(tmpdir):
         ),
     ):
         Zip.load(zip_file)
+    os.unlink(zip_file)
 
 
 def assert_zipapp(