Add support for Python 3.12. (#2172)

The crux here is supporting a version of Pip that works in 3.12. There
is no such released version yet; so this change adds an unreleased
Pip version but goes to some length to hide this version from users
and make it only activatable by those in the know / CI. What follows
is fixing or adjusting many tests. The result is Pex known to work with
Python 3.12 ahead of its release by several months and the spectre of
Pex 3 / a Pex branch split, etc., being forced by Python 3.12 support
dispelled.

It turns out Pex can still ship supporting Python 2.7, 3.5, etc.
along side supporting 3.12. The main trick here is to use
`python3.12 -mvenv` to spirit up a bootstrap Pip that works at least
enough to install the unreleased Pip that truly works with Python 3.12.
Previously all Pip version bootstrapping was handled exclusively by the
vendored Pip.

.github/workflows/ci.yml

@@ -51,15 +51,27 @@ jobs:
           - os: macos-12
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 22_3_1
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 23_1_2
+            tox-env-python: python
+          - os: macos-12
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
+          - os: ubuntu-22.04
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
     steps:
       - name: Calculate Pythons to Expose
         id: calculate-pythons-to-expose
@@ -85,8 +97,9 @@ jobs:
           path: ${{ env._PEX_TEST_PYENV_ROOT }}
           key: ${{ matrix.os }}-${{ runner.arch }}-pyenv-root-v1
       - name: Run Unit Tests
-        uses: pantsbuild/actions/run-tox@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
+        uses: pantsbuild/actions/run-tox@addae8ed8cce2b0a359f447d1bb2ec69ff18f9ca
         with:
+          python: ${{ matrix.tox-env-python }}
           tox-env: py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-pip${{ matrix.pip-version }}
   cpython-unit-tests-legacy:
     name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}
@@ -192,15 +205,27 @@ jobs:
           - os: macos-12
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 20
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 22_3_1
+            tox-env-python: python
           - os: ubuntu-22.04
             python-version: [ 3, 11 ]
             pip-version: 23_1_2
+            tox-env-python: python
+          - os: macos-12
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
+          - os: ubuntu-22.04
+            python-version: [ 3, 12, "0-beta.3" ]
+            pip-version: 23_2
+            tox-env-python: python3.11
     steps:
       - name: Calculate Pythons to Expose
         id: calculate-pythons-to-expose
@@ -236,8 +261,9 @@ jobs:
         with:
           ssh-private-key: ${{ env.SSH_PRIVATE_KEY }}
       - name: Run Integration Tests
-        uses: pantsbuild/actions/run-tox@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
+        uses: pantsbuild/actions/run-tox@addae8ed8cce2b0a359f447d1bb2ec69ff18f9ca
         with:
+          python: ${{ matrix.tox-env-python }}
           tox-env: py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-pip${{ matrix.pip-version }}-integration
   cpython-integration-tests-legacy:
     name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-integration

pex/build_system/pep_517.py

@@ -30,27 +30,31 @@
 
 
 def _default_build_system(
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> BuildSystem
     global _DEFAULT_BUILD_SYSTEMS
-    build_system = _DEFAULT_BUILD_SYSTEMS.get(pip_version)
+    selected_pip_version = pip_version or PipVersion.DEFAULT
+    build_system = _DEFAULT_BUILD_SYSTEMS.get(selected_pip_version)
     if build_system is None:
         with TRACER.timed(
             "Building {build_backend} build_backend PEX".format(build_backend=DEFAULT_BUILD_BACKEND)
         ):
             extra_env = {}  # type: Dict[str, str]
-            if pip_version is PipVersion.VENDORED:
+            if selected_pip_version is PipVersion.VENDORED:
                 requires = ["setuptools", "wheel"]
                 resolved = tuple(
                     Distribution.load(dist_location)
                     for dist_location in third_party.expose(requires)
                 )
                 extra_env.update(__PEX_UNVENDORED__="1")
             else:
-                requires = [pip_version.setuptools_requirement, pip_version.wheel_requirement]
+                requires = [
+                    selected_pip_version.setuptools_requirement,
+                    selected_pip_version.wheel_requirement,
+                ]
                 resolved = tuple(
                     installed_distribution.fingerprinted_distribution.distribution
                     for installed_distribution in resolver.resolve_requirements(
@@ -68,24 +72,24 @@ def _default_build_system(
                     **extra_env
                 )
             )
-            _DEFAULT_BUILD_SYSTEMS[pip_version] = build_system
+            _DEFAULT_BUILD_SYSTEMS[selected_pip_version] = build_system
     return build_system
 
 
 def _get_build_system(
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
     project_directory,  # type: str
     extra_requirements=None,  # type: Optional[Iterable[str]]
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> Union[BuildSystem, Error]
     custom_build_system_or_error = load_build_system(
         target, resolver, project_directory, extra_requirements=extra_requirements
     )
     if custom_build_system_or_error:
         return custom_build_system_or_error
-    return _default_build_system(pip_version, target, resolver)
+    return _default_build_system(target, resolver, pip_version=pip_version)
 
 
 # Exit code 75 is EX_TEMPFAIL defined in /usr/include/sysexits.h
@@ -100,7 +104,6 @@ def is_hook_unavailable_error(error):
 
 def _invoke_build_hook(
     project_directory,  # type: str
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
     hook_method,  # type: str
@@ -109,6 +112,7 @@ def _invoke_build_hook(
     hook_kwargs=None,  # type: Optional[Mapping[str, Any]]
     stdout=None,  # type: Optional[int]
     stderr=None,  # type: Optional[int]
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> Union[SpawnedJob[Any], Error]
 
@@ -126,7 +130,11 @@ def _invoke_build_hook(
         )
 
     build_system_or_error = _get_build_system(
-        pip_version, target, resolver, project_directory, extra_requirements=hook_extra_requirements
+        target,
+        resolver,
+        project_directory,
+        extra_requirements=hook_extra_requirements,
+        pip_version=pip_version,
     )
     if isinstance(build_system_or_error, Error):
         return build_system_or_error
@@ -180,19 +188,19 @@ def _invoke_build_hook(
 def build_sdist(
     project_directory,  # type: str
     dist_dir,  # type: str
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> Union[Text, Error]
 
     extra_requirements = []
     spawned_job_or_error = _invoke_build_hook(
         project_directory,
-        pip_version,
         target,
         resolver,
         hook_method="get_requires_for_build_sdist",
+        pip_version=pip_version,
     )
     if isinstance(spawned_job_or_error, Error):
         return spawned_job_or_error
@@ -208,12 +216,12 @@ def build_sdist(
 
     spawned_job_or_error = _invoke_build_hook(
         project_directory,
-        pip_version,
         target,
         resolver,
         hook_method="build_sdist",
         hook_args=[dist_dir],
         hook_extra_requirements=extra_requirements,
+        pip_version=pip_version,
     )
     if isinstance(spawned_job_or_error, Error):
         return spawned_job_or_error
@@ -229,20 +237,20 @@ def build_sdist(
 
 def spawn_prepare_metadata(
     project_directory,  # type: str
-    pip_version,  # type: PipVersionValue
     target,  # type: Target
     resolver,  # type: Resolver
+    pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> SpawnedJob[DistMetadata]
 
     extra_requirements = []
     spawned_job = try_(
         _invoke_build_hook(
             project_directory,
-            pip_version,
             target,
             resolver,
             hook_method="get_requires_for_build_wheel",
+            pip_version=pip_version,
         )
     )
     try:
@@ -256,12 +264,12 @@ def spawn_prepare_metadata(
     spawned_job = try_(
         _invoke_build_hook(
             project_directory,
-            pip_version,
             target,
             resolver,
             hook_method="prepare_metadata_for_build_wheel",
             hook_args=[build_dir],
             hook_extra_requirements=extra_requirements,
+            pip_version=pip_version,
         )
     )
     return spawned_job.map(lambda _: DistMetadata.load(build_dir))

pex/build_system/testing.py

@@ -37,16 +37,16 @@ def assert_expected_dist(dist):
 
     sdist_dir = os.path.join(str(tmpdir), "sdist_dir")
 
-    # This test utility is used by all versions of Python Pex supports; so we need to use the
-    # vendored Pip which is guaranteed to work with all those Python versions.
-    pip_version = PipVersion.VENDORED
+    # This test utility is used by all versions of Python Pex supports; so we need to use a Pip
+    # which is guaranteed to work with the current Python version.
+    pip_version = PipVersion.DEFAULT
 
     location = build_sdist(
         project_dir,
         sdist_dir,
-        pip_version,
         LocalInterpreter.create(),
         ConfiguredResolver(PipConfiguration(version=pip_version)),
+        pip_version=pip_version,
     )
     assert not isinstance(location, Error), location
     assert sdist_dir == os.path.dirname(location)
@@ -56,7 +56,9 @@ def assert_expected_dist(dist):
 
     # Verify the sdist is valid such that we can build a wheel from it.
     wheel_dir = os.path.join(str(tmpdir), "wheel_dir")
-    get_pip().spawn_build_wheels(distributions=[sdist.location], wheel_dir=wheel_dir).wait()
+    get_pip(resolver=ConfiguredResolver(pip_configuration=PipConfiguration())).spawn_build_wheels(
+        distributions=[sdist.location], wheel_dir=wheel_dir
+    ).wait()
     wheels = glob.glob(os.path.join(wheel_dir, "*.whl"))
     assert 1 == len(wheels)
     assert_expected_dist(Distribution.load(wheels[0]))

pex/common.py

@@ -630,6 +630,8 @@ def get_parent_dir(path):
 
             def maybe_write_parent_dirs(path):
                 # type: (str) -> None
+                if path == strip_prefix:
+                    return
                 parent_dir = get_parent_dir(path)
                 if parent_dir is None or parent_dir in written_dirs:
                     return

pex/environment.py

@@ -14,12 +14,13 @@
 from pex.dist_metadata import Distribution, Requirement
 from pex.fingerprinted_distribution import FingerprintedDistribution
 from pex.inherit_path import InheritPath
+from pex.interpreter import PythonInterpreter
 from pex.layout import maybe_install
 from pex.orderedset import OrderedSet
 from pex.pep_425 import CompatibilityTags, TagRank
 from pex.pep_503 import ProjectName
 from pex.pex_info import PexInfo
-from pex.targets import Target
+from pex.targets import LocalInterpreter, Target
 from pex.third_party.packaging import specifiers
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
@@ -52,9 +53,12 @@ def _import_pkg_resources():
         from pex import third_party
 
         third_party.install(expose=["setuptools"])
-        import pkg_resources  # vendor:skip
+        try:
+            import pkg_resources  # vendor:skip
 
-        return pkg_resources, True
+            return pkg_resources, True
+        except ImportError:
+            return None, False
 
 
 @attr.s(frozen=True)
@@ -623,26 +627,26 @@ def _declare_namespace_packages(cls, resolved_dists):
         # will be active in the pex environment at runtime and, as such, care must be taken.
         #
         # Properly behaved distributions will declare a dependency on `setuptools`, in which case we
-        # use that (non-vendored) distribution. A side-effect of importing `pkg_resources` from that
+        # use that (non-vendored) distribution. A side effect of importing `pkg_resources` from that
         # distribution is that a global `pkg_resources.working_set` will be populated. For various
         # `pkg_resources` distribution discovery functions to work, that global
-        # `pkg_resources.working_set` must be built with the `sys.path` fully settled. Since all dists
-        # in the dependency set (`resolved_dists`) have already been resolved and added to the
+        # `pkg_resources.working_set` must be built with the `sys.path` fully settled. Since all
+        # dists in the dependency set (`resolved_dists`) have already been resolved and added to the
         # `sys.path` we're safe to proceed here.
         #
         # Other distributions (notably `twitter.common.*`) in the wild declare `setuptools`-specific
-        # `namespace_packages` but do not properly declare a dependency on `setuptools` which they must
-        # use to:
+        # `namespace_packages` but do not properly declare a dependency on `setuptools` which they
+        # must use to:
         # 1. Declare `namespace_packages` metadata which we just verified they have with the check
         #    above.
         # 2. Declare namespace packages at runtime via the canonical:
         #    `__import__('pkg_resources').declare_namespace(__name__)`
         #
         # For such distributions we fall back to our vendored version of `setuptools`. This is safe,
-        # since we'll only introduce our shaded version when no other standard version is present and
-        # even then tear it all down when we hand off from the bootstrap to user code.
+        # since we'll only introduce our shaded version when no other standard version is present
+        # and even then tear it all down when we hand off from the bootstrap to user code.
         pkg_resources, vendored = _import_pkg_resources()
-        if vendored:
+        if not pkg_resources or vendored:
             dists = "\n".join(
                 "\n{index}. {dist} namespace packages:\n  {ns_packages}".format(
                     index=index + 1,
@@ -651,6 +655,21 @@ def _declare_namespace_packages(cls, resolved_dists):
                 )
                 for index, (dist, ns_packages) in enumerate(namespace_packages_by_dist.items())
             )
+            if not pkg_resources:
+                current_interpreter = PythonInterpreter.get()
+                pex_warnings.warn(
+                    "The legacy `pkg_resources` package cannot be imported by the "
+                    "{interpreter} {version} interpreter at {path}.\n"
+                    "The following distributions need `pkg_resources` to load some legacy "
+                    "namespace packages and may fail to work properly:\n{dists}".format(
+                        interpreter=current_interpreter.identity.interpreter,
+                        version=current_interpreter.python,
+                        path=current_interpreter.binary,
+                        dists=dists,
+                    )
+                )
+                return
+
             pex_warnings.warn(
                 "The `pkg_resources` package was loaded from a pex vendored version when "
                 "declaring namespace packages defined by:\n{dists}\n\nThese distributions "

pex/interpreter_constraints.py

@@ -316,11 +316,12 @@ def iter_compatible_versions(
     # N.B.: Pex does not support the missing 3.x versions here.
     PythonVersion(Lifecycle.EOL, 3, 5, 10),
     PythonVersion(Lifecycle.EOL, 3, 6, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 7, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 8, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 9, 15),
-    PythonVersion(Lifecycle.STABLE, 3, 10, 8),
-    PythonVersion(Lifecycle.STABLE, 3, 11, 0),
+    # ^-- EOL --^
+    PythonVersion(Lifecycle.STABLE, 3, 7, 17),
+    PythonVersion(Lifecycle.STABLE, 3, 8, 17),
+    PythonVersion(Lifecycle.STABLE, 3, 9, 17),
+    PythonVersion(Lifecycle.STABLE, 3, 10, 12),
+    PythonVersion(Lifecycle.STABLE, 3, 11, 4),
     PythonVersion(Lifecycle.DEV, 3, 12, 0),
 )
 

pex/pex_builder.py

@@ -752,7 +752,7 @@ def zip_cache_dir(path):
             os.mkdir(internal_cache)
             for location, fingerprint in pex_info.distributions.items():
                 cached_installed_wheel_zip_dir = zip_cache_dir(
-                    os.path.join(pex_info.pex_root, "installed_wheel_zips", fingerprint)
+                    os.path.join(pex_info.pex_root, "packed_wheels", fingerprint)
                 )
                 with atomic_directory(cached_installed_wheel_zip_dir) as atomic_zip_dir:
                     if not atomic_zip_dir.is_finalized():