diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3a8848c4..bef77bbd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,15 +51,27 @@ jobs: - os: macos-12 python-version: [ 3, 11 ] pip-version: 20 + tox-env-python: python - os: ubuntu-22.04 python-version: [ 3, 11 ] pip-version: 20 + tox-env-python: python - os: ubuntu-22.04 python-version: [ 3, 11 ] pip-version: 22_3_1 + tox-env-python: python - os: ubuntu-22.04 python-version: [ 3, 11 ] pip-version: 23_1_2 + tox-env-python: python + - os: macos-12 + python-version: [ 3, 12, "0-beta.3" ] + pip-version: 23_2 + tox-env-python: python3.11 + - os: ubuntu-22.04 + python-version: [ 3, 12, "0-beta.3" ] + pip-version: 23_2 + tox-env-python: python3.11 steps: - name: Calculate Pythons to Expose id: calculate-pythons-to-expose @@ -85,8 +97,9 @@ jobs: path: ${{ env._PEX_TEST_PYENV_ROOT }} key: ${{ matrix.os }}-${{ runner.arch }}-pyenv-root-v1 - name: Run Unit Tests - uses: pantsbuild/actions/run-tox@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb + uses: pantsbuild/actions/run-tox@addae8ed8cce2b0a359f447d1bb2ec69ff18f9ca with: + python: ${{ matrix.tox-env-python }} tox-env: py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-pip${{ matrix.pip-version }} cpython-unit-tests-legacy: name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }} @@ -192,15 +205,27 @@ jobs: - os: macos-12 python-version: [ 3, 11 ] pip-version: 20 + tox-env-python: python - os: ubuntu-22.04 python-version: [ 3, 11 ] pip-version: 20 + tox-env-python: python - os: ubuntu-22.04 python-version: [ 3, 11 ] pip-version: 22_3_1 + tox-env-python: python - os: ubuntu-22.04 python-version: [ 3, 11 ] pip-version: 23_1_2 + tox-env-python: python + - os: macos-12 + python-version: [ 3, 12, "0-beta.3" ] + pip-version: 23_2 + tox-env-python: python3.11 + - os: ubuntu-22.04 + python-version: [ 3, 12, "0-beta.3" ] + pip-version: 23_2 + tox-env-python: python3.11 steps: - name: Calculate Pythons to Expose id: calculate-pythons-to-expose @@ -236,8 +261,9 @@ jobs: with: ssh-private-key: ${{ env.SSH_PRIVATE_KEY }} - name: Run Integration Tests - uses: pantsbuild/actions/run-tox@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb + uses: pantsbuild/actions/run-tox@addae8ed8cce2b0a359f447d1bb2ec69ff18f9ca with: + python: ${{ matrix.tox-env-python }} tox-env: py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-pip${{ matrix.pip-version }}-integration cpython-integration-tests-legacy: name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-integration diff --git a/pex/build_system/pep_517.py b/pex/build_system/pep_517.py index 57a32c91f..0abed252b 100644 --- a/pex/build_system/pep_517.py +++ b/pex/build_system/pep_517.py @@ -30,19 +30,20 @@ def _default_build_system( - pip_version, # type: PipVersionValue target, # type: Target resolver, # type: Resolver + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> BuildSystem global _DEFAULT_BUILD_SYSTEMS - build_system = _DEFAULT_BUILD_SYSTEMS.get(pip_version) + selected_pip_version = pip_version or PipVersion.DEFAULT + build_system = _DEFAULT_BUILD_SYSTEMS.get(selected_pip_version) if build_system is None: with TRACER.timed( "Building {build_backend} build_backend PEX".format(build_backend=DEFAULT_BUILD_BACKEND) ): extra_env = {} # type: Dict[str, str] - if pip_version is PipVersion.VENDORED: + if selected_pip_version is PipVersion.VENDORED: requires = ["setuptools", "wheel"] resolved = tuple( Distribution.load(dist_location) @@ -50,7 +51,10 @@ def _default_build_system( ) extra_env.update(__PEX_UNVENDORED__="1") else: - requires = [pip_version.setuptools_requirement, pip_version.wheel_requirement] + requires = [ + selected_pip_version.setuptools_requirement, + selected_pip_version.wheel_requirement, + ] resolved = tuple( installed_distribution.fingerprinted_distribution.distribution for installed_distribution in resolver.resolve_requirements( @@ -68,16 +72,16 @@ def _default_build_system( **extra_env ) ) - _DEFAULT_BUILD_SYSTEMS[pip_version] = build_system + _DEFAULT_BUILD_SYSTEMS[selected_pip_version] = build_system return build_system def _get_build_system( - pip_version, # type: PipVersionValue target, # type: Target resolver, # type: Resolver project_directory, # type: str extra_requirements=None, # type: Optional[Iterable[str]] + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> Union[BuildSystem, Error] custom_build_system_or_error = load_build_system( @@ -85,7 +89,7 @@ def _get_build_system( ) if custom_build_system_or_error: return custom_build_system_or_error - return _default_build_system(pip_version, target, resolver) + return _default_build_system(target, resolver, pip_version=pip_version) # Exit code 75 is EX_TEMPFAIL defined in /usr/include/sysexits.h @@ -100,7 +104,6 @@ def is_hook_unavailable_error(error): def _invoke_build_hook( project_directory, # type: str - pip_version, # type: PipVersionValue target, # type: Target resolver, # type: Resolver hook_method, # type: str @@ -109,6 +112,7 @@ def _invoke_build_hook( hook_kwargs=None, # type: Optional[Mapping[str, Any]] stdout=None, # type: Optional[int] stderr=None, # type: Optional[int] + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> Union[SpawnedJob[Any], Error] @@ -126,7 +130,11 @@ def _invoke_build_hook( ) build_system_or_error = _get_build_system( - pip_version, target, resolver, project_directory, extra_requirements=hook_extra_requirements + target, + resolver, + project_directory, + extra_requirements=hook_extra_requirements, + pip_version=pip_version, ) if isinstance(build_system_or_error, Error): return build_system_or_error @@ -180,19 +188,19 @@ def _invoke_build_hook( def build_sdist( project_directory, # type: str dist_dir, # type: str - pip_version, # type: PipVersionValue target, # type: Target resolver, # type: Resolver + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> Union[Text, Error] extra_requirements = [] spawned_job_or_error = _invoke_build_hook( project_directory, - pip_version, target, resolver, hook_method="get_requires_for_build_sdist", + pip_version=pip_version, ) if isinstance(spawned_job_or_error, Error): return spawned_job_or_error @@ -208,12 +216,12 @@ def build_sdist( spawned_job_or_error = _invoke_build_hook( project_directory, - pip_version, target, resolver, hook_method="build_sdist", hook_args=[dist_dir], hook_extra_requirements=extra_requirements, + pip_version=pip_version, ) if isinstance(spawned_job_or_error, Error): return spawned_job_or_error @@ -229,9 +237,9 @@ def build_sdist( def spawn_prepare_metadata( project_directory, # type: str - pip_version, # type: PipVersionValue target, # type: Target resolver, # type: Resolver + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> SpawnedJob[DistMetadata] @@ -239,10 +247,10 @@ def spawn_prepare_metadata( spawned_job = try_( _invoke_build_hook( project_directory, - pip_version, target, resolver, hook_method="get_requires_for_build_wheel", + pip_version=pip_version, ) ) try: @@ -256,12 +264,12 @@ def spawn_prepare_metadata( spawned_job = try_( _invoke_build_hook( project_directory, - pip_version, target, resolver, hook_method="prepare_metadata_for_build_wheel", hook_args=[build_dir], hook_extra_requirements=extra_requirements, + pip_version=pip_version, ) ) return spawned_job.map(lambda _: DistMetadata.load(build_dir)) diff --git a/pex/build_system/testing.py b/pex/build_system/testing.py index 974969526..eca8ec8ad 100644 --- a/pex/build_system/testing.py +++ b/pex/build_system/testing.py @@ -37,16 +37,16 @@ def assert_expected_dist(dist): sdist_dir = os.path.join(str(tmpdir), "sdist_dir") - # This test utility is used by all versions of Python Pex supports; so we need to use the - # vendored Pip which is guaranteed to work with all those Python versions. - pip_version = PipVersion.VENDORED + # This test utility is used by all versions of Python Pex supports; so we need to use a Pip + # which is guaranteed to work with the current Python version. + pip_version = PipVersion.DEFAULT location = build_sdist( project_dir, sdist_dir, - pip_version, LocalInterpreter.create(), ConfiguredResolver(PipConfiguration(version=pip_version)), + pip_version=pip_version, ) assert not isinstance(location, Error), location assert sdist_dir == os.path.dirname(location) @@ -56,7 +56,9 @@ def assert_expected_dist(dist): # Verify the sdist is valid such that we can build a wheel from it. wheel_dir = os.path.join(str(tmpdir), "wheel_dir") - get_pip().spawn_build_wheels(distributions=[sdist.location], wheel_dir=wheel_dir).wait() + get_pip(resolver=ConfiguredResolver(pip_configuration=PipConfiguration())).spawn_build_wheels( + distributions=[sdist.location], wheel_dir=wheel_dir + ).wait() wheels = glob.glob(os.path.join(wheel_dir, "*.whl")) assert 1 == len(wheels) assert_expected_dist(Distribution.load(wheels[0])) diff --git a/pex/common.py b/pex/common.py index 8031243c7..a4bada46a 100644 --- a/pex/common.py +++ b/pex/common.py @@ -630,6 +630,8 @@ def get_parent_dir(path): def maybe_write_parent_dirs(path): # type: (str) -> None + if path == strip_prefix: + return parent_dir = get_parent_dir(path) if parent_dir is None or parent_dir in written_dirs: return diff --git a/pex/environment.py b/pex/environment.py index df49a6ace..ba4694b77 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -14,12 +14,13 @@ from pex.dist_metadata import Distribution, Requirement from pex.fingerprinted_distribution import FingerprintedDistribution from pex.inherit_path import InheritPath +from pex.interpreter import PythonInterpreter from pex.layout import maybe_install from pex.orderedset import OrderedSet from pex.pep_425 import CompatibilityTags, TagRank from pex.pep_503 import ProjectName from pex.pex_info import PexInfo -from pex.targets import Target +from pex.targets import LocalInterpreter, Target from pex.third_party.packaging import specifiers from pex.tracer import TRACER from pex.typing import TYPE_CHECKING @@ -52,9 +53,12 @@ def _import_pkg_resources(): from pex import third_party third_party.install(expose=["setuptools"]) - import pkg_resources # vendor:skip + try: + import pkg_resources # vendor:skip - return pkg_resources, True + return pkg_resources, True + except ImportError: + return None, False @attr.s(frozen=True) @@ -623,26 +627,26 @@ def _declare_namespace_packages(cls, resolved_dists): # will be active in the pex environment at runtime and, as such, care must be taken. # # Properly behaved distributions will declare a dependency on `setuptools`, in which case we - # use that (non-vendored) distribution. A side-effect of importing `pkg_resources` from that + # use that (non-vendored) distribution. A side effect of importing `pkg_resources` from that # distribution is that a global `pkg_resources.working_set` will be populated. For various # `pkg_resources` distribution discovery functions to work, that global - # `pkg_resources.working_set` must be built with the `sys.path` fully settled. Since all dists - # in the dependency set (`resolved_dists`) have already been resolved and added to the + # `pkg_resources.working_set` must be built with the `sys.path` fully settled. Since all + # dists in the dependency set (`resolved_dists`) have already been resolved and added to the # `sys.path` we're safe to proceed here. # # Other distributions (notably `twitter.common.*`) in the wild declare `setuptools`-specific - # `namespace_packages` but do not properly declare a dependency on `setuptools` which they must - # use to: + # `namespace_packages` but do not properly declare a dependency on `setuptools` which they + # must use to: # 1. Declare `namespace_packages` metadata which we just verified they have with the check # above. # 2. Declare namespace packages at runtime via the canonical: # `__import__('pkg_resources').declare_namespace(__name__)` # # For such distributions we fall back to our vendored version of `setuptools`. This is safe, - # since we'll only introduce our shaded version when no other standard version is present and - # even then tear it all down when we hand off from the bootstrap to user code. + # since we'll only introduce our shaded version when no other standard version is present + # and even then tear it all down when we hand off from the bootstrap to user code. pkg_resources, vendored = _import_pkg_resources() - if vendored: + if not pkg_resources or vendored: dists = "\n".join( "\n{index}. {dist} namespace packages:\n {ns_packages}".format( index=index + 1, @@ -651,6 +655,21 @@ def _declare_namespace_packages(cls, resolved_dists): ) for index, (dist, ns_packages) in enumerate(namespace_packages_by_dist.items()) ) + if not pkg_resources: + current_interpreter = PythonInterpreter.get() + pex_warnings.warn( + "The legacy `pkg_resources` package cannot be imported by the " + "{interpreter} {version} interpreter at {path}.\n" + "The following distributions need `pkg_resources` to load some legacy " + "namespace packages and may fail to work properly:\n{dists}".format( + interpreter=current_interpreter.identity.interpreter, + version=current_interpreter.python, + path=current_interpreter.binary, + dists=dists, + ) + ) + return + pex_warnings.warn( "The `pkg_resources` package was loaded from a pex vendored version when " "declaring namespace packages defined by:\n{dists}\n\nThese distributions " diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py index 226968546..410aba441 100644 --- a/pex/interpreter_constraints.py +++ b/pex/interpreter_constraints.py @@ -316,11 +316,12 @@ def iter_compatible_versions( # N.B.: Pex does not support the missing 3.x versions here. PythonVersion(Lifecycle.EOL, 3, 5, 10), PythonVersion(Lifecycle.EOL, 3, 6, 15), - PythonVersion(Lifecycle.STABLE, 3, 7, 15), - PythonVersion(Lifecycle.STABLE, 3, 8, 15), - PythonVersion(Lifecycle.STABLE, 3, 9, 15), - PythonVersion(Lifecycle.STABLE, 3, 10, 8), - PythonVersion(Lifecycle.STABLE, 3, 11, 0), + # ^-- EOL --^ + PythonVersion(Lifecycle.STABLE, 3, 7, 17), + PythonVersion(Lifecycle.STABLE, 3, 8, 17), + PythonVersion(Lifecycle.STABLE, 3, 9, 17), + PythonVersion(Lifecycle.STABLE, 3, 10, 12), + PythonVersion(Lifecycle.STABLE, 3, 11, 4), PythonVersion(Lifecycle.DEV, 3, 12, 0), ) diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 54b089aa9..8805b1a05 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -752,7 +752,7 @@ def zip_cache_dir(path): os.mkdir(internal_cache) for location, fingerprint in pex_info.distributions.items(): cached_installed_wheel_zip_dir = zip_cache_dir( - os.path.join(pex_info.pex_root, "installed_wheel_zips", fingerprint) + os.path.join(pex_info.pex_root, "packed_wheels", fingerprint) ) with atomic_directory(cached_installed_wheel_zip_dir) as atomic_zip_dir: if not atomic_zip_dir.is_finalized(): diff --git a/pex/pip/installation.py b/pex/pip/installation.py index 09c82db4c..c935a88a8 100644 --- a/pex/pip/installation.py +++ b/pex/pip/installation.py @@ -4,25 +4,30 @@ from __future__ import absolute_import import os +import sys from textwrap import dedent from pex import pex_warnings, third_party from pex.atomic_directory import atomic_directory +from pex.common import safe_mkdtemp +from pex.dist_metadata import Requirement from pex.interpreter import PythonInterpreter from pex.orderedset import OrderedSet from pex.pex import PEX from pex.pex_bootstrapper import ensure_venv -from pex.pip.tool import Pip +from pex.pip.tool import Pip, PipVenv from pex.pip.version import PipVersion, PipVersionValue from pex.resolve.resolvers import Resolver +from pex.result import Error, try_ from pex.targets import LocalInterpreter, RequiresPythonError, Targets from pex.third_party import isolated from pex.typing import TYPE_CHECKING from pex.util import named_temporary_file from pex.variables import ENV +from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Callable, Dict, Iterator, Optional + from typing import Callable, Dict, Iterator, Optional, Union import attr # vendor:skip else: @@ -69,7 +74,8 @@ def _pip_installation( isolated_pip_builder.freeze() pip_cache = os.path.join(pip_root, "pip_cache") pip_pex = ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter)) - return Pip(pip_pex=pip_pex, pip_cache=pip_cache) + pip_venv = PipVenv(venv_dir=pip_pex.venv_dir, execute_args=tuple(pip_pex.execute_args())) + return Pip(pip=pip_venv, pip_cache=pip_cache) def _vendored_installation(interpreter=None): @@ -82,19 +88,61 @@ def _vendored_installation(interpreter=None): ) +def _bootstrap_pip( + version, # type: PipVersionValue + interpreter=None, # type: Optional[PythonInterpreter] +): + # type: (...) -> Callable[[], Iterator[str]] + + def bootstrap_pip(): + # type: () -> Iterator[str] + + chroot = safe_mkdtemp() + venv = Virtualenv.create(venv_dir=os.path.join(chroot, "pip"), interpreter=interpreter) + venv.install_pip(upgrade=True) + + for req in version.requirements: + project_name = Requirement.parse(req).name + target_dir = os.path.join(chroot, "reqs", project_name) + venv.interpreter.execute(["-m", "pip", "install", "--target", target_dir, req]) + yield target_dir + + return bootstrap_pip + + def _resolved_installation( version, # type: PipVersionValue - resolver, # type: Resolver + resolver=None, # type: Optional[Resolver] interpreter=None, # type: Optional[PythonInterpreter] ): # type: (...) -> Pip - if version is PipVersion.VENDORED: - return _vendored_installation(interpreter=interpreter) + targets = Targets.from_target(LocalInterpreter.create(interpreter)) + + bootstrap_pip_version = try_( + compatible_version( + targets, + PipVersion.VENDORED, + context="Bootstrapping Pip {version}".format(version=version), + ) + ) + if bootstrap_pip_version is not PipVersion.VENDORED: + return _pip_installation( + version=version, + iter_distribution_locations=_bootstrap_pip(version, interpreter=interpreter), + interpreter=interpreter, + ) + + if resolver is None: + raise ValueError( + "A resolver is required to install {requirement}".format( + requirement=version.requirement + ) + ) def resolve_distribution_locations(): for installed_distribution in resolver.resolve_requirements( requirements=version.requirements, - targets=Targets(interpreters=(interpreter or PythonInterpreter.get(),)), + targets=targets, pip_version=PipVersion.VENDORED, ).installed_distributions: yield installed_distribution.distribution.location @@ -170,7 +218,7 @@ def compatible_version( requested_version, # type: PipVersionValue context, # type: str ): - # type: (...) -> PipVersionValue + # type: (...) -> Union[PipVersionValue, Error] try: validate_targets(targets, requested_version, context) return requested_version @@ -186,7 +234,13 @@ def compatible_version( return version except RequiresPythonError: continue - return PipVersion.v20_3_4_patched + return Error( + "No supported version of Pip is compatible with the given targets:\n{targets}".format( + targets="\n".join( + sorted(target.render_description() for target in targets.unique_targets()) + ) + ) + ) _PIP = {} # type: Dict[PipInstallation, Pip] @@ -194,29 +248,40 @@ def compatible_version( def get_pip( interpreter=None, - version=PipVersion.VENDORED, # type: PipVersionValue + version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] ): # type: (...) -> Pip """Returns a lazily instantiated global Pip object that is safe for un-coordinated use.""" + if version: + calculated_version = version + elif PipVersion.DEFAULT is PipVersion.VENDORED: + calculated_version = PipVersion.VENDORED + else: + # If no explicit Pip version was requested, and we're using Python 3.12+, the new semantic + # is to allow selecting the appropriate Pip for the interpreter at hand without warning. + # This is required since Python 3.12+ do not work with the vendored Pip. + target = LocalInterpreter.create(interpreter) + calculated_version = try_( + compatible_version( + targets=Targets.from_target(target), + requested_version=PipVersion.DEFAULT, + context="Selecting Pip for {target}".format(target=target.render_description()), + ) + ) + installation = PipInstallation( interpreter=interpreter or PythonInterpreter.get(), - version=version, + version=calculated_version, ) pip = _PIP.get(installation) if pip is None: installation.check_python_applies() - if version is PipVersion.VENDORED: + if installation.version is PipVersion.VENDORED: pip = _vendored_installation(interpreter=interpreter) else: - if resolver is None: - raise ValueError( - "A resolver is required to install {requirement}".format( - requirement=version.requirement - ) - ) pip = _resolved_installation( - version=version, resolver=resolver, interpreter=interpreter + version=installation.version, resolver=resolver, interpreter=interpreter ) _PIP[installation] = pip return pip diff --git a/pex/pip/local_project.py b/pex/pip/local_project.py index f2d70ebae..dedeca8ce 100644 --- a/pex/pip/local_project.py +++ b/pex/pip/local_project.py @@ -25,10 +25,10 @@ def digest_local_project( directory, # type: str digest, # type: HintedDigest - pip_version, # type: PipVersionValue target, # type: Target resolver, # type: Resolver dest_dir=None, # type: Optional[str] + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> Union[str, Error] with TRACER.timed("Fingerprinting local project at {directory}".format(directory=directory)): diff --git a/pex/pip/tool.py b/pex/pip/tool.py index d694a7e8c..08b5613bc 100644 --- a/pex/pip/tool.py +++ b/pex/pip/tool.py @@ -20,7 +20,6 @@ from pex.network_configuration import NetworkConfiguration from pex.pep_376 import Record from pex.pep_425 import CompatibilityTags -from pex.pex_bootstrapper import VenvPex from pex.pip import foreign_platform from pex.pip.download_observer import DownloadObserver, PatchSet from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage, LogAnalyzer, LogScrapeJob @@ -134,7 +133,7 @@ def _calculate_env( @classmethod def create( cls, - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] resolver_version=None, # type: Optional[ResolverVersion.Value] indexes=None, # type: Optional[Sequence[str]] find_links=None, # type: Optional[Iterable[str]] @@ -164,22 +163,22 @@ def create( def __init__( self, - pip_version, # type: PipVersionValue resolver_version, # type: ResolverVersion.Value network_configuration, # type: NetworkConfiguration args, # type: Iterable[str] env, # type: Iterable[Tuple[str, str]] isolated, # type: bool password_entries=(), # type: Iterable[PasswordEntry] + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> None - self.pip_version = pip_version # type: PipVersionValue self.resolver_version = resolver_version # type: ResolverVersion.Value self.network_configuration = network_configuration # type: NetworkConfiguration self.args = tuple(args) # type: Iterable[str] self.env = dict(env) # type: Mapping[str, str] self.isolated = isolated # type: bool self.password_entries = password_entries # type: Iterable[PasswordEntry] + self.pip_version = pip_version # type: Optional[PipVersionValue] if TYPE_CHECKING: @@ -218,12 +217,22 @@ def analyze(self, line): return self.Continue() +@attr.s(frozen=True) +class PipVenv(object): + venv_dir = attr.ib() # type: str + _execute_args = attr.ib() # type: Tuple[str, ...] + + def execute_args(self, *args): + # type: (*str) -> List[str] + return list(self._execute_args + args) + + @attr.s(frozen=True) class Pip(object): _PATCHES_PACKAGE_ENV_VAR_NAME = "_PEX_PIP_RUNTIME_PATCHES_PACKAGE" _PATCHES_PACKAGE_NAME = "_pex_pip_patches" - _pip_pex = attr.ib() # type: VenvPex + _pip = attr.ib() # type: PipVenv _pip_cache = attr.ib() # type: str @staticmethod @@ -350,7 +359,7 @@ def _spawn_pip_isolated( popen_kwargs["stdout"] = sys.stderr.fileno() popen_kwargs.update(stderr=subprocess.PIPE) - args = self._pip_pex.execute_args(*command) + args = self._pip.execute_args(*command) rendered_env = " ".join( "{}={}".format(key, shlex_quote(value)) for key, value in env.items() diff --git a/pex/pip/version.py b/pex/pip/version.py index 9b308c60b..1663bf337 100644 --- a/pex/pip/version.py +++ b/pex/pip/version.py @@ -3,18 +3,36 @@ from __future__ import absolute_import +import os + +from pex import targets from pex.dist_metadata import Requirement from pex.enum import Enum from pex.pep_440 import Version from pex.targets import LocalInterpreter, Target from pex.third_party.packaging.specifiers import SpecifierSet -from pex.typing import TYPE_CHECKING +from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: from typing import Iterable, Optional, Tuple class PipVersionValue(Enum.Value): + @classmethod + def overridden(cls): + # type: () -> Optional[PipVersionValue] + if not hasattr(cls, "_overridden"): + setattr(cls, "_overridden", None) + + # We make an affordance for CI with a purposefully undocumented PEX env var. + overriden_value = os.environ.get("_PEX_PIP_VERSION") + if overriden_value: + for version in cls._iter_values(): + if version.value == overriden_value: + setattr(cls, "_overridden", version) + break + return cast("Optional[PipVersionValue]", getattr(cls, "_overridden")) + def __init__( self, version, # type: str @@ -23,6 +41,7 @@ def __init__( setuptools_version=None, # type: Optional[str] wheel_version=None, # type: Optional[str] requires_python=None, # type: Optional[str] + hidden=False, # type: bool ): # type: (...) -> None super(PipVersionValue, self).__init__(name or version) @@ -45,6 +64,7 @@ def to_requirement( self.setuptools_requirement = to_requirement("setuptools", setuptools_version) self.wheel_requirement = to_requirement("wheel", wheel_version) self.requires_python = SpecifierSet(requires_python) if requires_python else None + self.hidden = hidden @property def requirements(self): @@ -65,16 +85,51 @@ def requires_python_applies(self, target): class LatestPipVersion(object): def __get__(self, obj, objtype=None): if not hasattr(self, "_latest"): - self._latest = max(PipVersionValue._iter_values(), key=lambda pv: pv.version) + self._latest = max( + (version for version in PipVersionValue._iter_values() if not version.hidden), + key=lambda pv: pv.version, + ) return self._latest +class DefaultPipVersion(object): + def __init__(self, preferred): + # type: (Iterable[PipVersionValue]) -> None + self._preferred = preferred + + def __get__(self, obj, objtype=None): + if not hasattr(self, "_default"): + self._default = None + current_target = targets.current() + preferred_versions = ( + [PipVersionValue.overridden()] if PipVersionValue.overridden() else self._preferred + ) + for preferred_version in preferred_versions: + if preferred_version.requires_python_applies(current_target): + self._default = preferred_version + break + if self._default is None: + self._default = max( + ( + version + for version in PipVersionValue._iter_values() + if not version.hidden and version.requires_python_applies(current_target) + ), + key=lambda pv: pv.version, + ) + return self._default + + class PipVersion(Enum["PipVersionValue"]): @classmethod def values(cls): # type: () -> Tuple[PipVersionValue, ...] if cls._values is None: - cls._values = tuple(PipVersionValue._iter_values()) + cls._values = tuple( + version + for version in PipVersionValue._iter_values() + if version is PipVersionValue.overridden() or not version.hidden + ) return cls._values v20_3_4_patched = PipVersionValue( @@ -83,6 +138,7 @@ def values(cls): requirement=( "pip @ git+https://github.com/pantsbuild/pip@386a54f097ece66775d0c7f34fd29bb596c6b0be" ), + requires_python="<3.12", ) # TODO(John Sirois): Expose setuptools and wheel version flags - these don't affect @@ -93,57 +149,67 @@ def values(cls): version="22.2.2", setuptools_version="65.3.0", wheel_version="0.37.1", - requires_python=">=3.7", + requires_python=">=3.7,<3.12", ) v22_3 = PipVersionValue( version="22.3", setuptools_version="65.5.0", wheel_version="0.37.1", - requires_python=">=3.7", + requires_python=">=3.7,<3.12", ) v22_3_1 = PipVersionValue( version="22.3.1", setuptools_version="65.5.1", wheel_version="0.37.1", - requires_python=">=3.7", + requires_python=">=3.7,<3.12", ) v23_0 = PipVersionValue( version="23.0", setuptools_version="67.2.0", wheel_version="0.38.4", - requires_python=">=3.7", + requires_python=">=3.7,<3.12", ) v23_0_1 = PipVersionValue( version="23.0.1", setuptools_version="67.4.0", wheel_version="0.38.4", - requires_python=">=3.7", + requires_python=">=3.7,<3.12", ) v23_1 = PipVersionValue( version="23.1", setuptools_version="67.6.1", wheel_version="0.40.0", - requires_python=">=3.7", + requires_python=">=3.7,<3.12", ) v23_1_1 = PipVersionValue( version="23.1.1", setuptools_version="67.7.1", wheel_version="0.40.0", - requires_python=">=3.7", + requires_python=">=3.7,<3.12", ) v23_1_2 = PipVersionValue( version="23.1.2", setuptools_version="67.7.2", wheel_version="0.40.0", + requires_python=">=3.7,<3.12", + ) + + v23_2 = PipVersionValue( + version="23.2.dev0+8a1eea4a", + requirement="pip @ git+https://github.com/pypa/pip@8a1eea4aaedb1fb1c6b4c652cd0c43502f05ff37", + setuptools_version="67.8.0", + wheel_version="0.40.0", requires_python=">=3.7", + hidden=True, ) VENDORED = v20_3_4_patched LATEST = LatestPipVersion() + DEFAULT = DefaultPipVersion(preferred=(VENDORED, v23_2)) diff --git a/pex/platforms.py b/pex/platforms.py index f4ce3c311..355836083 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -234,10 +234,15 @@ def parse_tags(output): if count != 0: raise AssertionError("Finished with count {}.".format(count)) + from pex.resolve.configured_resolver import ConfiguredResolver + from pex.resolve.resolver_configuration import PipConfiguration + job = SpawnedJob.stdout( - # TODO(John Sirois): Plumb pip_version and resolver: + # TODO(John Sirois): Plumb pip_version and the user-configured resolver: # https://github.com/pantsbuild/pex/issues/1894 - job=get_pip().spawn_debug(platform=self, manylinux=manylinux), + job=get_pip( + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()) + ).spawn_debug(platform=self, manylinux=manylinux), result_func=parse_tags, ) return job.await_result() diff --git a/pex/resolve/config.py b/pex/resolve/config.py index 64afb1e52..c84f7d9bb 100644 --- a/pex/resolve/config.py +++ b/pex/resolve/config.py @@ -34,8 +34,10 @@ def _finalize_pip_configuration( ): # type: (...) -> Union[PipConfiguration, Error] version = pip_version or pip_configuration.version - if pip_configuration.allow_version_fallback: - return attr.evolve(pip_configuration, version=compatible_version(targets, version, context)) + if version and pip_configuration.allow_version_fallback: + return attr.evolve( + pip_configuration, version=try_(compatible_version(targets, version, context)) + ) result = catch(validate_targets, targets, version, context) if isinstance(result, Error): diff --git a/pex/resolve/configured_resolver.py b/pex/resolve/configured_resolver.py index e83e8d4c4..661ee0343 100644 --- a/pex/resolve/configured_resolver.py +++ b/pex/resolve/configured_resolver.py @@ -43,7 +43,6 @@ def resolve_lock( pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> Installed - # TODO(John Sirois): Use pip_version. return try_( lock_resolver.resolve_from_lock( targets=targets, diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index ea3022160..fac944ce9 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -83,7 +83,7 @@ def __init__( use_pep517=None, # type: Optional[bool] build_isolation=True, # type: bool pex_root=None, # type: Optional[str] - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] ): super(VCSArtifactDownloadManager, self).__init__( @@ -160,8 +160,8 @@ def __init__( self, target, # type: Target file_lock_style, # type: FileLockStyle.Value - pip_version, # type: PipVersionValue resolver, # type: Resolver + pip_version=None, # type: Optional[PipVersionValue] pex_root=None, # type: Optional[str] ): super(LocalProjectDownloadManager, self).__init__( @@ -242,7 +242,7 @@ def resolve_from_lock( transitive=True, # type: bool verify_wheels=True, # type: bool max_parallel_jobs=None, # type: Optional[int] - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> Union[Installed, Error] diff --git a/pex/resolve/locker.py b/pex/resolve/locker.py index f47b6391a..9ed27b028 100644 --- a/pex/resolve/locker.py +++ b/pex/resolve/locker.py @@ -250,11 +250,11 @@ def __init__( self, target, # type: Target root_requirements, # type: Iterable[ParsedRequirement] - pip_version, # type: PipVersionValue resolver, # type: Resolver lock_configuration, # type: LockConfiguration download_dir, # type: str fingerprint_service=None, # type: Optional[FingerprintService] + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> None diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index c2e5fef06..cd3248409 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -20,6 +20,7 @@ from pex.pep_503 import ProjectName from pex.pip.download_observer import DownloadObserver from pex.pip.tool import PackageIndexConfiguration +from pex.pip.version import PipVersion from pex.resolve import lock_resolver, locker, resolvers from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.downloads import ArtifactDownloader diff --git a/pex/resolve/lockfile/json_codec.py b/pex/resolve/lockfile/json_codec.py index 465bdedaa..27e46248b 100644 --- a/pex/resolve/lockfile/json_codec.py +++ b/pex/resolve/lockfile/json_codec.py @@ -302,7 +302,9 @@ def assemble_tag( requires_python=get("requires_python", list), target_systems=target_systems, pip_version=get_enum_value( - PipVersion, "pip_version", default=_DEFAULT_PIP_CONFIGURATION.version + PipVersion, + "pip_version", + default=_DEFAULT_PIP_CONFIGURATION.version or PipVersion.DEFAULT, ), resolver_version=get_enum_value(ResolverVersion, "resolver_version"), requirements=requirements, diff --git a/pex/resolve/lockfile/model.py b/pex/resolve/lockfile/model.py index b78b886c3..2afd17504 100644 --- a/pex/resolve/lockfile/model.py +++ b/pex/resolve/lockfile/model.py @@ -7,7 +7,7 @@ from pex.dist_metadata import Requirement from pex.orderedset import OrderedSet -from pex.pip.version import PipVersionValue +from pex.pip.version import PipVersion, PipVersionValue from pex.requirements import LocalProjectRequirement from pex.resolve.locked_resolve import LocalProjectArtifact, LockedResolve, LockStyle, TargetSystem from pex.resolve.resolved_requirement import Pin @@ -34,7 +34,6 @@ def create( style, # type: LockStyle.Value requires_python, # type: Iterable[str] target_systems, # type: Iterable[TargetSystem.Value] - pip_version, # type: PipVersionValue resolver_version, # type: ResolverVersion.Value requirements, # type: Iterable[Union[Requirement, ParsedRequirement]] constraints, # type: Iterable[Requirement] @@ -47,6 +46,7 @@ def create( transitive, # type: bool locked_resolves, # type: Iterable[LockedResolve] source=None, # type: Optional[str] + pip_version=None, # type: Optional[PipVersionValue] ): # type: (...) -> Lockfile @@ -93,7 +93,7 @@ def extract_requirement(req): style=style, requires_python=SortedTuple(requires_python), target_systems=SortedTuple(target_systems), - pip_version=pip_version, + pip_version=pip_version or PipVersion.DEFAULT, resolver_version=resolver_version, requirements=SortedTuple(resolve_requirements, key=str), constraints=SortedTuple(constraints, key=str), diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 40733993c..c19bbc28d 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -61,12 +61,6 @@ def create( password_entries = attr.ib(default=()) # type: Tuple[PasswordEntry, ...] -# We make an affordance for CI with a purposefully undocumented PEX env var. -_DEFAULT_PIP_VERSION = PipVersion.for_value( - os.environ.get("_PEX_PIP_VERSION", PipVersion.VENDORED.value) -) - - @attr.s(frozen=True) class PipConfiguration(object): resolver_version = attr.ib(default=ResolverVersion.PIP_LEGACY) # type: ResolverVersion.Value @@ -81,7 +75,7 @@ class PipConfiguration(object): transitive = attr.ib(default=True) # type: bool max_jobs = attr.ib(default=DEFAULT_MAX_JOBS) # type: int preserve_log = attr.ib(default=False) # type: bool - version = attr.ib(default=_DEFAULT_PIP_VERSION) # type: PipVersionValue + version = attr.ib(default=None) # type: Optional[PipVersionValue] allow_version_fallback = attr.ib(default=True) # type: bool diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index d1b83f29e..2baecf148 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -9,7 +9,7 @@ from pex.argparse import HandleBoolAction from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet -from pex.pip.version import PipVersion +from pex.pip.version import PipVersion, PipVersionValue from pex.resolve.lockfile import json_codec from pex.resolve.lockfile.model import Lockfile from pex.resolve.path_mappings import PathMapping, PathMappings @@ -83,7 +83,7 @@ def register( parser.add_argument( "--pip-version", dest="pip_version", - default=str(default_resolver_configuration.version), + default=str(PipVersion.DEFAULT), choices=["latest", "vendored"] + [str(value) for value in PipVersion.values()], help=( "The version of Pip to use for resolving dependencies. The `latest` version refers to " @@ -437,11 +437,12 @@ def create_pip_configuration(options): repos_configuration = create_repos_configuration(options) + pip_version = None # type: Optional[PipVersionValue] if options.pip_version == "latest": pip_version = PipVersion.LATEST elif options.pip_version == "vendored": pip_version = PipVersion.VENDORED - else: + elif options.pip_version: pip_version = PipVersion.for_value(options.pip_version) return PipConfiguration( diff --git a/pex/resolver.py b/pex/resolver.py index 457c2c478..730bed88c 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -89,7 +89,7 @@ class DownloadRequest(object): build_isolation = attr.ib(default=True) # type: bool observer = attr.ib(default=None) # type: Optional[ResolveObserver] preserve_log = attr.ib(default=False) # type: bool - pip_version = attr.ib(default=PipVersion.VENDORED) # type: PipVersionValue + pip_version = attr.ib(default=None) # type: Optional[PipVersionValue] resolver = attr.ib(default=None) # type: Optional[Resolver] def iter_local_projects(self): @@ -517,7 +517,7 @@ def __init__( use_pep517=None, # type: Optional[bool] build_isolation=True, # type: bool verify_wheels=True, # type: bool - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] ): # type: (...) -> None @@ -622,7 +622,7 @@ def __init__( use_pep517=None, # type: Optional[bool] build_isolation=True, # type: bool verify_wheels=True, # type: bool - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] ): # type: (...) -> None @@ -955,7 +955,7 @@ def resolve( ignore_errors=False, # type: bool verify_wheels=True, # type: bool preserve_log=False, # type: bool - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] ): # type: (...) -> Installed @@ -1115,7 +1115,7 @@ def _download_internal( max_parallel_jobs=None, # type: Optional[int] observer=None, # type: Optional[ResolveObserver] preserve_log=False, # type: bool - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] ): # type: (...) -> Tuple[List[BuildRequest], List[DownloadResult]] @@ -1200,7 +1200,7 @@ def download( max_parallel_jobs=None, # type: Optional[int] observer=None, # type: Optional[ResolveObserver] preserve_log=False, # type: bool - pip_version=PipVersion.VENDORED, # type: PipVersionValue + pip_version=None, # type: Optional[PipVersionValue] resolver=None, # type: Optional[Resolver] ): # type: (...) -> Downloaded diff --git a/pex/testing.py b/pex/testing.py index aa54f879e..99f9c0118 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -24,6 +24,8 @@ from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.pip.installation import get_pip +from pex.resolve.configured_resolver import ConfiguredResolver +from pex.resolve.resolver_configuration import PipConfiguration from pex.targets import LocalInterpreter from pex.typing import TYPE_CHECKING from pex.util import named_temporary_file @@ -125,6 +127,7 @@ def make_project( entry_points=None, # type: Optional[Union[str, Dict[str, List[str]]]] python_requires=None, # type: Optional[str] universal=False, # type: bool + prepare_project=None, # type: Optional[Callable[[str], None]] ): # type: (...) -> Iterator[str] project_content = { @@ -170,6 +173,8 @@ def make_project( } with temporary_content(project_content, interp=interp) as td: + if prepare_project: + prepare_project(td) yield td @@ -195,7 +200,10 @@ def __init__( def bdist(self): # type: () -> str - get_pip(interpreter=self._interpreter).spawn_build_wheels( + get_pip( + interpreter=self._interpreter, + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + ).spawn_build_wheels( distributions=[self._source_dir], wheel_dir=self._wheel_dir, interpreter=self._interpreter, @@ -220,6 +228,7 @@ def built_wheel( interpreter=None, # type: Optional[PythonInterpreter] python_requires=None, # type: Optional[str] universal=False, # type: bool + prepare_project=None, # type: Optional[Callable[[str], None]] **kwargs # type: Any ): # type: (...) -> Iterator[str] @@ -232,6 +241,7 @@ def built_wheel( entry_points=entry_points, python_requires=python_requires, universal=universal, + prepare_project=prepare_project, ) as td: builder = WheelBuilder(td, interpreter=interpreter, **kwargs) yield builder.bdist() @@ -272,7 +282,9 @@ def install_wheel( ): # type: (...) -> Distribution install_dir = os.path.join(safe_mkdtemp(), os.path.basename(wheel)) - get_pip(interpreter=interpreter).spawn_install_wheel( + get_pip( + interpreter=interpreter, resolver=ConfiguredResolver(pip_configuration=PipConfiguration()) + ).spawn_install_wheel( wheel=wheel, install_dir=install_dir, target=LocalInterpreter.create(interpreter), diff --git a/pex/tools/commands/repository.py b/pex/tools/commands/repository.py index f6a83e5ec..aa7a59102 100644 --- a/pex/tools/commands/repository.py +++ b/pex/tools/commands/repository.py @@ -15,6 +15,7 @@ from threading import Thread from pex import dist_metadata +from pex.atomic_directory import atomic_directory from pex.commands.command import JsonMixin, OutputMixin from pex.common import ( DETERMINISTIC_DATETIME_TIMESTAMP, @@ -26,16 +27,17 @@ from pex.compatibility import Queue from pex.dist_metadata import Distribution from pex.environment import PEXEnvironment -from pex.interpreter import PythonInterpreter, spawn_python_job -from pex.interpreter_constraints import InterpreterConstraint -from pex.jobs import Retain, SpawnedJob, execute_parallel +from pex.interpreter import PythonInterpreter +from pex.jobs import Job, Retain, SpawnedJob, execute_parallel from pex.pex import PEX from pex.result import Error, Ok, Result from pex.tools.command import PEXCommand from pex.typing import TYPE_CHECKING, cast +from pex.variables import ENV +from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import IO, Callable, Iterable, Iterator, List, Text, Tuple + from typing import IO, Any, Callable, Iterable, Iterator, List, Text, Tuple import attr # vendor:skip @@ -47,6 +49,25 @@ logger = logging.getLogger(__name__) +def spawn_python_job_with_setuptools_and_wheel( + interpreter, # type: PythonInterpreter + args, # type: Iterable[str] + **subprocess_kwargs # type: Any +): + # type: (...) -> Job + venv_dir = os.path.join(ENV.PEX_ROOT, "tools", "repository", str(interpreter.platform)) + with atomic_directory(venv_dir) as atomic_dir: + if not atomic_dir.is_finalized(): + venv = Virtualenv.create_atomic(venv_dir=atomic_dir, interpreter=interpreter) + venv.install_pip(upgrade=True) + venv.interpreter.execute(args=["-m", "pip", "install", "-U", "setuptools", "wheel"]) + + execute_python_args = [Virtualenv(venv_dir=venv_dir).interpreter.binary] + execute_python_args.extend(args) + process = subprocess.Popen(args=execute_python_args, **subprocess_kwargs) + return Job(command=execute_python_args, process=process) + + @attr.s(frozen=True) class FindLinksRepo(object): @classmethod @@ -261,10 +282,9 @@ def spawn_extract(distribution): # + https://reproducible-builds.org/docs/source-date-epoch/ # + https://github.com/pypa/wheel/blob/1b879e53fed1f179897ed47e55a68bc51df188db/wheel/archive.py#L36-L39 env.update(SOURCE_DATE_EPOCH=str(int(DETERMINISTIC_DATETIME_TIMESTAMP))) - job = spawn_python_job( + job = spawn_python_job_with_setuptools_and_wheel( args=["-m", "wheel", "pack", "--dest-dir", dest_dir, distribution.location], interpreter=pex.interpreter, - expose=["wheel"], stdout=subprocess.PIPE, env=env, ) @@ -422,9 +442,8 @@ def _extract_sdist( with open(os.path.join(chroot, "setup.py"), "w") as fp: fp.write("import setuptools; setuptools.setup()") - spawn_python_job( + spawn_python_job_with_setuptools_and_wheel( args=["setup.py", "sdist", "--dist-dir", dest_dir], interpreter=pex.interpreter, - expose=["setuptools"], cwd=chroot, ).wait() diff --git a/pyproject.toml b/pyproject.toml index b256729f0..dc8530318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Build Tools", @@ -33,7 +34,7 @@ classifiers = [ "Topic :: System :: Software Distribution", "Topic :: Utilities", ] -requires-python = ">=2.7,<3.12,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +requires-python = ">=2.7,<3.13,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" [tool.flit.metadata.requires-extra] # For improved subprocess robustness under python2.7. diff --git a/scripts/package.py b/scripts/package.py index f424029b3..96bf0968a 100755 --- a/scripts/package.py +++ b/scripts/package.py @@ -25,7 +25,7 @@ def python_requires() -> str: return cast(str, project_metadata["tool"]["flit"]["metadata"]["requires-python"].strip()) -def build_pex_pex(output_file: PurePath, local: bool = False, verbosity: int = 0) -> None: +def build_pex_pex(output_file: PurePath, verbosity: int = 0) -> None: # NB: We do not include the subprocess extra (which would be spelled: `.[subprocess]`) since we # would then produce a pex that would not be consumable by all python interpreters otherwise # meeting `python_requires`; ie: we'd need to then come up with a deploy environment / deploy @@ -54,8 +54,6 @@ def build_pex_pex(output_file: PurePath, local: bool = False, verbosity: int = 0 "pex", pex_requirement, ] - if not local: - args.extend(["--interpreter-constraint", python_requires()]) subprocess.run(args, check=True) @@ -100,12 +98,11 @@ def main( *additional_dist_formats: Format, verbosity: int = 0, pex_output_file: Optional[Path] = DIST_DIR / "pex", - local: bool = False, serve: bool = False ) -> None: if pex_output_file: print(f"Building Pex PEX to `{pex_output_file}` ...") - build_pex_pex(pex_output_file, local, verbosity) + build_pex_pex(pex_output_file, verbosity) git_rev = describe_git_rev() sha256, size = describe_file(pex_output_file) @@ -169,12 +166,6 @@ def main( type=Path, help="Build the Pex PEX at this path.", ) - parser.add_argument( - "--local", - default=False, - action="store_true", - help="Build Pex PEX with just a single local interpreter.", - ) parser.add_argument( "--serve", default=False, @@ -187,6 +178,5 @@ def main( *(args.additional_formats or ()), verbosity=args.verbosity, pex_output_file=None if args.no_pex else args.pex_output_file, - local=args.local, serve=args.serve ) diff --git a/scripts/typecheck.py b/scripts/typecheck.py index 71b90192b..69ca2109c 100755 --- a/scripts/typecheck.py +++ b/scripts/typecheck.py @@ -43,7 +43,7 @@ def main() -> None: source_and_tests = sorted( find_files_to_check(include=["pex", "tests"], exclude=["pex/vendor/_vendored"]) ) - for python_version in ("3.11", "3.10", "3.5", "2.7"): + for python_version in ("3.12", "3.11", "3.5", "2.7"): run_mypy(python_version, files=source_and_tests) diff --git a/tests/bin/test_sh_boot.py b/tests/bin/test_sh_boot.py index deae812dc..b86ca6874 100644 --- a/tests/bin/test_sh_boot.py +++ b/tests/bin/test_sh_boot.py @@ -45,6 +45,7 @@ def expected(*names): all_names.add(current_interpreter_identity.binary_name(version_components=2)) all_names.update( [ + "python3.12", "python3.11", "python3.10", "python3.9", @@ -53,6 +54,7 @@ def expected(*names): "python3.6", "python3.5", "python2.7", + "pypy3.12", "pypy3.11", "pypy3.10", "pypy3.9", diff --git a/tests/build_system/test_issue_2125.py b/tests/build_system/test_issue_2125.py index a9a8897ea..361fe81e9 100644 --- a/tests/build_system/test_issue_2125.py +++ b/tests/build_system/test_issue_2125.py @@ -81,7 +81,7 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): ) ) - pip_version = PipVersion.VENDORED + pip_version = PipVersion.DEFAULT dist_metadata = pep_517.spawn_prepare_metadata( project_directory=project_directory, pip_version=pip_version, diff --git a/tests/build_system/test_pep_517.py b/tests/build_system/test_pep_517.py index 5cc386702..55e32e8e4 100644 --- a/tests/build_system/test_pep_517.py +++ b/tests/build_system/test_pep_517.py @@ -26,7 +26,6 @@ def test_build_sdist_project_directory_dne(tmpdir): result = build_sdist( project_dir, dist_dir, - PipVersion.VENDORED, LocalInterpreter.create(), ConfiguredResolver.default(), ) @@ -45,7 +44,6 @@ def test_build_sdist_project_directory_is_file(tmpdir): result = build_sdist( project_dir, dist_dir, - PipVersion.VENDORED, LocalInterpreter.create(), ConfiguredResolver.default(), ) diff --git a/tests/build_system/test_pep_518.py b/tests/build_system/test_pep_518.py index 6842b4de8..bb44cafc2 100644 --- a/tests/build_system/test_pep_518.py +++ b/tests/build_system/test_pep_518.py @@ -26,7 +26,7 @@ def load_build_system(project_directory): # type: (...) -> Union[Optional[BuildSystem], Error] return pep_518.load_build_system( LocalInterpreter.create(), - ConfiguredResolver(PipConfiguration(version=PipVersion.VENDORED)), + ConfiguredResolver(PipConfiguration(version=PipVersion.DEFAULT)), project_directory, ) diff --git a/tests/integration/build_system/test_issue_2063.py b/tests/integration/build_system/test_issue_2063.py index 156875020..869559772 100644 --- a/tests/integration/build_system/test_issue_2063.py +++ b/tests/integration/build_system/test_issue_2063.py @@ -13,8 +13,11 @@ @pytest.mark.skipif( - sys.version_info[:2] < (3, 5), - reason="The tested distribution is only compatible with Python >= 3.5", + sys.version_info[:2] < (3, 5) or sys.version_info[:2] >= (3, 12), + reason=( + "The tested distribution is only compatible with Python >= 3.5 and it requires lxml (4.9.2)" + " which only has pre-built wheels available through 3.11." + ), ) def test_build_system_no_build_backend(tmpdir): # type: (Any) -> None diff --git a/tests/integration/build_system/test_pep_518.py b/tests/integration/build_system/test_pep_518.py index 1affed785..11fe7c279 100644 --- a/tests/integration/build_system/test_pep_518.py +++ b/tests/integration/build_system/test_pep_518.py @@ -26,7 +26,7 @@ def test_load_build_system_pyproject_custom_repos( pip_version = ( PipVersion.v22_2_2 if PipVersion.v22_2_2.requires_python_applies(current_target) - else PipVersion.VENDORED + else PipVersion.DEFAULT ) build_system = load_build_system( current_target, diff --git a/tests/integration/cli/commands/test_export.py b/tests/integration/cli/commands/test_export.py index c1730fb9b..894309f4f 100644 --- a/tests/integration/cli/commands/test_export.py +++ b/tests/integration/cli/commands/test_export.py @@ -34,7 +34,7 @@ style=LockStyle.UNIVERSAL, requires_python=SortedTuple(), target_systems=SortedTuple(), - pip_version=PipVersion.VENDORED, + pip_version=PipVersion.DEFAULT, resolver_version=ResolverVersion.PIP_2020, requirements=SortedTuple([Requirement.parse("ansicolors")]), constraints=SortedTuple(), diff --git a/tests/integration/cli/commands/test_issue_2050.py b/tests/integration/cli/commands/test_issue_2050.py index d0081d06f..685b5ebe1 100644 --- a/tests/integration/cli/commands/test_issue_2050.py +++ b/tests/integration/cli/commands/test_issue_2050.py @@ -71,7 +71,6 @@ def func(project_directory): pep_517.build_sdist( project_directory=project_directory, dist_dir=find_links, - pip_version=PipVersion.VENDORED, target=LocalInterpreter.create(), resolver=ConfiguredResolver.default(), ) @@ -140,7 +139,7 @@ def test_lock_uncompilable_sdist( lock, env=make_env( SETUP_KWARGS_JSON=json.dumps( - dict(install_requires=["ansicolors==1.1.8"], python_requires=">=3.5,<3.12") + dict(install_requires=["ansicolors==1.1.8"], python_requires=">=3.5") ) ), ).assert_success() @@ -154,7 +153,7 @@ def test_lock_uncompilable_sdist( } # type: Dict[ProjectName, LockedRequirement] bad = locked_requirements.pop(ProjectName("pex.tests.bad-c-extension")) assert Version("0.1.0+test") == bad.pin.version - assert SpecifierSet(">=3.5,<3.12") == bad.requires_python + assert SpecifierSet(">=3.5") == bad.requires_python assert SortedTuple([Requirement.parse("ansicolors==1.1.8")]) == bad.requires_dists assert locked_requirements.pop(ProjectName("ansicolors")) is not None assert not locked_requirements diff --git a/tests/integration/cli/commands/test_lock_update.py b/tests/integration/cli/commands/test_lock_update.py index 1e26802c0..3d3fe601f 100644 --- a/tests/integration/cli/commands/test_lock_update.py +++ b/tests/integration/cli/commands/test_lock_update.py @@ -11,6 +11,7 @@ from pex.cli.testing import run_pex3 from pex.common import safe_open from pex.fetcher import URLFetcher +from pex.pip.version import PipVersion from pex.resolve.locked_resolve import Artifact, FileArtifact, LockedRequirement from pex.resolve.lockfile import json_codec from pex.resolve.lockfile.model import Lockfile @@ -74,7 +75,7 @@ def find_links( # We need the current default --pip-version requirements for some tests that do PyPI offline # resolves. - pip_version = PipConfiguration().version + pip_version = PipVersion.DEFAULT repository_pex = os.path.join(str(tmpdir), "repository.pex") run_pex_command( args=[ @@ -213,7 +214,7 @@ def test_lock_update_repo_migration_corrupted( "Detected fingerprint changes in the following locked project for lock generated by " "universal!\n" "ansicolors 1.1.8\n" - ) == result.error + ) in result.error def test_lock_update_repo_migration_artifacts_removed( diff --git a/tests/integration/cli/commands/test_vcs_lock.py b/tests/integration/cli/commands/test_vcs_lock.py index 0c34a77ca..c0c0e2f12 100644 --- a/tests/integration/cli/commands/test_vcs_lock.py +++ b/tests/integration/cli/commands/test_vcs_lock.py @@ -235,7 +235,7 @@ def third_worst(): subprocess.check_call(args=["git", "config", "user.name", "Douglas Adams"], cwd=src) subprocess.check_call(args=["git", "checkout", "-b", "Golgafrincham"], cwd=src) subprocess.check_call(args=["git", "add", "."], cwd=src) - subprocess.check_call(args=["git", "commit", "-m", "Only commit."], cwd=src) + subprocess.check_call(args=["git", "commit", "--no-gpg-sign", "-m", "Only commit."], cwd=src) lock = test_tool.create_lock("git+file://{src}#egg=poetry".format(src=src)) pex = test_tool.create_pex(lock, "-c", "recite") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c6859d24b..c4d4ce027 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,6 +3,7 @@ from __future__ import absolute_import +import glob import os import subprocess from contextlib import contextmanager @@ -60,9 +61,9 @@ def pex_bdist( args=[pex_pex, "repository", "extract", "-f", extract_dir], env=make_env(PEX_TOOLS=True), ) - wheels = os.listdir(wheels_dir) + wheels = glob.glob(os.path.join(wheels_dir, "pex-*.whl")) assert 1 == len(wheels) - return os.path.join(wheels_dir, wheels[0]) + return wheels[0] @pytest.fixture diff --git a/tests/integration/test_inject_env_and_args.py b/tests/integration/test_inject_env_and_args.py index 28b13b839..e1db13e85 100644 --- a/tests/integration/test_inject_env_and_args.py +++ b/tests/integration/test_inject_env_and_args.py @@ -145,7 +145,12 @@ def assert_argv( ) -@pytest.mark.skipif(PY_VER < (3, 7), reason="Uvicorn only support Python 3.7+.") +@pytest.mark.skipif( + PY_VER < (3, 7) or PY_VER >= (3, 12), + reason=( + "Uvicorn only supports Python 3.7+ and pre-built wheels are only available through 3.11." + ), +) @parametrize_execution_mode_args def test_complex( tmpdir, # type: Any diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 332c49bb4..a8d7f8500 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -16,7 +16,7 @@ import pytest -from pex.common import safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch +from pex.common import is_exe, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch from pex.compatibility import WINDOWS, commonpath from pex.dist_metadata import Distribution, Requirement from pex.fetcher import URLFetcher @@ -819,7 +819,7 @@ def test_multiplatform_entrypoint(): interpreter = ensure_python_interpreter(PY38) res = run_pex_command( [ - "p537==1.0.5", + "p537==1.0.6", "--no-build", "--python={}".format(interpreter), "--python-shebang=#!{}".format(interpreter), @@ -840,6 +840,7 @@ def test_multiplatform_entrypoint(): def test_pex_console_script_custom_setuptools_useable(): # type: () -> None + setuptools_version = "67.7.2" if sys.version_info[:2] >= (3, 12) else "43.0.0" setup_py = dedent( """ from setuptools import setup @@ -849,19 +850,19 @@ def test_pex_console_script_custom_setuptools_useable(): version='0.0.0', zip_safe=True, packages=[''], - install_requires=['setuptools==43.0.0'], - entry_points={'console_scripts': ['my_app_function = my_app:do_something']}, + install_requires=['setuptools=={version}'], + entry_points={{'console_scripts': ['my_app_function = my_app:do_something']}}, ) - """ - ) + """ + ).format(version=setuptools_version) my_app = dedent( """ def do_something(): import setuptools - assert '43.0.0' == setuptools.__version__ - """ - ) + assert '{version}' == setuptools.__version__ + """ + ).format(version=setuptools_version) with temporary_content({"setup.py": setup_py, "my_app.py": my_app}) as project_dir: with temporary_dir() as out: @@ -884,17 +885,22 @@ def do_something(): @contextmanager def pex_with_no_entrypoints(): # type: () -> Iterator[Tuple[str, bytes, str]] + setuptools_version = "67.7.2" if sys.version_info[:2] >= (3, 12) else "43.0.0" with temporary_dir() as out: pex = os.path.join(out, "pex.pex") - run_pex_command(["setuptools==43.0.0", "-o", pex]) - test_script = dedent( - """\ - import sys - import setuptools + run_pex_command(["setuptools=={version}".format(version=setuptools_version), "-o", pex]) + test_script = ( + dedent( + """\ + import sys + import setuptools - sys.exit(0 if '43.0.0' == setuptools.__version__ else 1) - """ - ).encode("utf-8") + sys.exit(0 if '{version}' == setuptools.__version__ else 1) + """ + ) + .format(version=setuptools_version) + .encode("utf-8") + ) yield pex, test_script, out @@ -927,14 +933,35 @@ def test_setup_python(): subprocess.check_call([pex, "-c", "import jsonschema"]) -def test_setup_interpreter_constraint(): - # type: () -> None +@pytest.fixture +def path_with_git(tmpdir): + # type: (Any) -> Iterator[Callable[[str], str]] + + # N.B.: This ensures git is available for handling any git VCS requirements needed. + + git_path = None # type: Optional[str] + for entry in os.environ.get("PATH", "").split(os.pathsep): + if is_exe(os.path.join(entry, "git")): + git_path = entry + break + + def _path_with_git(path): + # type: (str) -> str + if git_path: + return os.pathsep.join((path, git_path)) + return path + + yield _path_with_git + + +def test_setup_interpreter_constraint(path_with_git): + # type: (Callable[[str], str]) -> None interpreter = ensure_python_interpreter(PY39) with temporary_dir() as out: pex = os.path.join(out, "pex.pex") env = make_env( PEX_IGNORE_RCFILES="1", - PATH=os.path.dirname(interpreter), + PATH=path_with_git(os.path.dirname(interpreter)), ) results = run_pex_command( [ @@ -952,8 +979,8 @@ def test_setup_interpreter_constraint(): assert rc == 0 -def test_setup_python_path(): - # type: () -> None +def test_setup_python_path(path_with_git): + # type: (Callable[[str], str]) -> None """Check that `--python-path` is used rather than the default $PATH.""" py38_interpreter_dir = os.path.dirname(ensure_python_interpreter(PY38)) py39_interpreter_dir = os.path.dirname(ensure_python_interpreter(PY39)) @@ -972,7 +999,7 @@ def test_setup_python_path(): "-o", pex, ], - env=make_env(PEX_IGNORE_RCFILES="1", PATH=""), + env=make_env(PEX_IGNORE_RCFILES="1", PATH=path_with_git("")), ) results.assert_success() @@ -1120,10 +1147,10 @@ def test_emit_warnings_default(): assert stderr -def test_no_emit_warnings(): +def test_no_emit_warnings_2(): # type: () -> None stderr = build_and_execute_pex_with_warnings("--no-emit-warnings") - assert not stderr + assert not stderr, stderr def test_no_emit_warnings_emit_env_override(): @@ -1234,7 +1261,7 @@ def test_pex_cache_dir_and_pex_root(): pex_file = os.path.join(td, "pex_file") run_pex_command( python=python, - args=["--cache-dir", cache_dir, "--pex-root", cache_dir, "p537==1.0.5", "-o", pex_file], + args=["--cache-dir", cache_dir, "--pex-root", cache_dir, "p537==1.0.6", "-o", pex_file], ).assert_success() dists = list(iter_distributions(pex_root=cache_dir, project_name="p537")) @@ -1246,7 +1273,7 @@ def test_pex_cache_dir_and_pex_root(): # When the options have conflicting values they should be rejected. run_pex_command( python=python, - args=["--cache-dir", cache_dir, "--pex-root", pex_root, "p537==1.0.5", "-o", pex_file], + args=["--cache-dir", cache_dir, "--pex-root", pex_root, "p537==1.0.6", "-o", pex_file], ).assert_failure() assert not os.path.exists(cache_dir) @@ -1261,7 +1288,7 @@ def test_disable_cache(): pex_file = os.path.join(td, "pex_file") run_pex_command( python=python, - args=["--disable-cache", "p537==1.0.5", "-o", pex_file], + args=["--disable-cache", "p537==1.0.6", "-o", pex_file], env=make_env(PEX_ROOT=pex_root), ).assert_success() @@ -1597,8 +1624,12 @@ def test_pip_issues_9420_workaround(): @pytest.mark.skipif( - PY_VER <= (3, 5), - reason="The example python requirements URL has requirements that only work with Python 3.6+.", + PY_VER <= (3, 5) or PY_VER >= (3, 12), + reason=( + "The example python requirements URL has requirements that only work with Python 3.6-3.11. " + "Python 3.12 in particular is cut off by an old setuptools version that assumes the stdlib " + "distutils packages which is gone as of Python 3.12." + ), ) def test_requirement_file_from_url(tmpdir): # type: (Any) -> None @@ -1699,6 +1730,14 @@ def test_invalid_macosx_platform_tag(tmpdir): subprocess.check_call(args=[setproctitle_pex, "-c", "import setproctitle"]) +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "The urllib3 dependency embeds six which uses a meta path importer that only implements " + "the PEP-302 finder spec and not the modern spec. Only the modern finder spec is supported " + "by Python 3.12+." + ), +) def test_require_hashes(tmpdir): # type: (Any) -> None requirements = os.path.join(str(tmpdir), "requirements.txt") diff --git a/tests/integration/test_interpreter_selection.py b/tests/integration/test_interpreter_selection.py index 164ad376a..c904efeaa 100644 --- a/tests/integration/test_interpreter_selection.py +++ b/tests/integration/test_interpreter_selection.py @@ -37,14 +37,17 @@ def test_interpreter_constraints_to_pex_info_py2(): [ "--disable-cache", "--interpreter-constraint=>=2.7,<3", - "--interpreter-constraint=>=3.5", + "--interpreter-constraint=>=3.5,<3.12", "-o", pex_out_path, ] ) res.assert_success() pex_info = PexInfo.from_pex(pex_out_path) - assert InterpreterConstraints.parse(">=2.7,<3", ">=3.5") == pex_info.interpreter_constraints + assert ( + InterpreterConstraints.parse(">=2.7,<3", ">=3.5,<3.12") + == pex_info.interpreter_constraints + ) def test_interpreter_constraints_to_pex_info_py3(): diff --git a/tests/integration/test_issue_1179.py b/tests/integration/test_issue_1179.py index 0031594d0..cfccfd1e9 100644 --- a/tests/integration/test_issue_1179.py +++ b/tests/integration/test_issue_1179.py @@ -1,9 +1,21 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import sys + +import pytest + from pex.testing import run_pex_command +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "There is an indirect urllib3 dependency which embeds six which uses a meta path importer " + "that only implements the PEP-302 finder spec and not the modern spec. Only the modern " + "finder spec is supported by Python 3.12+." + ), +) def test_pip_2020_resolver_engaged(): # type: () -> None diff --git a/tests/integration/test_issue_1218.py b/tests/integration/test_issue_1218.py index fc45eb230..a6d5d26f2 100644 --- a/tests/integration/test_issue_1218.py +++ b/tests/integration/test_issue_1218.py @@ -2,6 +2,9 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import sys + +import pytest from pex.pex_info import PexInfo from pex.testing import run_pex_command, run_simple_pex @@ -11,13 +14,21 @@ from typing import Any, Dict, Tuple +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "The invoke dependency embeds six which uses a meta path importer that only implements the " + "PEP-302 finder spec and not the modern spec. Only the modern finder spec is supported by " + "Python 3.12+." + ), +) def test_venv_mode_dir_hash_includes_all_pex_info_metadata(tmpdir): # type: (Any) -> None def get_fabric_versions(pex): # type: (str) -> Dict[str, str] output, returncode = run_simple_pex(pex, args=["--version"]) - assert 0 == returncode + assert 0 == returncode, output.decode("utf-8") return dict( cast("Tuple[str, str]", line.split(" ", 1)) for line in output.decode("utf-8").splitlines() diff --git a/tests/integration/test_issue_1809.py b/tests/integration/test_issue_1809.py index 76a29d54c..6834c88d2 100644 --- a/tests/integration/test_issue_1809.py +++ b/tests/integration/test_issue_1809.py @@ -4,8 +4,11 @@ import os.path import shutil import subprocess +import sys from textwrap import dedent +import pytest + from pex.common import safe_open from pex.testing import PY310, ensure_python_distribution, make_project, run_pex_command from pex.typing import TYPE_CHECKING @@ -14,6 +17,10 @@ from typing import Any +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="The test requires using pex 2.1.92 which only supports up to Python 3.11", +) def test_excepthook_scrubbing(tmpdir): # type: (Any) -> None diff --git a/tests/integration/test_issue_1872.py b/tests/integration/test_issue_1872.py index 2990720f9..9d7dbbc6f 100644 --- a/tests/integration/test_issue_1872.py +++ b/tests/integration/test_issue_1872.py @@ -30,7 +30,7 @@ def test_pep_518_venv_pex_env_scrubbing( package_script = os.path.join(pex_project_dir, "scripts", "package.py") pex_pex = os.path.join(str(tmpdir), "pex") - subprocess.check_call(args=[python, package_script, "--local", "--pex-output-file", pex_pex]) + subprocess.check_call(args=[python, package_script, "--pex-output-file", pex_pex]) lock = os.path.join(str(tmpdir), "lock.json") subprocess.check_call( diff --git a/tests/integration/test_issue_2006.py b/tests/integration/test_issue_2006.py index 7da9e3eb9..3eda37764 100644 --- a/tests/integration/test_issue_2006.py +++ b/tests/integration/test_issue_2006.py @@ -36,7 +36,7 @@ def test_symlink_preserved_in_argv0( pex = os.path.join(str(tmpdir), "speak.pex") run_pex_command( - args=["conscript==0.1.5", "cowsay==5.0", "fortune==1.1.0", "-c", "conscript", "-o", pex] + args=["conscript==0.1.6", "cowsay==5.0", "fortune==1.1.0", "-c", "conscript", "-o", pex] + boot_args ).assert_success() diff --git a/tests/integration/test_issue_729.py b/tests/integration/test_issue_729.py index 4f870c1df..e6ee32473 100644 --- a/tests/integration/test_issue_729.py +++ b/tests/integration/test_issue_729.py @@ -4,8 +4,11 @@ from __future__ import print_function import os +import sys from textwrap import dedent +import pytest + from pex.testing import run_pex_command from pex.typing import TYPE_CHECKING @@ -13,6 +16,14 @@ from typing import Any +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "The setuptools dependency embeds pkg_resources which uses a vendor meta path importer " + "that only implements the PEP-302 finder spec and not the modern spec. Only the modern " + "finder spec is supported by Python 3.12+." + ), +) def test_undeclared_setuptools_import_on_pex_path(tmpdir): # type: (Any) -> None """Test that packages which access pkg_resources at import time can be found with pkg_resources. diff --git a/tests/integration/test_issue_940.py b/tests/integration/test_issue_940.py index ee8899b9b..e1f52d5a4 100644 --- a/tests/integration/test_issue_940.py +++ b/tests/integration/test_issue_940.py @@ -2,14 +2,38 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import sys +from textwrap import dedent + +import pytest from pex.common import temporary_dir from pex.testing import built_wheel, run_pex_command, run_simple_pex +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason="We need to use setuptools<66 but Python 3.12+ require greater.", +) def test_resolve_arbitrary_equality(): # type: () -> None + + def prepare_project(project_dir): + # type: (str) -> None + with open(os.path.join(project_dir, "pyproject.toml"), "w") as fp: + fp.write( + dedent( + """\ + [build-system] + # Setuptools 66 removed support for PEP-440 non-compliant versions. + # See: https://setuptools.pypa.io/en/stable/history.html#v66-0-0 + requires = ["setuptools<66"] + """ + ) + ) + with temporary_dir() as tmpdir, built_wheel( + prepare_project=prepare_project, name="foo", version="1.0.2-fba4511", # We need this to allow the invalid version above to sneak by pip wheel metadata diff --git a/tests/integration/test_locked_resolve.py b/tests/integration/test_locked_resolve.py index 0cc8079cf..d6ce73ab3 100644 --- a/tests/integration/test_locked_resolve.py +++ b/tests/integration/test_locked_resolve.py @@ -69,7 +69,11 @@ def create_lock( ): # type: (...) -> Tuple[Downloaded, Tuple[LockedResolve, ...]] lock_observer = create_lock_observer(lock_configuration) - downloaded = resolver.download(observer=lock_observer, **kwargs) + downloaded = resolver.download( + observer=lock_observer, + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + **kwargs + ) return downloaded, lock_observer.lock(downloaded) diff --git a/tests/integration/tools/commands/test_issue_2105.py b/tests/integration/tools/commands/test_issue_2105.py index e284c0737..9c31eeaed 100644 --- a/tests/integration/tools/commands/test_issue_2105.py +++ b/tests/integration/tools/commands/test_issue_2105.py @@ -16,7 +16,7 @@ from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Any, Iterable, Mapping + from typing import Any, Iterable, Mapping, Optional @pytest.fixture(scope="module") @@ -40,7 +40,7 @@ def baseline_venv_with_pip(td): baseline_venv = Virtualenv.create(venv_dir=str(td.join("baseline.venv"))) baseline_venv.install_pip() baseline_venv_distributions = index_distributions(baseline_venv.iter_distributions()) - assert {PIP_PROJECT_NAME, SETUPTOOLS_PROJECT_NAME} == set(baseline_venv_distributions) + assert PIP_PROJECT_NAME in baseline_venv_distributions return baseline_venv_distributions @@ -52,19 +52,19 @@ def baseline_venv_pip_version(baseline_venv_with_pip): @pytest.fixture(scope="module") def baseline_venv_setuptools_version(baseline_venv_with_pip): - # type: (Mapping[ProjectName, Version]) -> Version - return baseline_venv_with_pip[SETUPTOOLS_PROJECT_NAME] + # type: (Mapping[ProjectName, Version]) -> Optional[Version] + return baseline_venv_with_pip.get(SETUPTOOLS_PROJECT_NAME) def assert_venv_dists( venv_dir, # type: str expected_pip_version, # type: Version - expected_setuptools_version, # type: Version + expected_setuptools_version, # type: Optional[Version] ): virtualenv = Virtualenv(venv_dir) dists = index_distributions(virtualenv.iter_distributions()) assert expected_pip_version == dists[PIP_PROJECT_NAME] - assert expected_setuptools_version == dists[SETUPTOOLS_PROJECT_NAME] + assert expected_setuptools_version == dists.get(SETUPTOOLS_PROJECT_NAME) def reported_version(module): # type: (str) -> Version @@ -79,14 +79,15 @@ def reported_version(module): ) assert expected_pip_version == reported_version("pip") - assert expected_setuptools_version == reported_version("setuptools") + if expected_setuptools_version: + assert expected_setuptools_version == reported_version("setuptools") def assert_venv_dists_no_conflicts( tmpdir, # type: Any pex, # type: str expected_pip_version, # type: Version - expected_setuptools_version, # type: Version + expected_setuptools_version, # type: Optional[Version] ): # type: (...) -> None venv_dir = os.path.join(str(tmpdir), "venv_dir") @@ -97,7 +98,7 @@ def assert_venv_dists_no_conflicts( def test_pip_empty_pex( tmpdir, # type: Any baseline_venv_pip_version, # type: Version - baseline_venv_setuptools_version, # type: Version + baseline_venv_setuptools_version, # type: Optional[Version] ): # type: (...) -> None @@ -115,20 +116,16 @@ def test_pip_empty_pex( def test_pip_pex_no_conflicts( tmpdir, # type: Any baseline_venv_pip_version, # type: Version - baseline_venv_setuptools_version, # type: Version + baseline_venv_setuptools_version, # type: Optional[Version] ): # type: (...) -> None pex = os.path.join(str(tmpdir), "pex") - run_pex_command( - args=[ - "-o", - pex, - "pip=={version}".format(version=baseline_venv_pip_version), - "setuptools=={version}".format(version=baseline_venv_setuptools_version), - "--include-tools", - ] - ).assert_success() + args = ["-o", pex, "pip=={version}".format(version=baseline_venv_pip_version)] + if baseline_venv_setuptools_version: + args.append("setuptools=={version}".format(version=baseline_venv_setuptools_version)) + args.append("--include-tools") + run_pex_command(args).assert_success() assert_venv_dists_no_conflicts( tmpdir, @@ -142,9 +139,9 @@ def assert_venv_dists_conflicts( tmpdir, # type: Any pex, # type: str baseline_venv_pip_version, # type: Version - baseline_venv_setuptools_version, # type: Version + baseline_venv_setuptools_version, # type: Optional[Version] expected_pip_version, # type: Version - expected_setuptools_version, # type: Version + expected_setuptools_version, # type: Optional[Version] ): # type: (...) -> None @@ -211,7 +208,7 @@ def assert_venv_dists_conflicts( def test_pip_pex_pip_conflict( tmpdir, # type: Any baseline_venv_pip_version, # type: Version - baseline_venv_setuptools_version, # type: Version + baseline_venv_setuptools_version, # type: Optional[Version] ): # type: (...) -> None @@ -239,10 +236,13 @@ def test_pip_pex_pip_conflict( def test_pip_pex_setuptools_conflict( tmpdir, # type: Any baseline_venv_pip_version, # type: Version - baseline_venv_setuptools_version, # type: Version + baseline_venv_setuptools_version, # type: Optional[Version] ): # type: (...) -> None + if not baseline_venv_setuptools_version: + return + pex = os.path.join(str(tmpdir), "pex") run_pex_command( args=[ @@ -252,7 +252,7 @@ def test_pip_pex_setuptools_conflict( "--include-tools", ] ).assert_success() - pex_setuptools_version = index_distributions(PEX(pex).resolve())[SETUPTOOLS_PROJECT_NAME] + pex_setuptools_version = index_distributions(PEX(pex).resolve()).get(SETUPTOOLS_PROJECT_NAME) assert_venv_dists_conflicts( tmpdir, @@ -267,22 +267,18 @@ def test_pip_pex_setuptools_conflict( def test_pip_pex_both_conflict( tmpdir, # type: Any baseline_venv_pip_version, # type: Version - baseline_venv_setuptools_version, # type: Version + baseline_venv_setuptools_version, # type: Optional[Version] ): # type: (...) -> None pex = os.path.join(str(tmpdir), "pex") - run_pex_command( - args=[ - "-o", - pex, - "pip!={version}".format(version=baseline_venv_pip_version), - "setuptools!={version}".format(version=baseline_venv_setuptools_version), - "--include-tools", - ] - ).assert_success() + args = ["-o", pex, "pip!={version}".format(version=baseline_venv_pip_version)] + if baseline_venv_setuptools_version: + args.append("setuptools!={version}".format(version=baseline_venv_setuptools_version)) + args.append("--include-tools") + run_pex_command(args).assert_success() pex_pip_version = index_distributions(PEX(pex).resolve())[PIP_PROJECT_NAME] - pex_setuptools_version = index_distributions(PEX(pex).resolve())[SETUPTOOLS_PROJECT_NAME] + pex_setuptools_version = index_distributions(PEX(pex).resolve()).get(SETUPTOOLS_PROJECT_NAME) assert_venv_dists_conflicts( tmpdir, diff --git a/tests/pip/test_version.py b/tests/pip/test_version.py index 9818cdaa8..d112cbf63 100644 --- a/tests/pip/test_version.py +++ b/tests/pip/test_version.py @@ -9,4 +9,10 @@ def test_latest(): assert PipVersion.LATEST != PipVersion.VENDORED assert PipVersion.LATEST >= PipVersion.v23_1 - assert max(PipVersion.values(), key=lambda pv: pv.version) is PipVersion.LATEST + assert ( + max( + (version for version in PipVersion.values() if not version.hidden), + key=lambda pv: pv.version, + ) + is PipVersion.LATEST + ) diff --git a/tests/resolve/test_pex_repository_resolver.py b/tests/resolve/test_pex_repository_resolver.py index 9beb58ca2..a11358164 100644 --- a/tests/resolve/test_pex_repository_resolver.py +++ b/tests/resolve/test_pex_repository_resolver.py @@ -12,7 +12,9 @@ from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.platforms import Platform +from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.pex_repository_resolver import resolve_from_pex +from pex.resolve.resolver_configuration import PipConfiguration from pex.resolve.resolvers import Unsatisfiable from pex.resolver import resolve from pex.targets import Targets @@ -42,6 +44,7 @@ def create_pex_repository( requirements=requirements, requirement_files=requirement_files, constraint_files=constraint_files, + resolver=ConfiguredResolver(PipConfiguration()), ).installed_distributions: pex_builder.add_distribution(installed_dist.distribution) for direct_req in installed_dist.direct_requirements: diff --git a/tests/test_bdist_pex.py b/tests/test_bdist_pex.py index b33417c2e..641d31a5a 100644 --- a/tests/test_bdist_pex.py +++ b/tests/test_bdist_pex.py @@ -3,9 +3,12 @@ import os import subprocess +import sys from contextlib import contextmanager from textwrap import dedent +import pytest + from pex.common import open_zip, safe_mkdtemp, temporary_dir from pex.testing import WheelBuilder, make_project, pex_project_dir, temporary_content from pex.typing import TYPE_CHECKING @@ -15,6 +18,15 @@ from typing import Dict, Iterable, Iterator, List, Optional, Union +skip_if_no_disutils = pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "Since distutils is gone in Python 3.12+, we don't want to test these commands in those " + "versions." + ), +) + + BDIST_PEX_VENV = None # type: Optional[Virtualenv] @@ -84,11 +96,13 @@ def assert_pex_args_shebang(shebang): assert fp.readline().decode().rstrip() == shebang +@skip_if_no_disutils def test_entry_points_dict(): # type: () -> None (_,) = assert_entry_points({"console_scripts": ["my_app = my_app.my_module:do_something"]}) +@skip_if_no_disutils def test_entry_points_ini_string(): # type: () -> None (_,) = assert_entry_points( @@ -101,6 +115,7 @@ def test_entry_points_ini_string(): ) +@skip_if_no_disutils def test_bdist_all_single_entry_point_dict(): # type: () -> None assert {"first_app"} == set( @@ -110,6 +125,7 @@ def test_bdist_all_single_entry_point_dict(): ) +@skip_if_no_disutils def test_bdist_all_two_entry_points_dict(): # type: () -> None assert {"first_app", "second_app"} == set( @@ -125,6 +141,7 @@ def test_bdist_all_two_entry_points_dict(): ) +@skip_if_no_disutils def test_bdist_all_single_entry_point_ini_string(): # type: () -> None (my_app,) = assert_entry_points( @@ -139,6 +156,7 @@ def test_bdist_all_single_entry_point_ini_string(): assert "my_app" == my_app +@skip_if_no_disutils def test_bdist_all_two_entry_points_ini_string(): # type: () -> None assert {"first_app", "second_app"} == set( @@ -155,16 +173,19 @@ def test_bdist_all_two_entry_points_ini_string(): ) +@skip_if_no_disutils def test_pex_args_shebang_with_spaces(): # type: () -> None assert_pex_args_shebang("#!/usr/bin/env python") +@skip_if_no_disutils def test_pex_args_shebang_without_spaces(): # type: () -> None assert_pex_args_shebang("#!/usr/bin/python") +@skip_if_no_disutils def test_unwriteable_contents(): # type: () -> None my_app_setup_py = dedent( diff --git a/tests/test_environment.py b/tests/test_environment.py index 6f692f8b7..86336edab 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -4,6 +4,7 @@ import os import platform import subprocess +import sys from contextlib import contextmanager from textwrap import dedent @@ -22,6 +23,8 @@ from pex.pex import PEX from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo +from pex.resolve.configured_resolver import ConfiguredResolver +from pex.resolve.resolver_configuration import PipConfiguration from pex.targets import LocalInterpreter, Targets from pex.testing import ( IS_LINUX_X86_64, @@ -133,11 +136,12 @@ def main(): } ) - def add_requirements(builder, cache): - # type: (PEXBuilder, str) -> None + def add_requirements(builder): + # type: (PEXBuilder) -> None for installed_dist in resolver.resolve( targets=Targets(interpreters=(builder.interpreter,)), requirements=requirements, + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), ).installed_distributions: builder.add_distribution(installed_dist.distribution) for direct_req in installed_dist.direct_requirements: @@ -155,11 +159,11 @@ def add_sources(builder, content): for path in content.keys(): builder.add_source(os.path.join(project, path), path) - with temporary_dir() as root, temporary_dir() as cache: + with temporary_dir() as root: pex_info1 = PexInfo.default() pex1 = os.path.join(root, "pex1.pex") builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1) - add_requirements(builder1, cache) + add_requirements(builder1) add_wheel(builder1, content1) add_sources(builder1, content2) builder1.build(pex1) @@ -168,7 +172,7 @@ def add_sources(builder, content): pex_info2.pex_path = [pex1] pex2 = os.path.join(root, "pex2") builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2) - add_requirements(builder2, cache) + add_requirements(builder2) add_wheel(builder2, content3) builder2.set_script("foobaz") builder2.freeze() @@ -188,6 +192,18 @@ def get_setuptools_requirement(interpreter=None): ) +skip_if_no_pkg_resources = pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "These tests requires pex.third_party.pkg_resources from vendored setuptools and that " + "version of pkg_resources uses a vendor meta path importer that only implements the " + "PEP-302 finder spec and not the modern spec. Only the modern finder spec is supported by " + "Python 3.12+." + ), +) + + +@skip_if_no_pkg_resources @pytest.mark.xfail(IS_PYPY3, reason="https://github.com/pantsbuild/pex/issues/1210") def test_issues_598_explicit_any_interpreter(): # type: () -> None @@ -196,6 +212,7 @@ def test_issues_598_explicit_any_interpreter(): ) +@skip_if_no_pkg_resources def test_issues_598_explicit_missing_requirement(): # type: () -> None assert_force_local_implicit_ns_packages_issues_598(create_ns_packages=True) @@ -208,6 +225,7 @@ def python_38_interpreter(): return PythonInterpreter.from_binary(ensure_python_interpreter(PY38)) +@skip_if_no_pkg_resources def test_issues_598_implicit(python_38_interpreter): # type: (PythonInterpreter) -> None assert_force_local_implicit_ns_packages_issues_598( @@ -215,6 +233,7 @@ def test_issues_598_implicit(python_38_interpreter): ) +@skip_if_no_pkg_resources def test_issues_598_implicit_explicit_mixed(python_38_interpreter): # type: (PythonInterpreter) -> None assert_force_local_implicit_ns_packages_issues_598( @@ -255,6 +274,7 @@ def bad_interpreter(): for installed_dist in resolver.resolve( targets=Targets(interpreters=(pb.interpreter,)), requirements=["psutil==5.4.3"], + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), ).installed_distributions: pb.add_dist_location(installed_dist.distribution.location) pb.build(pex_file) @@ -307,12 +327,22 @@ def run(args, **env): assert "macosx-{}-{}".format(major_minor, machine) == stdout.strip() +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "Pex 1.6.3 attempts an import of pex.third_party.pkg_resources from vendored setuptools " + "and that version of pkg_resources uses a vendor meta path importer that only implements " + "the PEP-302 finder spec and not the modern spec. Only the modern finder spec is supported " + "by Python 3.12+." + ), +) def test_activate_extras_issue_615(): # type: () -> None with yield_pex_builder() as pb: for installed_dist in resolver.resolve( targets=Targets(interpreters=(pb.interpreter,)), requirements=["pex[requests]==1.6.3"], + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), ).installed_distributions: for direct_req in installed_dist.direct_requirements: pb.add_requirement(direct_req) @@ -328,7 +358,7 @@ def test_activate_extras_issue_615(): ) stdout, stderr = process.communicate() assert 0 == process.returncode, "Process failed with exit code {} and output:\n{}".format( - process.returncode, stderr + process.returncode, stderr.decode("utf-8") ) assert to_bytes("{} 1.6.3".format(os.path.basename(pb.path()))) == stdout.strip() @@ -337,7 +367,10 @@ def assert_namespace_packages_warning(distribution, version, expected_warning): # type: (str, str, bool) -> None requirement = "{}=={}".format(distribution, version) pb = PEXBuilder() - for installed_dist in resolver.resolve(requirements=[requirement]).installed_distributions: + for installed_dist in resolver.resolve( + requirements=[requirement], + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + ).installed_distributions: pb.add_dist_location(installed_dist.distribution.location) pb.freeze() @@ -345,15 +378,20 @@ def assert_namespace_packages_warning(distribution, version, expected_warning): _, stderr = process.communicate() stderr_text = stderr.decode("utf8") - partial_warning_preamble = "PEXWarning: The `pkg_resources` package was loaded" + if sys.version_info[:2] >= (3, 12): + partial_warning_preamble = ( + "PEXWarning: The legacy `pkg_resources` package cannot be imported by" + ) + else: + partial_warning_preamble = "PEXWarning: The `pkg_resources` package was loaded" partial_warning_detail = "{} namespace packages:".format(requirement) if expected_warning: - assert partial_warning_preamble in stderr_text - assert partial_warning_detail in stderr_text + assert partial_warning_preamble in stderr_text, stderr_text + assert partial_warning_detail in stderr_text, stderr_text else: - assert partial_warning_preamble not in stderr_text - assert partial_warning_detail not in stderr_text + assert partial_warning_preamble not in stderr_text, stderr_text + assert partial_warning_detail not in stderr_text, stderr_text def test_present_non_empty_namespace_packages_metadata_does_warn(): diff --git a/tests/test_pex.py b/tests/test_pex.py index 65b0f5bfc..cd44ce688 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -17,11 +17,13 @@ from pex import resolver from pex.common import safe_mkdir, safe_open, temporary_dir from pex.compatibility import PY2, WINDOWS, to_bytes -from pex.dist_metadata import Distribution +from pex.dist_metadata import Distribution, Requirement from pex.interpreter import PythonInterpreter from pex.pex import PEX, IsolatedSysPath from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo +from pex.resolve.configured_resolver import ConfiguredResolver +from pex.resolve.resolver_configuration import PipConfiguration from pex.testing import ( PY39, PY310, @@ -818,26 +820,52 @@ def test_pex_run_strip_env(): }, "Expected the parent environment to be left un-stripped." -def test_pex_run_custom_setuptools_useable(): - # type: () -> None - result = resolver.resolve(requirements=["setuptools==43.0.0"]) +@pytest.fixture +def setuptools_version(): + # type: () -> str + return "67.8.0" if sys.version_info[:2] >= (3, 12) else "43.0.0" + + +@pytest.fixture +def setuptools_requirement(setuptools_version): + # type: (str) -> str + return "setuptools=={version}".format(version=setuptools_version) + + +def test_pex_run_custom_setuptools_useable( + setuptools_requirement, # type: str + setuptools_version, # type: str +): + # type: (...) -> None + result = resolver.resolve( + requirements=[setuptools_requirement], + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + ) dists = [installed_dist.distribution for installed_dist in result.installed_distributions] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, - "import setuptools, sys; sys.exit(0 if '43.0.0' == setuptools.__version__ else 1)", + "import setuptools, sys; sys.exit(0 if '{version}' == setuptools.__version__ else 1)".format( + version=setuptools_version + ), dists=dists, ) rc = PEX(pex.path()).run() assert rc == 0 -def test_pex_run_conflicting_custom_setuptools_useable(): - # type: () -> None +def test_pex_run_conflicting_custom_setuptools_useable( + setuptools_requirement, # type: str + setuptools_version, # type: str +): + # type: (...) -> None # Here we use our vendored, newer setuptools to build the pex which has an older setuptools # requirement. - result = resolver.resolve(requirements=["setuptools==43.0.0"]) + result = resolver.resolve( + requirements=[setuptools_requirement], + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + ) dists = [installed_dist.distribution for installed_dist in result.installed_distributions] with temporary_dir() as temp_dir: pex = write_simple_pex( @@ -847,8 +875,10 @@ def test_pex_run_conflicting_custom_setuptools_useable(): import sys import setuptools - sys.exit(0 if '43.0.0' == setuptools.__version__ else 1) - """ + sys.exit(0 if '{version}' == setuptools.__version__ else 1) + """.format( + version=setuptools_version + ) ), dists=dists, ) @@ -860,7 +890,8 @@ def test_pex_run_custom_pex_useable(): # type: () -> None old_pex_version = "0.7.0" result = resolver.resolve( - requirements=["pex=={}".format(old_pex_version), "setuptools==40.6.3"] + requirements=["pex=={}".format(old_pex_version), "setuptools==40.6.3"], + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), ) dists = [installed_dist.distribution for installed_dist in result.installed_distributions] with temporary_dir() as temp_dir: @@ -878,8 +909,13 @@ def test_pex_run_custom_pex_useable(): from pex.version import __version__ sys.exit(1) except ImportError: - import pkg_resources - dist = pkg_resources.working_set.find(pkg_resources.Requirement.parse('pex')) + # N.B.: pkg_resources is not supported by Python >= 3.12. + if sys.version_info[:2] >= (3, 12): + from importlib.metadata import distribution + dist = distribution('pex') + else: + import pkg_resources + dist = pkg_resources.working_set.find(pkg_resources.Requirement.parse('pex')) print(dist.version) """ ), diff --git a/tests/test_pex_binary.py b/tests/test_pex_binary.py index ef51cc34d..385009116 100644 --- a/tests/test_pex_binary.py +++ b/tests/test_pex_binary.py @@ -117,9 +117,6 @@ def test_clp_prereleases_resolver(): options = parser.parse_args( args=[ - # This test is run against all Pythons; so ensure we have a Pip that works with all - # the pythons we support. - "--pip-version=vendored", "--no-index", "--find-links", dist_dir, @@ -144,9 +141,6 @@ def test_clp_prereleases_resolver(): # When we specify `--pre`, allow_prereleases is True options = parser.parse_args( args=[ - # This test is run against all Pythons; so ensure we have a Pip that works with all - # the pythons we support. - "--pip-version=vendored", "--no-index", "--find-links", dist_dir, diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index b5bcd3c1f..aabf5af8d 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -322,7 +322,7 @@ def test_pex_builder_packed(tmpdir): spread_dist_zip = os.path.join(pex_app, pb.info.internal_cache, location) assert zipfile.is_zipfile(spread_dist_zip) - cached_dist_zip = os.path.join(pex_root, "installed_wheel_zips", sha, location) + cached_dist_zip = os.path.join(pex_root, "packed_wheels", sha, location) assert zipfile.is_zipfile(cached_dist_zip) assert filecmp.cmp(spread_dist_zip, cached_dist_zip, shallow=False) diff --git a/tests/test_pip.py b/tests/test_pip.py index 2f83b2307..8598b04e7 100644 --- a/tests/test_pip.py +++ b/tests/test_pip.py @@ -30,7 +30,7 @@ class CreatePip(Protocol): def __call__( self, interpreter, # type: Optional[PythonInterpreter] - version=PipVersion.VENDORED, # type: PipVersionValue + version=PipVersion.DEFAULT, # type: PipVersionValue **extra_env # type: str ): # type: (...) -> Pip @@ -59,7 +59,7 @@ def create_pip( def create_pip( interpreter, # type: Optional[PythonInterpreter] - version=PipVersion.VENDORED, # type: PipVersionValue + version=PipVersion.DEFAULT, # type: PipVersionValue **extra_env # type: str ): # type: (...) -> Pip @@ -303,9 +303,9 @@ def test_pip_pex_interpreter_venv_hash_issue_1885( # Remove any existing pip.pex which may exist as a result of other test suites. installation = PipInstallation( interpreter=current_interpreter, - version=PipVersion.VENDORED, + version=PipVersion.DEFAULT, ) - del _PIP[installation] + _PIP.pop(installation, None) binary = current_interpreter.binary binary_link = os.path.join(str(tmpdir), "python") os.symlink(binary, binary_link) @@ -321,4 +321,4 @@ def test_pip_pex_interpreter_venv_hash_issue_1885( sort_keys=True, ).encode("utf-8") ).hexdigest() - assert venv_contents_hash in pip_w_linked_ppp._pip_pex.venv_dir + assert venv_contents_hash in pip_w_linked_ppp._pip.venv_dir diff --git a/tests/test_resolver.py b/tests/test_resolver.py index e40496150..a22f0effe 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -13,13 +13,16 @@ import pytest from pex import targets +from pex.build_system.pep_517 import build_sdist from pex.common import safe_copy, safe_mkdtemp, temporary_dir from pex.dist_metadata import Requirement -from pex.interpreter import PythonInterpreter, spawn_python_job +from pex.interpreter import PythonInterpreter from pex.platforms import Platform -from pex.resolve.resolver_configuration import ResolverVersion -from pex.resolve.resolvers import InstalledDistribution, Unsatisfiable -from pex.resolver import download, resolve +from pex.resolve.configured_resolver import ConfiguredResolver +from pex.resolve.resolver_configuration import PipConfiguration, ResolverVersion +from pex.resolve.resolvers import Installed, InstalledDistribution, Unsatisfiable +from pex.resolver import download +from pex.resolver import resolve as resolve_under_test from pex.targets import Targets from pex.testing import ( IS_LINUX, @@ -45,14 +48,12 @@ def create_sdist(**kwargs): dist_dir = safe_mkdtemp() with make_project(**kwargs) as project_dir: - cmd = ["setup.py", "sdist", "--dist-dir={}".format(dist_dir)] - spawn_python_job( - args=cmd, - cwd=project_dir, - expose=["setuptools"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ).communicate() + build_sdist( + project_directory=project_dir, + dist_dir=dist_dir, + target=targets.current(), + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), + ) dists = os.listdir(dist_dir) assert len(dists) == 1 @@ -65,6 +66,12 @@ def build_wheel(**kwargs): return whl +def resolve(**kwargs): + # type: (**Any) -> Installed + kwargs.setdefault("resolver", ConfiguredResolver(pip_configuration=PipConfiguration())) + return resolve_under_test(**kwargs) + + def local_resolve(*args, **kwargs): # type: (*Any, **Any) -> List[InstalledDistribution] # Skip remote lookups. @@ -267,7 +274,15 @@ def resolve_p537_wheel_names( ): # type: (...) -> List[str] with cache(cache_dir): - return resolve_wheel_names(requirements=["p537==1.0.5"], transitive=False, **kwargs) + return resolve_wheel_names( + requirements=[ + "p537=={version}".format( + version="1.0.6" if sys.version_info[:2] >= (3, 6) else "1.0.5" + ) + ], + transitive=False, + **kwargs + ) @pytest.fixture(scope="module") @@ -485,6 +500,7 @@ def test_download(): result = download( requirements=["{}[foo]".format(project1_sdist)], find_links=[os.path.dirname(project2_wheel)], + resolver=ConfiguredResolver(pip_configuration=PipConfiguration()), ) for local_distribution in result.local_distributions: distribution = pkginfo.get_metadata(local_distribution.path) @@ -508,19 +524,39 @@ def assert_dist(project_name, dist_type, version): assert_dist("setuptools", pkginfo.Wheel, "44.1.0") +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason="We need to use setuptools<66 ut Python 3.12+ require greater.", +) def test_resolve_arbitrary_equality_issues_940(): - # type: () -> None + # type: (...) -> None + + def prepare_project(project_dir): + # type: (str) -> None + with open(os.path.join(project_dir, "pyproject.toml"), "w") as fp: + fp.write( + dedent( + """\ + [build-system] + # Setuptools 66 removed support for PEP-440 non-compliant versions. + # See: https://setuptools.pypa.io/en/stable/history.html#v66-0-0 + requires = ["setuptools<66"] + """ + ) + ) + dist = create_sdist( + prepare_project=prepare_project, name="foo", version="1.0.2-fba4511", python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*", ) - installed_distributions = local_resolve( + installed_distributions = resolve( requirements=[dist], # We need this to allow the invalid version above to sneak by pip wheel metadata # verification. verify_wheels=False, - ) + ).installed_distributions assert len(installed_distributions) == 1 requirements = installed_distributions[0].direct_requirements diff --git a/tests/test_vendor.py b/tests/test_vendor.py index 536cc58d2..7edd9dd85 100644 --- a/tests/test_vendor.py +++ b/tests/test_vendor.py @@ -41,7 +41,7 @@ def test_git_prep_command(tmpdir): touch(os.path.join(repo, "README")) subprocess.check_call(["git", "add", "README"], cwd=repo) - subprocess.check_call(["git", "commit", "-m", "Initial Commit."], cwd=repo) + subprocess.check_call(["git", "commit", "--no-gpg-sign", "-m", "Initial Commit."], cwd=repo) commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo).decode("utf-8").strip() prep_file = os.path.join(repo, "prep") diff --git a/tests/tools/commands/test_venv.py b/tests/tools/commands/test_venv.py index 8db7c08e2..af97a15f5 100644 --- a/tests/tools/commands/test_venv.py +++ b/tests/tools/commands/test_venv.py @@ -152,6 +152,14 @@ def parse_fabric_version_output(output): return dict(cast("Tuple[Text, Text]", line.split(" ", 1)) for line in output.splitlines()) +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "Fabric depends on invoke which embeds six which uses a meta path importer that only " + "implements the PEP-302 finder spec and not the modern spec. Only the modern finder spec " + "is supported by Python 3.12+." + ), +) def test_venv_pex(create_pex_venv): # type: (CreatePexVenv) -> None venv = create_pex_venv() @@ -259,6 +267,14 @@ def try_invoke(*args): assert 0 == returncode +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 12), + reason=( + "Fabric depends on invoke which embeds six which uses a meta path importer that only " + "implements the PEP-302 finder spec and not the modern spec. Only the modern finder spec " + "is supported by Python 3.12+." + ), +) def test_venv_pex_interpreter_special_modes(create_pex_venv): # type: (CreatePexVenv) -> None venv = create_pex_venv() diff --git a/tox.ini b/tox.ini index abacfe928..799d5bd0f 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,8 @@ isolated_build = true skip_missing_interpreters = True minversion = 3.25.1 requires = + # Ensure tox and virtualenv compatible back through Python 2.7. + tox<4 virtualenv<20.16 [tox:.package] @@ -17,6 +19,9 @@ commands = python scripts/print_env.py [testenv] +# N.B.: We need modern setuptools downloaded out of band by virtualenv to work with the Python 3.12 +# case. Trying to upgrade via Pip is too late and Pip blows up. +download = true commands = {[_printenv]commands} pytest --ignore=tests/integration {posargs:-vvs} @@ -59,6 +64,7 @@ setenv = pip23_1: _PEX_PIP_VERSION=23.1 pip23_1_1: _PEX_PIP_VERSION=23.1.1 pip23_1_2: _PEX_PIP_VERSION=23.1.2 + pip23_2: _PEX_PIP_VERSION=23.2.dev0+8a1eea4a # Python 3 (until a fix here in 3.9: https://bugs.python.org/issue13601) switched from stderr # being unbuffered to stderr being buffered by default. This can lead to tests checking stderr # failing to see what they expect if the stderr buffer block has not been flushed. Force stderr @@ -70,7 +76,7 @@ whitelist_externals = bash git -[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,27,35,36,37,38,39,310,311}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-,pip23_1_2-}integration] +[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,27,35,36,37,38,39,310,311,312}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-,pip23_1_2-,pip23_2-}integration] deps = pytest-xdist==1.34.0 {[testenv]deps}