From e95a5c79ce90d713f120dcfb045dee478e50fe59 Mon Sep 17 00:00:00 2001 From: Kyle Finley Date: Fri, 13 Sep 2024 11:17:01 -0400 Subject: [PATCH] lambda functions built using pipenv now require >= 2022.8.13 --- .../aws_lambda.upload_lambda_functions.rst | 3 + .../cfngin/hooks/awslambda.PythonFunction.rst | 4 + .../cfngin/hooks/awslambda.PythonLayer.rst | 3 + poetry.lock | 79 +++++++------------ pyproject.toml | 3 +- runway/cfngin/hooks/aws_lambda.py | 11 ++- runway/dependency_managers/_pipenv.py | 14 ++-- tests/unit/cfngin/hooks/test_awslambda.py | 39 ++------- .../unit/dependency_managers/test__pipenv.py | 48 +++++++---- 9 files changed, 92 insertions(+), 112 deletions(-) diff --git a/docs/source/cfngin/hooks/aws_lambda.upload_lambda_functions.rst b/docs/source/cfngin/hooks/aws_lambda.upload_lambda_functions.rst index 08835a245..708bd5164 100644 --- a/docs/source/cfngin/hooks/aws_lambda.upload_lambda_functions.rst +++ b/docs/source/cfngin/hooks/aws_lambda.upload_lambda_functions.rst @@ -30,6 +30,9 @@ This can be done by including the ``dockerize_pip`` configuration option which c Payloads are uploaded to either the |cfngin_bucket| or an explicitly specified bucket, with the object key containing it's checksum to allow repeated uploads to be skipped in subsequent runs. +.. versionchanged:: 2.8.0 + Use of pipenv now requires version ``>= 2022.8.13``. + This is the version that changed how ``requirements.txt`` files are generated. **** diff --git a/docs/source/cfngin/hooks/awslambda.PythonFunction.rst b/docs/source/cfngin/hooks/awslambda.PythonFunction.rst index 700a59c81..96df47e19 100644 --- a/docs/source/cfngin/hooks/awslambda.PythonFunction.rst +++ b/docs/source/cfngin/hooks/awslambda.PythonFunction.rst @@ -23,6 +23,10 @@ It also ensures that binary files built during the install process are compatibl .. versionadded:: 2.5.0 +.. versionchanged:: 2.8.0 + Use of pipenv now requires version ``>= 2022.8.13``. + This is the version that changed how ``requirements.txt`` files are generated. + **** diff --git a/docs/source/cfngin/hooks/awslambda.PythonLayer.rst b/docs/source/cfngin/hooks/awslambda.PythonLayer.rst index b3ab86e31..c4f92ba51 100644 --- a/docs/source/cfngin/hooks/awslambda.PythonLayer.rst +++ b/docs/source/cfngin/hooks/awslambda.PythonLayer.rst @@ -21,6 +21,9 @@ It also ensures that binary files built during the install process are compatibl .. versionadded:: 2.5.0 +.. versionchanged:: 2.8.0 + Use of pipenv now requires version ``>= 2022.8.13``. + This is the version that changed how ``requirements.txt`` files are generated. **** diff --git a/poetry.lock b/poetry.lock index 819311845..c0f78df3b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1089,13 +1089,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "distlib" -version = "0.3.6" +version = "0.3.8" description = "Distribution utilities" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] [[package]] @@ -1933,53 +1933,41 @@ files = [ {file = "pbr-6.0.0.tar.gz", hash = "sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9"}, ] -[[package]] -name = "pip" -version = "23.3.1" -description = "The PyPA recommended tool for installing Python packages." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pip-23.3.1-py3-none-any.whl", hash = "sha256:55eb67bb6171d37447e82213be585b75fe2b12b359e993773aca4de9247a052b"}, - {file = "pip-23.3.1.tar.gz", hash = "sha256:1fcaa041308d01f14575f6d0d2ea4b75a3e2871fe4f9c694976f908768e14174"}, -] - [[package]] name = "pipenv" -version = "2022.1.8" +version = "2024.0.1" description = "Python Development Workflow for Humans." optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "pipenv-2022.1.8-py2.py3-none-any.whl", hash = "sha256:3b80b4512934b9d8e8ce12c988394642ff96bb697680e5b092e59af503178327"}, - {file = "pipenv-2022.1.8.tar.gz", hash = "sha256:f84d7119239b22ab2ac2b8fbc7d619d83cf41135206d72a17c4f151cda529fd0"}, + {file = "pipenv-2024.0.1-py3-none-any.whl", hash = "sha256:5360835c613837423a99b8c94952b139b777b3eaabb42cb9edb34556245b4c25"}, + {file = "pipenv-2024.0.1.tar.gz", hash = "sha256:ae5a83fa5b66065cebd2bd8f73f0b281b3bd202a13d58cc644f0b9765128c990"}, ] [package.dependencies] certifi = "*" -pip = ">=18.0" -setuptools = ">=36.2.1" -virtualenv = "*" -virtualenv-clone = ">=0.2.5" +setuptools = ">=67" +virtualenv = ">=20.24.2" [package.extras] -dev = ["black", "bs4", "flake8 (>=3.3.0,<4.0)", "invoke", "parver", "sphinx", "towncrier", "twine"] -tests = ["flaky", "mock", "pytest (>=5.0)", "pytest-timeout", "pytest-xdist"] +dev = ["beautifulsoup4", "black (==23.3)", "flake8 (>=3.3,<4.0)", "invoke", "parver", "sphinx", "towncrier"] +tests = ["flaky", "mock", "pytest (>=5)", "pytest-timeout", "pytest-xdist"] [[package]] name = "platformdirs" -version = "3.1.1" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "4.3.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-3.1.1-py3-none-any.whl", hash = "sha256:e5986afb596e4bb5bde29a79ac9061aa955b94fca2399b7aaac4090860920dd8"}, - {file = "platformdirs-3.1.1.tar.gz", hash = "sha256:024996549ee88ec1a9aa99ff7f8fc819bb59e2c3477b410d90a16d32d6e707aa"}, + {file = "platformdirs-4.3.2-py3-none-any.whl", hash = "sha256:eb1c8582560b34ed4ba105009a4badf7f6f85768b30126f351328507b2beb617"}, + {file = "platformdirs-4.3.2.tar.gz", hash = "sha256:9e5e27a08aa095dd127b9f2e764d74254f482fef22b0970773bfba79d091ab8c"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] [[package]] name = "pluggy" @@ -3324,34 +3312,23 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.21.0" +version = "20.26.4" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.21.0-py3-none-any.whl", hash = "sha256:31712f8f2a17bd06234fa97fdf19609e789dd4e3e4bf108c3da71d710651adbc"}, - {file = "virtualenv-20.21.0.tar.gz", hash = "sha256:f50e3e60f990a0757c9b68333c9fdaa72d7188caa417f96af9e52407831a3b68"}, + {file = "virtualenv-20.26.4-py3-none-any.whl", hash = "sha256:48f2695d9809277003f30776d155615ffc11328e6a0a8c1f0ec80188d7874a55"}, + {file = "virtualenv-20.26.4.tar.gz", hash = "sha256:c17f4e0f3e6036e9f26700446f85c76ab11df65ff6d8a9cbfad9f71aabfcf23c"}, ] [package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<4" +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] -test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "virtualenv-clone" -version = "0.5.7" -description = "script to clone virtualenvs." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "virtualenv-clone-0.5.7.tar.gz", hash = "sha256:418ee935c36152f8f153c79824bb93eaf6f0f7984bae31d3f48f350b9183501a"}, - {file = "virtualenv_clone-0.5.7-py3-none-any.whl", hash = "sha256:44d5263bceed0bac3e1424d64f798095233b64def1c5689afa43dc3223caf5b0"}, -] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] [[package]] name = "wcmatch" @@ -3431,4 +3408,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9, <3.13" -content-hash = "88a102cd6e35dd8d3bc8b7263c470cd11e4a263628623941fc81c5a553fe7f0d" +content-hash = "63385fc0e05806d5476714ce3f6253b7109a7ee21544787cbcad805b98268c2d" diff --git a/pyproject.toml b/pyproject.toml index b74787462..4d6db0c5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ gitpython = "*" igittigitt = ">=2.0.5" jinja2 = ">=2.7" # used in runway.cfngin.blueprints.raw packaging = "*" # component of setuptools needed for version compare -pipenv = "2022.1.8" pyOpenSSL = "*" # For embedded hook & associated script usage pydantic = "^2.8.0" pyhcl = "^0.4" # does not support HCL2, possibly move to extras_require in the future @@ -82,7 +81,7 @@ ruff = "^0.6.4" [tool.poetry.group.test.dependencies] coverage = {extras = ["toml"], version = "^7.6.1"} moto = {extras = ["ec2", "ecs", "iam", "s3", "ssm"], version = "^5.0.14"} -pipenv = "^2022.1.8" # only used in tests +pipenv = "^2024.0.1" # only used in tests pytest = "^8.3.3" pytest-cov = "^5.0.0" pytest-mock = "^3.14.0" diff --git a/runway/cfngin/hooks/aws_lambda.py b/runway/cfngin/hooks/aws_lambda.py index 00c515f8e..fabbad2a1 100644 --- a/runway/cfngin/hooks/aws_lambda.py +++ b/runway/cfngin/hooks/aws_lambda.py @@ -354,14 +354,21 @@ def _handle_use_pipenv( sys.exit(1) LOGGER.info("creating requirements.txt from Pipfile...") req_path = os.path.join(dest_path, "requirements.txt") # noqa: PTH118 - cmd = ["pipenv", "lock", "--requirements", "--keep-outdated"] + cmd = ["pipenv", "requirements"] + environ = os.environ.copy() + environ["PIPENV_IGNORE_VIRTUALENVS"] = "1" + + if not (Path(package_root) / "Pipfile.lock").is_file(): + LOGGER.warning("Pipfile.lock does not exist! creating it...") + subprocess.check_call(["pipenv", "lock"], cwd=package_root, env=environ) + if python_path: cmd.insert(0, python_path) cmd.insert(1, "-m") with ( open(req_path, "w", encoding="utf-8") as requirements, # noqa: PTH123 subprocess.Popen( - cmd, cwd=package_root, stdout=requirements, stderr=subprocess.PIPE + cmd, cwd=package_root, env=environ, stdout=requirements, stderr=subprocess.PIPE ) as pipenv_process, ): _stdout, stderr = pipenv_process.communicate(timeout=timeout) diff --git a/runway/dependency_managers/_pipenv.py b/runway/dependency_managers/_pipenv.py index 17d03f439..2f4d2e722 100644 --- a/runway/dependency_managers/_pipenv.py +++ b/runway/dependency_managers/_pipenv.py @@ -87,21 +87,23 @@ def export(self, *, dev: bool = False, output: StrPath) -> Path: """Export the lock file to other formats (requirements.txt only). The underlying command being executed by this method is - ``pipenv lock --requirements``. + ``pipenv requirements``. Args: dev: Include development dependencies. output: Path to the output file. """ + environ = self.ctx.env.vars.copy() + environ["PIPENV_IGNORE_VIRTUALENVS"] = "1" output = Path(output) + if not (self.cwd / "Pipfile.lock").is_file(): + LOGGER.warning("Pipfile.lock does not exist! creating it...") + self._run_command(self.generate_command("lock", quiet=True), env=environ) try: result = self._run_command( - self.generate_command( - "lock", - dev=dev, - requirements=True, - ), + self.generate_command("requirements", dev=dev), + env=environ, suppress_output=True, ) except subprocess.CalledProcessError as exc: diff --git a/tests/unit/cfngin/hooks/test_awslambda.py b/tests/unit/cfngin/hooks/test_awslambda.py index 66ae10032..cc69e905b 100644 --- a/tests/unit/cfngin/hooks/test_awslambda.py +++ b/tests/unit/cfngin/hooks/test_awslambda.py @@ -694,8 +694,9 @@ class TestHandleRequirements: """Test handle_requirements.""" PIPFILE = ( - '[[source]]\nurl = "https://pypi.org/simple"\nverify_ssl = true\nname = "pypi"\n' - "[packages]\n[dev-packages]" + '[[source]]\nurl = "https://pypi.org/simple"\nverify_ssl = false\n' + 'name = "pip_conf_index_global"\n' + '[packages]\nurllib3 = "~=2.2"\n[dev-packages]' ) REQUIREMENTS = "-i https://pypi.org/simple\n\n" @@ -733,22 +734,7 @@ def test_explicit_pipenv(self, tmp_path: Path) -> None: assert req_path == str(requirements_txt) assert (tmp_path / "Pipfile.lock").is_file() - expected_text = [ - "#", - "# These requirements were autogenerated by pipenv", - "# To regenerate from the project's Pipfile, run:", - "#", - "# pipenv lock --requirements", - "#", - "", - "-i https://pypi.org/simple", - "", - ] - if platform.system() == "Windows": - expected_text.append("") - assert requirements_txt.read_text() == "\n".join(expected_text) - else: - assert requirements_txt.read_text() == "\n".join(expected_text) + "\n" + assert "urllib3==" in requirements_txt.read_text() def test_frozen_pipenv( self, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, tmp_path: Path @@ -785,22 +771,7 @@ def test_implicit_pipenv(self, tmp_path: Path) -> None: assert req_path == str(requirements_txt) assert (tmp_path / "Pipfile.lock").is_file() - expected_text = [ - "#", - "# These requirements were autogenerated by pipenv", - "# To regenerate from the project's Pipfile, run:", - "#", - "# pipenv lock --requirements", - "#", - "", - "-i https://pypi.org/simple", - "", - ] - if platform.system() == "Windows": - expected_text.append("") - assert requirements_txt.read_text() == "\n".join(expected_text) - else: - assert requirements_txt.read_text() == "\n".join(expected_text) + "\n" + assert "urllib3==" in requirements_txt.read_text() def test_raise_not_implimented(self) -> None: """Test NotImplimentedError is raised when no requirements file.""" diff --git a/tests/unit/dependency_managers/test__pipenv.py b/tests/unit/dependency_managers/test__pipenv.py index 8ac918da2..ad5e87dc1 100644 --- a/tests/unit/dependency_managers/test__pipenv.py +++ b/tests/unit/dependency_managers/test__pipenv.py @@ -5,7 +5,7 @@ import logging import subprocess from typing import TYPE_CHECKING, Any -from unittest.mock import Mock +from unittest.mock import Mock, call import pytest @@ -16,6 +16,8 @@ from pytest_mock import MockerFixture + from ..factories import MockCfnginContext + MODULE = "runway.dependency_managers._pipenv" @@ -61,48 +63,60 @@ def test_dir_is_project( def test_export( self, export_kwargs: dict[str, Any], + cfngin_context: MockCfnginContext, mocker: MockerFixture, tmp_path: Path, ) -> None: """Test export.""" expected = tmp_path / "expected" / "test.requirements.txt" mock_generate_command = mocker.patch.object( - Pipenv, "generate_command", return_value="generate_command" + Pipenv, "generate_command", side_effect=["lock", "requirements"] ) mock_run_command = mocker.patch.object(Pipenv, "_run_command", return_value="_run_command") - obj = Pipenv(Mock(), tmp_path) + obj = Pipenv(cfngin_context, tmp_path) assert obj.export(output=expected, **export_kwargs) == expected assert expected.is_file() export_kwargs.setdefault("dev", False) - export_kwargs["requirements"] = True # hardcoded in the method - mock_generate_command.assert_called_once_with("lock", **export_kwargs) - mock_run_command.assert_called_once_with( - mock_generate_command.return_value, suppress_output=True + mock_generate_command.assert_has_calls( + [call("lock", quiet=True), call("requirements", **export_kwargs)] + ) + cfngin_context.env.vars["PIPENV_IGNORE_VIRTUALENVS"] = "1" + mock_run_command.assert_has_calls( + [ + call("lock", env=cfngin_context.env.vars), + call("requirements", env=cfngin_context.env.vars, suppress_output=True), + ] ) def test_export_raise_from_called_process_error( self, + cfngin_context: MockCfnginContext, mocker: MockerFixture, tmp_path: Path, ) -> None: """Test export raise PoetryExportFailedError from CalledProcessError.""" output = tmp_path / "expected" / "test.requirements.txt" - mock_generate_command = mocker.patch.object( - Pipenv, "generate_command", return_value="generate_command" - ) + mocker.patch.object(Pipenv, "generate_command", side_effect=["lock", "requirements"]) mock_run_command = mocker.patch.object( Pipenv, "_run_command", - side_effect=subprocess.CalledProcessError( - returncode=1, - cmd=mock_generate_command.return_value, - ), + side_effect=[ + None, + subprocess.CalledProcessError( + returncode=1, + cmd="pipenv requirements", + ), + ], ) with pytest.raises(PipenvExportFailedError): - assert Pipenv(Mock(), tmp_path).export(output=output) - mock_run_command.assert_called_once_with( - mock_generate_command.return_value, suppress_output=True + assert Pipenv(cfngin_context, tmp_path).export(output=output) + cfngin_context.env.vars["PIPENV_IGNORE_VIRTUALENVS"] = "1" + mock_run_command.assert_has_calls( + [ + call("lock", env=cfngin_context.env.vars), + call("requirements", env=cfngin_context.env.vars, suppress_output=True), + ] ) @pytest.mark.parametrize(