From c2f0978a12d06fe250810c352ed8b0cc58ccb4bc Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 2 Aug 2024 12:11:41 -0400 Subject: [PATCH 01/82] build!: update to pydantic 2 (#401) --- craft_application/application.py | 4 +- craft_application/models/__init__.py | 3 +- craft_application/models/base.py | 21 +- craft_application/models/constraints.py | 214 ++++++++++++++----- craft_application/models/grammar.py | 98 ++++----- craft_application/models/metadata.py | 16 +- craft_application/models/project.py | 163 +++++++------- craft_application/services/lifecycle.py | 4 +- craft_application/util/error_formatting.py | 11 +- craft_application/util/snap_config.py | 6 +- pyproject.toml | 19 +- tests/integration/conftest.py | 9 +- tests/integration/services/test_lifecycle.py | 16 +- tests/integration/services/test_provider.py | 12 +- tests/integration/test_application.py | 8 +- tests/unit/commands/test_lifecycle.py | 15 +- tests/unit/models/test_base.py | 10 +- tests/unit/models/test_constraints.py | 84 ++++++-- tests/unit/models/test_project.py | 116 +++++----- tests/unit/services/test_lifecycle.py | 2 +- tests/unit/test_application.py | 30 +-- tests/unit/test_errors.py | 8 +- tests/unit/util/test_snap_config.py | 4 +- 23 files changed, 531 insertions(+), 342 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index 6636adf1..89471684 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -658,7 +658,7 @@ def _expand_environment(self, yaml_data: dict[str, Any], build_for: str) -> None info = craft_parts.ProjectInfo( application_name=self.app.name, # not used in environment expansion cache_dir=pathlib.Path(), # not used in environment expansion - arch=util.convert_architecture_deb_to_platform(build_for_arch), + arch=build_for_arch, project_name=yaml_data.get("name", ""), project_dirs=project_dirs, project_vars=environment_vars, @@ -682,7 +682,7 @@ def _get_project_vars(self, yaml_data: dict[str, Any]) -> dict[str, str]: """Return a dict with project variables to be expanded.""" pvars: dict[str, str] = {} for var in self.app.project_variables: - pvars[var] = yaml_data.get(var, "") + pvars[var] = str(yaml_data.get(var, "")) return pvars def _set_global_environment(self, info: craft_parts.ProjectInfo) -> None: diff --git a/craft_application/models/__init__.py b/craft_application/models/__init__.py index cb5665ee..9b841cf7 100644 --- a/craft_application/models/__init__.py +++ b/craft_application/models/__init__.py @@ -15,7 +15,7 @@ # with this program. If not, see . """General-purpose models for *craft applications.""" -from craft_application.models.base import CraftBaseConfig, CraftBaseModel +from craft_application.models.base import CraftBaseModel from craft_application.models.constraints import ( ProjectName, ProjectTitle, @@ -43,7 +43,6 @@ "BuildInfo", "DEVEL_BASE_INFOS", "DEVEL_BASE_WARNING", - "CraftBaseConfig", "CraftBaseModel", "get_grammar_aware_part_keywords", "GrammarAwareProject", diff --git a/craft_application/models/base.py b/craft_application/models/base.py index da7932e1..4ac8d815 100644 --- a/craft_application/models/base.py +++ b/craft_application/models/base.py @@ -30,24 +30,19 @@ def alias_generator(s: str) -> str: return s.replace("_", "-") -class CraftBaseConfig(pydantic.BaseConfig): # pylint: disable=too-few-public-methods - """Pydantic model configuration.""" - - validate_assignment = True - extra = pydantic.Extra.forbid - allow_mutation = True - allow_population_by_field_name = True - alias_generator = alias_generator - - class CraftBaseModel(pydantic.BaseModel): """Base model for craft-application classes.""" - Config = CraftBaseConfig + model_config = pydantic.ConfigDict( + validate_assignment=True, + extra="forbid", + populate_by_name=True, + alias_generator=alias_generator, + ) def marshal(self) -> dict[str, str | list[str] | dict[str, Any]]: """Convert to a dictionary.""" - return self.dict(by_alias=True, exclude_unset=True) + return self.model_dump(mode="json", by_alias=True, exclude_unset=True) @classmethod def unmarshal(cls, data: dict[str, Any]) -> Self: @@ -62,7 +57,7 @@ def unmarshal(cls, data: dict[str, Any]) -> Self: if not isinstance(data, dict): # pyright: ignore[reportUnnecessaryIsInstance] raise TypeError("Project data is not a dictionary") - return cls(**data) + return cls.model_validate(data) @classmethod def from_yaml_file(cls, path: pathlib.Path) -> Self: diff --git a/craft_application/models/constraints.py b/craft_application/models/constraints.py index 2dd429b5..6bef5c71 100644 --- a/craft_application/models/constraints.py +++ b/craft_application/models/constraints.py @@ -14,78 +14,182 @@ # You should have received a copy of the GNU Lesser General Public License along # with this program. If not, see . """Constrained pydantic types for *craft applications.""" +import collections import re +from collections.abc import Callable +from typing import Annotated, TypeVar -from pydantic import ConstrainedList, ConstrainedStr, StrictStr +import pydantic +T = TypeVar("T") +Tv = TypeVar("Tv") -class ProjectName(ConstrainedStr): - """A constrained string for describing a project name. - Project name rules: - * Valid characters are lower-case ASCII letters, numerals and hyphens. - * Must contain at least one letter - * May not start or end with a hyphen - * May not have two hyphens in a row - """ +def _validate_list_is_unique(value: list[T]) -> list[T]: + value_set = set(value) + if len(value_set) == len(value): + return value + dupes = [item for item, count in collections.Counter(value).items() if count > 1] + raise ValueError(f"duplicate values in list: {dupes}") + + +def _get_validator_by_regex( + regex: re.Pattern[str], error_msg: str +) -> Callable[[str], str]: + """Get a string validator by regular expression with a known error message. - min_length = 1 - max_length = 40 - strict = True - strip_whitespace = True - regex = re.compile(r"^([a-z0-9][a-z0-9-]?)*[a-z]+([a-z0-9-]?[a-z0-9])*$") + This allows providing better error messages for regex-based validation than the + standard message provided by pydantic. Simply place the result of this function in + a BeforeValidator attached to your annotated type. + :param regex: a compiled regular expression on a string. + :param error_msg: The error message to raise if the value is invalid. + :returns: A validator function ready to be used by pydantic.BeforeValidator + """ + def validate(value: str) -> str: + """Validate the given string with the outer regex, raising the error message. + + :param value: a string to be validated + :returns: that same string if it's valid. + :raises: ValueError if the string is invalid. + """ + value = str(value) + if not regex.match(value): + raise ValueError(error_msg) + return value + + return validate + + +UniqueList = Annotated[ + list[T], + pydantic.AfterValidator(_validate_list_is_unique), + pydantic.Field(json_schema_extra={"uniqueItems": True}), +] + +SingleEntryList = Annotated[ + list[T], + pydantic.Field(min_length=1, max_length=1), +] + +SingleEntryDict = Annotated[ + dict[T, Tv], + pydantic.Field(min_length=1, max_length=1), +] + +_PROJECT_NAME_DESCRIPTION = """\ +The name of this project. This is used when uploading, publishing, or installing. + +Project name rules: +* Valid characters are lower-case ASCII letters, numerals and hyphens. +* Must contain at least one letter +* May not start or end with a hyphen +* May not have two hyphens in a row +""" + +_PROJECT_NAME_REGEX = r"^([a-z0-9][a-z0-9-]?)*[a-z]+([a-z0-9-]?[a-z0-9])*$" +_PROJECT_NAME_COMPILED_REGEX = re.compile(_PROJECT_NAME_REGEX) MESSAGE_INVALID_NAME = ( "invalid name: Names can only use ASCII lowercase letters, numbers, and hyphens. " "They must have at least one letter, may not start or end with a hyphen, " "and may not have two hyphens in a row." ) - -class ProjectTitle(StrictStr): - """A constrained string for describing a project title.""" - - min_length = 2 - max_length = 40 - strip_whitespace = True - - -class SummaryStr(ConstrainedStr): - """A constrained string for a short summary of a project.""" - - strip_whitespace = True - max_length = 78 - - -class UniqueStrList(ConstrainedList): - """A list of strings, each of which must be unique. - - This is roughly equivalent to an ordered set of strings, but implemented with a list. - """ - - __args__ = (str,) - item_type = str - unique_items = True - - -class VersionStr(ConstrainedStr): - """A valid version string. - - Should match snapd valid versions: - https://github.com/snapcore/snapd/blame/a39482ead58bf06cddbc0d3ffad3c17dfcf39913/snap/validate.go#L96 - Applications may use a different set of constraints if necessary, but - ideally they will retain this same constraint. - """ - - max_length = 32 - strip_whitespace = True - regex = re.compile(r"^[a-zA-Z0-9](?:[a-zA-Z0-9:.+~-]*[a-zA-Z0-9+~])?$") - - +ProjectName = Annotated[ + str, + pydantic.BeforeValidator( + _get_validator_by_regex(_PROJECT_NAME_COMPILED_REGEX, MESSAGE_INVALID_NAME) + ), + pydantic.Field( + min_length=1, + max_length=40, + strict=True, + pattern=_PROJECT_NAME_REGEX, + description=_PROJECT_NAME_DESCRIPTION, + title="Project Name", + examples=[ + "ubuntu", + "jupyterlab-desktop", + "lxd", + "digikam", + "kafka", + "mysql-router-k8s", + ], + ), +] + + +ProjectTitle = Annotated[ + str, + pydantic.Field( + min_length=2, + max_length=40, + title="Title", + description="A human-readable title.", + examples=[ + "Ubuntu Linux", + "Jupyter Lab Desktop", + "LXD", + "DigiKam", + "Apache Kafka", + "MySQL Router K8s charm", + ], + ), +] + +SummaryStr = Annotated[ + str, + pydantic.Field( + max_length=78, + title="Summary", + description="A short description of your project.", + examples=[ + "Linux for Human Beings", + "The cross-platform desktop application for JupyterLab", + "Container and VM manager", + "Photo Management Program", + "Charm for routing MySQL databases in Kubernetes", + "An open-source event streaming platform for high-performance data pipelines", + ], + ), +] + +UniqueStrList = UniqueList[str] + +_VERSION_STR_REGEX = r"^[a-zA-Z0-9](?:[a-zA-Z0-9:.+~-]*[a-zA-Z0-9+~])?$" +_VERSION_STR_COMPILED_REGEX = re.compile(_VERSION_STR_REGEX) MESSAGE_INVALID_VERSION = ( "invalid version: Valid versions consist of upper- and lower-case " "alphanumeric characters, as well as periods, colons, plus signs, tildes, " "and hyphens. They cannot begin with a period, colon, plus sign, tilde, or " "hyphen. They cannot end with a period, colon, or hyphen." ) + +VersionStr = Annotated[ + str, + pydantic.BeforeValidator( + _get_validator_by_regex(_VERSION_STR_COMPILED_REGEX, MESSAGE_INVALID_VERSION) + ), + pydantic.Field( + max_length=32, + pattern=_VERSION_STR_REGEX, + strict=False, + coerce_numbers_to_str=True, + title="version string", + description="A string containing the version of the project", + examples=[ + "0.1", + "1.0.0", + "v1.0.0", + "24.04", + ], + ), +] +"""A valid version string. + +Should match snapd valid versions: +https://github.com/snapcore/snapd/blame/a39482ead58bf06cddbc0d3ffad3c17dfcf39913/snap/validate.go#L96 +Applications may use a different set of constraints if necessary, but +ideally they will retain this same constraint. +""" diff --git a/craft_application/models/grammar.py b/craft_application/models/grammar.py index d13f8f0e..794d0f8f 100644 --- a/craft_application/models/grammar.py +++ b/craft_application/models/grammar.py @@ -18,65 +18,62 @@ from typing import Any import pydantic -from craft_grammar.models import ( # type: ignore[import-untyped] - GrammarBool, - GrammarDict, - GrammarDictList, - GrammarInt, - GrammarSingleEntryDictList, - GrammarStr, - GrammarStrList, -) +from craft_grammar.models import Grammar # type: ignore[import-untyped] +from pydantic import ConfigDict from craft_application.models.base import alias_generator +from craft_application.models.constraints import SingleEntryDict class _GrammarAwareModel(pydantic.BaseModel): - class Config: - """Default configuration for grammar-aware models.""" - - validate_assignment = True - extra = pydantic.Extra.allow # verify only grammar-aware parts - alias_generator = alias_generator - allow_population_by_field_name = True + model_config = ConfigDict( + validate_assignment=True, + extra="allow", + alias_generator=alias_generator, + populate_by_name=True, + ) class _GrammarAwarePart(_GrammarAwareModel): - plugin: GrammarStr | None - source: GrammarStr | None - source_checksum: GrammarStr | None - source_branch: GrammarStr | None - source_commit: GrammarStr | None - source_depth: GrammarInt | None - source_subdir: GrammarStr | None - source_submodules: GrammarStrList | None - source_tag: GrammarStr | None - source_type: GrammarStr | None - disable_parallel: GrammarBool | None - after: GrammarStrList | None - overlay_packages: GrammarStrList | None - stage_snaps: GrammarStrList | None - stage_packages: GrammarStrList | None - build_snaps: GrammarStrList | None - build_packages: GrammarStrList | None - build_environment: GrammarSingleEntryDictList | None - build_attributes: GrammarStrList | None - organize_files: GrammarDict | None = pydantic.Field(alias="organize") - overlay_files: GrammarStrList | None = pydantic.Field(alias="overlay") - stage_files: GrammarStrList | None = pydantic.Field(alias="stage") - prime_files: GrammarStrList | None = pydantic.Field(alias="prime") - override_pull: GrammarStr | None - overlay_script: GrammarStr | None - override_build: GrammarStr | None - override_stage: GrammarStr | None - override_prime: GrammarStr | None - permissions: GrammarDictList | None - parse_info: GrammarStrList | None + plugin: Grammar[str] | None = None + source: Grammar[str] | None = None + source_checksum: Grammar[str] | None = None + source_branch: Grammar[str] | None = None + source_commit: Grammar[str] | None = None + source_depth: Grammar[int] | None = None + source_subdir: Grammar[str] | None = None + source_submodules: Grammar[list[str]] | None = None + source_tag: Grammar[str] | None = None + source_type: Grammar[str] | None = None + disable_parallel: Grammar[bool] | None = None + after: Grammar[list[str]] | None = None + overlay_packages: Grammar[list[str]] | None = None + stage_snaps: Grammar[list[str]] | None = None + stage_packages: Grammar[list[str]] | None = None + build_snaps: Grammar[list[str]] | None = None + build_packages: Grammar[list[str]] | None = None + build_environment: Grammar[list[SingleEntryDict[str, str]]] | None = None + build_attributes: Grammar[list[str]] | None = None + organize_files: Grammar[dict[str, str]] | None = pydantic.Field( + default=None, alias="organize" + ) + overlay_files: Grammar[list[str]] | None = pydantic.Field(None, alias="overlay") + stage_files: Grammar[list[str]] | None = pydantic.Field(None, alias="stage") + prime_files: Grammar[list[str]] | None = pydantic.Field(None, alias="prime") + override_pull: Grammar[str] | None = None + overlay_script: Grammar[str] | None = None + override_build: Grammar[str] | None = None + override_stage: Grammar[str] | None = None + override_prime: Grammar[str] | None = None + permissions: Grammar[list[dict[str, int | str]]] | None = None + parse_info: Grammar[list[str]] | None = None def get_grammar_aware_part_keywords() -> list[str]: """Return all supported grammar keywords for a part.""" - keywords: list[str] = [item.alias for item in _GrammarAwarePart.__fields__.values()] + keywords: list[str] = [ + item.alias or name for name, item in _GrammarAwarePart.model_fields.items() + ] return keywords @@ -85,9 +82,8 @@ class GrammarAwareProject(_GrammarAwareModel): parts: "dict[str, _GrammarAwarePart]" - @pydantic.root_validator( # pyright: ignore[reportUntypedFunctionDecorator,reportUnknownMemberType] - pre=True - ) + @pydantic.model_validator(mode="before") + @classmethod def _ensure_parts(cls, data: dict[str, Any]) -> dict[str, Any]: """Ensure that the "parts" dictionary exists. @@ -101,4 +97,4 @@ def _ensure_parts(cls, data: dict[str, Any]) -> dict[str, Any]: @classmethod def validate_grammar(cls, data: dict[str, Any]) -> None: """Ensure grammar-enabled entries are syntactically valid.""" - cls(**data) + cls.model_validate(data) diff --git a/craft_application/models/metadata.py b/craft_application/models/metadata.py index 20abf84a..8a01f7b0 100644 --- a/craft_application/models/metadata.py +++ b/craft_application/models/metadata.py @@ -15,20 +15,20 @@ # with this program. If not, see . """Base project metadata model.""" import pydantic -from typing_extensions import override -from craft_application.models.base import CraftBaseConfig, CraftBaseModel +from craft_application.models import base -class BaseMetadata(CraftBaseModel): +class BaseMetadata(base.CraftBaseModel): """Project metadata base model. This model is the basis for output metadata files that are stored in the application's output. """ - @override - class Config(CraftBaseConfig): - """Allows writing of unknown fields.""" - - extra = pydantic.Extra.allow + model_config = pydantic.ConfigDict( + validate_assignment=True, + extra="allow", + populate_by_name=True, + alias_generator=base.alias_generator, + ) diff --git a/craft_application/models/project.py b/craft_application/models/project.py index bfbb3de0..e869a460 100644 --- a/craft_application/models/project.py +++ b/craft_application/models/project.py @@ -20,7 +20,7 @@ import abc import dataclasses from collections.abc import Mapping -from typing import Any +from typing import Annotated, Any import craft_parts import craft_providers.bases @@ -28,17 +28,18 @@ from craft_cli import emit from craft_providers import bases from craft_providers.errors import BaseConfigurationError -from pydantic import AnyUrl from typing_extensions import override from craft_application import errors -from craft_application.models.base import CraftBaseConfig, CraftBaseModel +from craft_application.models import base from craft_application.models.constraints import ( MESSAGE_INVALID_NAME, MESSAGE_INVALID_VERSION, ProjectName, ProjectTitle, + SingleEntryList, SummaryStr, + UniqueList, UniqueStrList, VersionStr, ) @@ -89,24 +90,14 @@ class BuildInfo: """The base to build on.""" -class BuildPlannerConfig(CraftBaseConfig): - """Config for BuildProjects.""" - - extra = pydantic.Extra.ignore - """The BuildPlanner model uses attributes from the project yaml.""" - - -class Platform(CraftBaseModel): +class Platform(base.CraftBaseModel): """Project platform definition.""" - build_on: list[str] | None = pydantic.Field(min_items=1, unique_items=True) - build_for: list[str] | None = pydantic.Field( - min_items=1, max_items=1, unique_items=True - ) + build_on: UniqueList[str] | None = pydantic.Field(min_length=1) + build_for: SingleEntryList[str] | None = None - @pydantic.validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - "build_on", "build_for" - ) + @pydantic.field_validator("build_on", "build_for", mode="after") + @classmethod def _validate_architectures(cls, values: list[str]) -> list[str]: """Validate the architecture entries.""" for architecture in values: @@ -118,15 +109,15 @@ def _validate_architectures(cls, values: list[str]) -> list[str]: return values - @pydantic.root_validator( # pyright: ignore[reportUntypedFunctionDecorator,reportUnknownMemberType] - skip_on_failure=True - ) + @pydantic.model_validator(mode="before") @classmethod - def _validate_platform_set(cls, values: Mapping[str, Any]) -> Mapping[str, Any]: + def _validate_platform_set( + cls, values: Mapping[str, list[str]] + ) -> Mapping[str, Any]: """If build_for is provided, then build_on must also be.""" - if not values.get("build_on") and values.get("build_for"): + if values.get("build_for") and not values.get("build_on"): raise errors.CraftValidationError( - "'build_for' expects 'build_on' to also be provided." + "'build-for' expects 'build-on' to also be provided." ) return values @@ -149,25 +140,28 @@ def _populate_platforms(platforms: dict[str, Platform]) -> dict[str, Platform]: return platforms -class BuildPlanner(CraftBaseModel, metaclass=abc.ABCMeta): +class BuildPlanner(base.CraftBaseModel, metaclass=abc.ABCMeta): """The BuildPlanner obtains a build plan for the project.""" - platforms: dict[str, Platform] - base: str | None - build_base: str | None + model_config = pydantic.ConfigDict( + validate_assignment=True, + extra="ignore", + populate_by_name=True, + alias_generator=base.alias_generator, + ) - Config = BuildPlannerConfig + platforms: dict[str, Platform] + base: str | None = None + build_base: str | None = None - @pydantic.validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - "platforms", pre=True - ) + @pydantic.field_validator("platforms", mode="before") + @classmethod def _populate_platforms(cls, platforms: dict[str, Platform]) -> dict[str, Platform]: """Populate empty platform entries.""" return _populate_platforms(platforms) - @pydantic.validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - "platforms", - ) + @pydantic.field_validator("platforms", mode="after") + @classmethod def _validate_platforms_all_keyword( cls, platforms: dict[str, Any] ) -> dict[str, Any]: @@ -231,41 +225,61 @@ def get_build_plan(self) -> list[BuildInfo]: return build_infos -class Project(CraftBaseModel): +def _validate_package_repository(repository: dict[str, Any]) -> dict[str, Any]: + """Validate a package repository with lazy loading of craft-archives. + + :param repository: a dictionary representing a package repository. + :returns: That same dictionary, if valid. + :raises: ValueError if the repository is not valid. + """ + # This check is not always used, import it here to avoid unnecessary + from craft_archives import repo # type: ignore[import-untyped] + + repo.validate_repository(repository) + return repository + + +class Project(base.CraftBaseModel): """Craft Application project definition.""" name: ProjectName - title: ProjectTitle | None - version: VersionStr | None - summary: SummaryStr | None - description: str | None + title: ProjectTitle | None = None + version: VersionStr | None = None + summary: SummaryStr | None = None + description: str | None = None - base: Any | None = None - build_base: Any | None = None + base: str | None = None + build_base: str | None = None platforms: dict[str, Platform] - contact: str | UniqueStrList | None - issues: str | UniqueStrList | None - source_code: AnyUrl | None - license: str | None + contact: str | UniqueStrList | None = None + issues: str | UniqueStrList | None = None + source_code: pydantic.AnyUrl | None = None + license: str | None = None - adopt_info: str | None + adopt_info: str | None = None parts: dict[str, dict[str, Any]] # parts are handled by craft-parts - package_repositories: list[dict[str, Any]] | None + package_repositories: ( + list[ + Annotated[ + dict[str, Any], pydantic.AfterValidator(_validate_package_repository) + ] + ] + | None + ) = None - @pydantic.validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - "parts", each_item=True - ) - def _validate_parts(cls, item: dict[str, Any]) -> dict[str, Any]: + @pydantic.field_validator("parts", mode="before") + @classmethod + def _validate_parts(cls, parts: dict[str, Any]) -> dict[str, Any]: """Verify each part (craft-parts will re-validate this).""" - craft_parts.validate_part(item) - return item + for part in parts.values(): + craft_parts.validate_part(part) + return parts - @pydantic.validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - "platforms", pre=True - ) + @pydantic.field_validator("platforms", mode="before") + @classmethod def _populate_platforms(cls, platforms: dict[str, Platform]) -> dict[str, Platform]: """Populate empty platform entries.""" return _populate_platforms(platforms) @@ -300,24 +314,24 @@ def _providers_base(cls, base: str) -> craft_providers.bases.BaseAlias | None: except (ValueError, BaseConfigurationError) as err: raise ValueError(f"Unknown base {base!r}") from err - @pydantic.root_validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - pre=False - ) - def _validate_devel(cls, values: dict[str, Any]) -> dict[str, Any]: + @pydantic.field_validator("build_base", mode="before") + @classmethod + def _validate_devel_base( + cls, build_base: str, info: pydantic.ValidationInfo + ) -> str: """Validate the build-base is 'devel' for the current devel base.""" - base = values.get("base") + base = info.data.get("base") # if there is no base, do not validate the build-base if not base: - return values + return build_base base_alias = cls._providers_base(base) # if the base does not map to a base alias, do not validate the build-base if not base_alias: - return values + return build_base - build_base = values.get("build_base") or base - build_base_alias = cls._providers_base(build_base) + build_base_alias = cls._providers_base(build_base or base) # warn if a devel build-base is being used, error if a devel build-base is not # used for a devel base @@ -330,7 +344,7 @@ def _validate_devel(cls, values: dict[str, Any]) -> dict[str, Any]: f"A development build-base must be used when base is {base!r}" ) - return values + return build_base @override @classmethod @@ -340,7 +354,7 @@ def transform_pydantic_error(cls, error: pydantic.ValidationError) -> None: ("name", "value_error.str.regex"): MESSAGE_INVALID_NAME, } - CraftBaseModel.transform_pydantic_error(error) + base.CraftBaseModel.transform_pydantic_error(error) for error_dict in error.errors(): loc_and_type = (str(error_dict["loc"][0]), error_dict["type"]) @@ -349,16 +363,3 @@ def transform_pydantic_error(cls, error: pydantic.ValidationError) -> None: # "input" key in the error dict, so we can't put the original # value in the error message. error_dict["msg"] = message - - @pydantic.validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - "package_repositories", each_item=True - ) - def _validate_package_repositories( - cls, repository: dict[str, Any] - ) -> dict[str, Any]: - # This check is not always used, import it here to avoid unnecessary - from craft_archives import repo # type: ignore[import-untyped] - - repo.validate_repository(repository) - - return repository diff --git a/craft_application/services/lifecycle.py b/craft_application/services/lifecycle.py index 3fd70ca8..b1498a9a 100644 --- a/craft_application/services/lifecycle.py +++ b/craft_application/services/lifecycle.py @@ -38,7 +38,7 @@ from craft_application import errors, models, util from craft_application.services import base -from craft_application.util import convert_architecture_deb_to_platform, repositories +from craft_application.util import repositories if TYPE_CHECKING: # pragma: no cover from pathlib import Path @@ -187,7 +187,7 @@ def _init_lifecycle_manager(self) -> LifecycleManager: return LifecycleManager( {"parts": self._project.parts}, application_name=self._app.name, - arch=convert_architecture_deb_to_platform(build_for), + arch=build_for, cache_dir=self._cache_dir, work_dir=self._work_dir, ignore_local_sources=self._app.source_ignore_patterns, diff --git a/craft_application/util/error_formatting.py b/craft_application/util/error_formatting.py index 4864a274..8f878013 100644 --- a/craft_application/util/error_formatting.py +++ b/craft_application/util/error_formatting.py @@ -14,11 +14,12 @@ # You should have received a copy of the GNU Lesser General Public License along # with this program. If not, see . """Helper utilities for formatting error messages.""" +from __future__ import annotations + from collections.abc import Iterable -from typing import TYPE_CHECKING, NamedTuple +from typing import NamedTuple -if TYPE_CHECKING: # pragma: no cover - from pydantic.error_wrappers import ErrorDict +from pydantic import error_wrappers class FieldLocationTuple(NamedTuple): @@ -28,7 +29,7 @@ class FieldLocationTuple(NamedTuple): location: str = "top-level" @classmethod - def from_str(cls, loc_str: str) -> "FieldLocationTuple": + def from_str(cls, loc_str: str) -> FieldLocationTuple: """Return split field location. If top-level, location is returned as unquoted "top-level". @@ -70,7 +71,7 @@ def format_pydantic_error(loc: Iterable[str | int], message: str) -> str: def format_pydantic_errors( - errors: "Iterable[ErrorDict]", *, file_name: str = "yaml file" + errors: Iterable[error_wrappers.ErrorDict], *, file_name: str = "yaml file" ) -> str: """Format errors. diff --git a/craft_application/util/snap_config.py b/craft_application/util/snap_config.py index ca36301d..98cb50d9 100644 --- a/craft_application/util/snap_config.py +++ b/craft_application/util/snap_config.py @@ -39,7 +39,7 @@ def is_running_from_snap(app_name: str) -> bool: return os.getenv("SNAP_NAME") == app_name and os.getenv("SNAP") is not None -class SnapConfig(pydantic.BaseModel, extra=pydantic.Extra.forbid): +class SnapConfig(pydantic.BaseModel, extra="forbid"): """Data stored in a snap config. :param provider: provider to use. Valid values are 'lxd' and 'multipass'. @@ -47,9 +47,7 @@ class SnapConfig(pydantic.BaseModel, extra=pydantic.Extra.forbid): provider: Literal["lxd", "multipass"] | None = None - @pydantic.validator( # pyright: ignore[reportUnknownMemberType,reportUntypedFunctionDecorator] - "provider", pre=True - ) + @pydantic.field_validator("provider", mode="before") @classmethod def normalize(cls, provider: str) -> str: """Normalize provider name.""" diff --git a/pyproject.toml b/pyproject.toml index 3b4289ac..f5043ca9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,15 +3,15 @@ name = "craft-application" description = "A framework for *craft applications." dynamic = ["version", "readme"] dependencies = [ - "craft-archives>=1.1.3", + "craft-archives@git+https://github.com/canonical/craft-archives@feature/2.0", "craft-cli>=2.6.0", - "craft-grammar>=1.2.0", - "craft-parts>=1.33.0", - "craft-providers>=1.20.0,<2.0", + "craft-grammar@git+https://github.com/canonical/craft-grammar@feature/pydantic-2", + "craft-parts@git+https://github.com/canonical/craft-parts@feature/2.0", + "craft-providers@git+https://github.com/canonical/craft-providers@feature/2.0-git-modules", "snap-helpers>=0.4.2", "platformdirs>=3.10", - "pydantic>=1.10,<2.0", - "pydantic-yaml<1.0", + "pydantic~=2.0", + # "pydantic-yaml<1.0", # Pygit2 and libgit2 need to match versions. # Further info: https://www.pygit2.org/install.html#version-numbers # Minor versions of pygit2 can include breaking changes, so we need to check @@ -88,6 +88,11 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool] +rye = { dev-dependencies = [ + "pytest-xdist>=3.6.1", +] } + [tool.setuptools.dynamic] readme = {file = "README.rst"} @@ -154,8 +159,6 @@ exclude = ["craft_application/_version.py"] strict = ["craft_application"] pythonVersion = "3.10" pythonPlatform = "Linux" -venvPath = ".tox" -venv = "typing" [tool.mypy] python_version = "3.10" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a4e9f7e3..a6ae6c48 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,7 +23,7 @@ import pytest from craft_application import launchpad from craft_application.services import provider, remotebuild -from craft_providers import lxd, multipass +from craft_providers import bases, lxd, multipass def pytest_configure(config: pytest.Config): @@ -85,3 +85,10 @@ def snap_safe_tmp_path(): dir=directory or pathlib.Path.home(), ) as temp_dir: yield pathlib.Path(temp_dir) + + +@pytest.fixture() +def pretend_jammy(mocker) -> None: + """Pretend we're running on jammy. Used for tests that use destructive mode.""" + fake_host = bases.BaseName(name="ubuntu", version="22.04") + mocker.patch("craft_application.util.get_host_base", return_value=fake_host) diff --git a/tests/integration/services/test_lifecycle.py b/tests/integration/services/test_lifecycle.py index 628aa976..c14455d3 100644 --- a/tests/integration/services/test_lifecycle.py +++ b/tests/integration/services/test_lifecycle.py @@ -18,8 +18,6 @@ import textwrap import craft_cli -import craft_parts -import craft_parts.overlays import pytest import pytest_check from craft_application.services.lifecycle import LifecycleService @@ -110,9 +108,9 @@ def test_package_repositories_in_overlay( # Mock overlay-related calls that need root; we won't be actually installing # any packages, just checking that the repositories are correctly installed # in the overlay. - mocker.patch.object(craft_parts.overlays.OverlayManager, "refresh_packages_list") # type: ignore[reportPrivateImportUsage] - mocker.patch.object(craft_parts.overlays.OverlayManager, "download_packages") # type: ignore[reportPrivateImportUsage] - mocker.patch.object(craft_parts.overlays.OverlayManager, "install_packages") # type: ignore[reportPrivateImportUsage] + mocker.patch("craft_parts.overlays.OverlayManager.refresh_packages_list") + mocker.patch("craft_parts.overlays.OverlayManager.download_packages") + mocker.patch("craft_parts.overlays.OverlayManager.install_packages") mocker.patch.object(os, "geteuid", return_value=0) parts = { @@ -165,6 +163,8 @@ def test_package_repositories_in_overlay( assert overlay_apt.is_dir() # Checking that the files are present should be enough - assert (overlay_apt / "keyrings/craft-CE49EC21.gpg").is_file() - assert (overlay_apt / "sources.list.d/craft-ppa-mozillateam_ppa.sources").is_file() - assert (overlay_apt / "preferences.d/craft-archives").is_file() + pytest_check.is_true((overlay_apt / "keyrings/craft-9BE21867.gpg").is_file()) + pytest_check.is_true( + (overlay_apt / "sources.list.d/craft-ppa-mozillateam_ppa.sources").is_file() + ) + pytest_check.is_true((overlay_apt / "preferences.d/craft-archives").is_file()) diff --git a/tests/integration/services/test_provider.py b/tests/integration/services/test_provider.py index 0f77c28d..68f5c85f 100644 --- a/tests/integration/services/test_provider.py +++ b/tests/integration/services/test_provider.py @@ -1,6 +1,6 @@ # This file is part of craft-application. # -# Copyright 2023 Canonical Ltd. +# Copyright 2023-2024 Canonical Ltd. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License version 3, as @@ -27,9 +27,13 @@ @pytest.mark.parametrize( "base_name", [ - # Skipping Oracular test for now; see - # https://github.com/canonical/craft-providers/issues/593 - pytest.param(("ubuntu", "24.10"), id="ubuntu_latest", marks=pytest.mark.skip), + pytest.param( + ("ubuntu", "24.10"), + id="ubuntu_latest", + marks=pytest.mark.skip( + reason="Skipping Oracular test for now; see https://github.com/canonical/craft-providers/issues/593" + ), + ), pytest.param(("ubuntu", "24.04"), id="ubuntu_lts"), pytest.param(("ubuntu", "22.04"), id="ubuntu_old_lts"), ], diff --git a/tests/integration/test_application.py b/tests/integration/test_application.py index f84893d8..7da91574 100644 --- a/tests/integration/test_application.py +++ b/tests/integration/test_application.py @@ -123,6 +123,7 @@ def test_special_inputs(capsys, monkeypatch, app, argv, stdout, stderr, exit_cod pytest_check.equal(captured.err, stderr, "stderr does not match") +@pytest.mark.usefixtures("pretend_jammy") @pytest.mark.parametrize("project", (d.name for d in VALID_PROJECTS_DIR.iterdir())) def test_project_managed(capsys, monkeypatch, tmp_path, project, create_app): monkeypatch.setenv("CRAFT_DEBUG", "1") @@ -144,7 +145,7 @@ def test_project_managed(capsys, monkeypatch, tmp_path, project, create_app): ) -@pytest.mark.usefixtures("full_build_plan") +@pytest.mark.usefixtures("full_build_plan", "pretend_jammy") @pytest.mark.parametrize("project", (d.name for d in VALID_PROJECTS_DIR.iterdir())) def test_project_destructive( capsys, @@ -365,6 +366,7 @@ def _inner(): return _inner +@pytest.mark.usefixtures("pretend_jammy") @pytest.mark.enable_features("build_secrets") def test_build_secrets_destructive( monkeypatch, setup_secrets_project, check_secrets_output @@ -381,6 +383,7 @@ def test_build_secrets_destructive( check_secrets_output() +@pytest.mark.usefixtures("pretend_jammy") @pytest.mark.enable_features("build_secrets") def test_build_secrets_managed( monkeypatch, tmp_path, setup_secrets_project, check_secrets_output @@ -406,6 +409,7 @@ def test_build_secrets_managed( check_secrets_output() +@pytest.mark.usefixtures("pretend_jammy") def test_lifecycle_error_logging(monkeypatch, tmp_path, create_app): monkeypatch.chdir(tmp_path) shutil.copytree(INVALID_PROJECTS_DIR / "build-error", tmp_path, dirs_exist_ok=True) @@ -424,6 +428,7 @@ def test_lifecycle_error_logging(monkeypatch, tmp_path, create_app): assert parts_message in log_contents +@pytest.mark.usefixtures("pretend_jammy") def test_runtime_error_logging(monkeypatch, tmp_path, create_app, mocker): monkeypatch.chdir(tmp_path) shutil.copytree(INVALID_PROJECTS_DIR / "build-error", tmp_path, dirs_exist_ok=True) @@ -437,6 +442,7 @@ def test_runtime_error_logging(monkeypatch, tmp_path, create_app, mocker): monkeypatch.setattr("sys.argv", ["testcraft", "pack", "--destructive-mode"]) app = create_app() + app.run() log_contents = craft_cli.emit._log_filepath.read_text() diff --git a/tests/unit/commands/test_lifecycle.py b/tests/unit/commands/test_lifecycle.py index 4800cdea..7c08030c 100644 --- a/tests/unit/commands/test_lifecycle.py +++ b/tests/unit/commands/test_lifecycle.py @@ -52,15 +52,16 @@ ({"destructive_mode": False, "use_lxd": True}, ["--use-lxd"]), ] STEP_NAMES = [step.name.lower() for step in craft_parts.Step] -MANAGED_LIFECYCLE_COMMANDS = { +MANAGED_LIFECYCLE_COMMANDS = ( PullCommand, OverlayCommand, BuildCommand, StageCommand, PrimeCommand, -} -UNMANAGED_LIFECYCLE_COMMANDS = {CleanCommand, PackCommand} -ALL_LIFECYCLE_COMMANDS = MANAGED_LIFECYCLE_COMMANDS | UNMANAGED_LIFECYCLE_COMMANDS +) +UNMANAGED_LIFECYCLE_COMMANDS = (CleanCommand, PackCommand) +ALL_LIFECYCLE_COMMANDS = MANAGED_LIFECYCLE_COMMANDS + UNMANAGED_LIFECYCLE_COMMANDS +NON_CLEAN_COMMANDS = (*MANAGED_LIFECYCLE_COMMANDS, PackCommand) def get_fake_command_class(parent_cls, managed): @@ -101,7 +102,7 @@ def test_get_lifecycle_command_group(enable_overlay, commands): actual = get_lifecycle_command_group() - assert set(actual.commands) == commands + assert set(actual.commands) == set(commands) Features.reset() @@ -170,7 +171,7 @@ def test_parts_command_get_managed_cmd( ) @pytest.mark.parametrize("parts", PARTS_LISTS) # clean command has different logic for `run_managed()` -@pytest.mark.parametrize("command_cls", ALL_LIFECYCLE_COMMANDS - {CleanCommand}) +@pytest.mark.parametrize("command_cls", NON_CLEAN_COMMANDS) def test_parts_command_run_managed( app_metadata, mock_services, @@ -553,7 +554,7 @@ def test_shell_after( mock_subprocess_run.assert_called_once_with(["bash"], check=False) -@pytest.mark.parametrize("command_cls", MANAGED_LIFECYCLE_COMMANDS | {PackCommand}) +@pytest.mark.parametrize("command_cls", [*MANAGED_LIFECYCLE_COMMANDS, PackCommand]) def test_debug(app_metadata, fake_services, mocker, mock_subprocess_run, command_cls): parsed_args = argparse.Namespace(parts=None, debug=True) error_message = "Lifecycle run failed!" diff --git a/tests/unit/models/test_base.py b/tests/unit/models/test_base.py index 126d8889..a65ecd51 100644 --- a/tests/unit/models/test_base.py +++ b/tests/unit/models/test_base.py @@ -27,11 +27,13 @@ class MyBaseModel(models.CraftBaseModel): value1: int value2: str - @pydantic.validator("value1") + @pydantic.field_validator("value1", mode="after") + @classmethod def _validate_value1(cls, _v): raise ValueError("Bad value1 value") - @pydantic.validator("value2") + @pydantic.field_validator("value2", mode="after") + @classmethod def _validate_value2(cls, _v): raise ValueError("Bad value2 value") @@ -51,8 +53,8 @@ def test_model_reference_slug_errors(): expected = ( "Bad testcraft.yaml content:\n" - "- bad value1 value (in field 'value1')\n" - "- bad value2 value (in field 'value2')" + "- value error, Bad value1 value (in field 'value1')\n" + "- value error, Bad value2 value (in field 'value2')" ) assert str(err.value) == expected assert err.value.doc_slug == "/mymodel.html" diff --git a/tests/unit/models/test_constraints.py b/tests/unit/models/test_constraints.py index 87d3cbbf..e8300fe2 100644 --- a/tests/unit/models/test_constraints.py +++ b/tests/unit/models/test_constraints.py @@ -19,6 +19,7 @@ import pydantic.errors import pytest +from craft_application.models import constraints from craft_application.models.constraints import ProjectName, VersionStr from hypothesis import given, strategies @@ -74,13 +75,58 @@ def string_or_unique_list(): ) +# endregion +# region Unique list values tests +@given( + strategies.sets( + strategies.one_of( + strategies.none(), + strategies.integers(), + strategies.floats(), + strategies.text(), + ) + ) +) +def test_validate_list_is_unique_hypothesis_success(values: set): + values_list = list(values) + constraints._validate_list_is_unique(values_list) + + +@pytest.mark.parametrize( + "values", [[], [None], [None, 0], [None, 0, ""], [True, 2, "True", "2", "two"]] +) +def test_validate_list_is_unique_success(values: list): + constraints._validate_list_is_unique(values) + + +@pytest.mark.parametrize( + ("values", "expected_dupes_text"), + [ + ([None, None], "[None]"), + ([0, 0], "[0]"), + ([1, True], "[1]"), + ([True, 1], "[True]"), + (["this", "that", "this"], "['this']"), + ], +) +def test_validate_list_is_unique_with_duplicates(values, expected_dupes_text): + with pytest.raises(ValueError, match="^duplicate values in list: ") as exc_info: + constraints._validate_list_is_unique(values) + + assert exc_info.value.args[0].endswith(expected_dupes_text) + + # endregion # region ProjectName tests +class _ProjectNameModel(pydantic.BaseModel): + name: ProjectName + + @given(name=valid_project_name_strategy()) def test_valid_project_name_hypothesis(name): - project_name = ProjectName.validate(name) + project = _ProjectNameModel(name=name) - assert project_name == name + assert project.name == name @pytest.mark.parametrize( @@ -97,9 +143,9 @@ def test_valid_project_name_hypothesis(name): ], ) def test_valid_project_name(name): - project_name = ProjectName.validate(name) + project = _ProjectNameModel(name=name) - assert project_name == name + assert project.name == name @pytest.mark.parametrize( @@ -113,48 +159,52 @@ def test_valid_project_name(name): ], ) def test_invalid_project_name(name): - with pytest.raises(pydantic.PydanticValueError): - ProjectName.validate(name) + with pytest.raises(pydantic.ValidationError): + _ProjectNameModel(name=name) # endregion # region VersionStr tests -@given(version=strategies.integers(min_value=0)) +class _VersionStrModel(pydantic.BaseModel): + version: VersionStr + + +@given(version=strategies.integers(min_value=0, max_value=10**32 - 1)) def test_version_str_hypothesis_integers(version): - version_str = VersionStr(version) - VersionStr.validate(version_str) + version_str = str(version) + _VersionStrModel(version=version_str) assert version_str == str(version) @given(version=strategies.floats(min_value=0.0)) def test_version_str_hypothesis_floats(version): - version_str = VersionStr(version) - VersionStr.validate(version_str) + version_str = str(version) + _VersionStrModel(version=version_str) assert version_str == str(version) @given(version=valid_version_strategy()) def test_version_str_hypothesis(version): - version_str = VersionStr(version) - VersionStr.validate(version) + version_str = str(version) + _VersionStrModel(version=version) assert version_str == str(version) @pytest.mark.parametrize("version", ["0", "1.0", "1.0.0.post10+git12345678"]) def test_valid_version_str(version): - version_str = VersionStr(version) - VersionStr.validate(version) + version_str = str(version) + _VersionStrModel(version=version) assert version_str == str(version) @pytest.mark.parametrize("version", [""]) def test_invalid_version_str(version): - with pytest.raises(pydantic.PydanticValueError): - VersionStr.validate(VersionStr(version)) + with pytest.raises(pydantic.ValidationError): + _VersionStrModel(version=str(version)) # endregion diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index 1c591880..f8fd3a19 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -42,10 +42,10 @@ def basic_project(): # pyright doesn't like these types and doesn't have a pydantic plugin like mypy. # Because of this, we need to silence several errors in these constants. - return Project( # pyright: ignore[reportCallIssue] - name="project-name", # pyright: ignore[reportGeneralTypeIssues] - version="1.0", # pyright: ignore[reportGeneralTypeIssues] - platforms={"arm64": None}, + return Project( + name="project-name", + version="1.0", + platforms={"arm64": None}, # pyright: ignore[reportArgumentType] parts=PARTS_DICT, ) @@ -71,19 +71,21 @@ def basic_project_dict(): @pytest.fixture() def full_project(): - return Project( # pyright: ignore[reportCallIssue] - name="full-project", # pyright: ignore[reportGeneralTypeIssues] - title="A fully-defined project", # pyright: ignore[reportGeneralTypeIssues] - base="ubuntu@24.04", - version="1.0.0.post64+git12345678", # pyright: ignore[reportGeneralTypeIssues] - contact="author@project.org", - issues="https://github.com/canonical/craft-application/issues", - source_code="https://github.com/canonical/craft-application", # pyright: ignore[reportGeneralTypeIssues] - summary="A fully-defined craft-application project.", # pyright: ignore[reportGeneralTypeIssues] - description="A fully-defined craft-application project.\nWith more than one line.\n", - license="LGPLv3", - platforms={"arm64": None}, - parts=PARTS_DICT, + return Project.model_validate( + { + "name": "full-project", + "title": "A fully-defined project", + "base": "ubuntu@24.04", + "version": "1.0.0.post64+git12345678", + "contact": "author@project.org", + "issues": "https://github.com/canonical/craft-application/issues", + "source_code": "https://github.com/canonical/craft-application", + "summary": "A fully-defined craft-application project.", + "description": "A fully-defined craft-application project.\nWith more than one line.\n", + "license": "LGPLv3", + "platforms": {"arm64": None}, + "parts": PARTS_DICT, + } ) @@ -237,7 +239,7 @@ def test_effective_base_is_base(full_project): class FakeBuildBaseProject(Project): - build_base: str | None # pyright: ignore[reportGeneralTypeIssues] + build_base: str | None = None def test_effective_base_is_build_base(): @@ -246,7 +248,7 @@ def test_effective_base_is_build_base(): name="project-name", # pyright: ignore[reportGeneralTypeIssues] version="1.0", # pyright: ignore[reportGeneralTypeIssues] parts={}, - platforms={"arm64": None}, + platforms={"arm64": None}, # pyright: ignore[reportArgumentType] base="ubuntu@22.04", build_base="ubuntu@24.04", ) @@ -255,11 +257,11 @@ def test_effective_base_is_build_base(): def test_effective_base_unknown(): - project = FakeBuildBaseProject( # pyright: ignore[reportCallIssue] - name="project-name", # pyright: ignore[reportGeneralTypeIssues] - version="1.0", # pyright: ignore[reportGeneralTypeIssues] + project = FakeBuildBaseProject( + name="project-name", + version="1.0", parts={}, - platforms={"arm64": None}, + platforms={"arm64": None}, # pyright: ignore[reportArgumentType] base=None, build_base=None, ) @@ -272,11 +274,11 @@ def test_effective_base_unknown(): def test_devel_base_devel_build_base(emitter): """Base can be 'devel' when the build-base is 'devel'.""" - _ = FakeBuildBaseProject( # pyright: ignore[reportCallIssue] - name="project-name", # pyright: ignore[reportGeneralTypeIssues] - version="1.0", # pyright: ignore[reportGeneralTypeIssues] + _ = FakeBuildBaseProject( + name="project-name", + version="1.0", parts={}, - platforms={"arm64": None}, + platforms={"arm64": None}, # pyright: ignore[reportArgumentType] base=f"ubuntu@{DEVEL_BASE_INFOS[0].current_devel_base.value}", build_base=f"ubuntu@{DEVEL_BASE_INFOS[0].current_devel_base.value}", ) @@ -286,11 +288,11 @@ def test_devel_base_devel_build_base(emitter): def test_devel_base_no_base(): """Do not validate the build-base if there is no base.""" - _ = FakeBuildBaseProject( # pyright: ignore[reportCallIssue] - name="project-name", # pyright: ignore[reportGeneralTypeIssues] - version="1.0", # pyright: ignore[reportGeneralTypeIssues] + _ = FakeBuildBaseProject( + name="project-name", + version="1.0", parts={}, - platforms={"arm64": None}, + platforms={"arm64": None}, # pyright: ignore[reportArgumentType] ) @@ -301,22 +303,22 @@ def test_devel_base_no_base_alias(mocker): return_value=None, ) - _ = FakeBuildBaseProject( # pyright: ignore[reportCallIssue] - name="project-name", # pyright: ignore[reportGeneralTypeIssues] - version="1.0", # pyright: ignore[reportGeneralTypeIssues] + _ = FakeBuildBaseProject( + name="project-name", + version="1.0", parts={}, - platforms={"arm64": None}, + platforms={"arm64": None}, # pyright: ignore[reportArgumentType] ) def test_devel_base_no_build_base(): """Base can be 'devel' if the build-base is not set.""" - _ = FakeBuildBaseProject( # pyright: ignore[reportCallIssue] - name="project-name", # pyright: ignore[reportGeneralTypeIssues] - version="1.0", # pyright: ignore[reportGeneralTypeIssues] + _ = FakeBuildBaseProject( + name="project-name", + version="1.0", parts={}, base=f"ubuntu@{DEVEL_BASE_INFOS[0].current_devel_base.value}", - platforms={"arm64": None}, + platforms={"arm64": None}, # pyright: ignore[reportArgumentType] ) @@ -340,7 +342,7 @@ def test_devel_base_error(): dedent( f""" Bad testcraft.yaml content: - - a development build-base must be used when base is 'ubuntu@{expected_devel}' + - value error, A development build-base must be used when base is 'ubuntu@{expected_devel}' """ ).strip() ) @@ -373,7 +375,7 @@ def test_invalid_field_message( full_expected_message = textwrap.dedent( f""" Bad myproject.yaml content: - - {expected_message} (in field '{field_name}') + - value error, {expected_message} (in field '{field_name}') """ ).strip() @@ -412,19 +414,37 @@ def test_unmarshal_undefined_repositories(full_project_dict): assert project.package_repositories is None -def test_unmarshal_invalid_repositories(full_project_dict): +@pytest.mark.parametrize( + ("repositories_val", "error_lines"), + [ + ( + [[]], + [ + "- input should be a valid dictionary (in field 'package-repositories[0]')" + ], + ), + ( + [{}], + [ + "- field 'type' required in 'package-repositories[0]' configuration", + "- field 'url' required in 'package-repositories[0]' configuration", + "- field 'key-id' required in 'package-repositories[0]' configuration", + ], + ), + ], +) +def test_unmarshal_invalid_repositories( + full_project_dict, repositories_val, error_lines +): """Test that package-repositories are validated in Project with package repositories feature.""" - full_project_dict["package-repositories"] = [[]] + full_project_dict["package-repositories"] = repositories_val project_path = pathlib.Path("myproject.yaml") with pytest.raises(CraftValidationError) as error: Project.from_yaml_data(full_project_dict, project_path) - assert error.value.args[0] == ( - "Bad myproject.yaml content:\n" - "- field 'type' required in 'package-repositories[0]' configuration\n" - "- field 'url' required in 'package-repositories[0]' configuration\n" - "- field 'key-id' required in 'package-repositories[0]' configuration" + assert error.value.args[0] == "\n".join( + ("Bad myproject.yaml content:", *error_lines) ) diff --git a/tests/unit/services/test_lifecycle.py b/tests/unit/services/test_lifecycle.py index 410eac13..315c6c97 100644 --- a/tests/unit/services/test_lifecycle.py +++ b/tests/unit/services/test_lifecycle.py @@ -710,7 +710,7 @@ def test_lifecycle_project_variables( """Test that project variables are set after the lifecycle runs.""" class LocalProject(models.Project): - color: str | None + color: str | None = None fake_project = LocalProject.unmarshal( { diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index fbcfbea3..0a85d79e 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -35,6 +35,7 @@ import craft_cli import craft_parts import craft_providers +import pydantic import pytest import pytest_check from craft_application import ( @@ -54,7 +55,6 @@ from craft_parts.plugins.plugins import PluginType from craft_providers import bases from overrides import override -from pydantic import validator EMPTY_COMMAND_GROUP = craft_cli.CommandGroup("FakeCommands", []) BASIC_PROJECT_YAML = """ @@ -670,8 +670,8 @@ def test_gets_project(monkeypatch, tmp_path, app_metadata, fake_services): app.run() - assert fake_services.project is not None - assert app.project is not None + pytest_check.is_not_none(fake_services.project) + pytest_check.is_not_none(app.project) def test_fails_without_project( @@ -2013,11 +2013,13 @@ class MyRaisingPlanner(models.BuildPlanner): value1: int value2: str - @validator("value1") + @pydantic.field_validator("value1", mode="after") + @classmethod def _validate_value1(cls, v): raise ValueError(f"Bad value1: {v}") - @validator("value2") + @pydantic.field_validator("value2", mode="after") + @classmethod def _validate_value(cls, v): raise ValueError(f"Bad value2: {v}") @@ -2037,13 +2039,13 @@ def test_build_planner_errors(tmp_path, monkeypatch, fake_services): app = FakeApplication(app_metadata, fake_services) project_contents = textwrap.dedent( """\ - name: my-project - base: ubuntu@24.04 - value1: 10 - value2: "banana" - platforms: - amd64: - """ + name: my-project + base: ubuntu@24.04 + value1: 10 + value2: "banana" + platforms: + amd64: + """ ).strip() project_path = tmp_path / "testcraft.yaml" project_path.write_text(project_contents) @@ -2053,8 +2055,8 @@ def test_build_planner_errors(tmp_path, monkeypatch, fake_services): expected = ( "Bad testcraft.yaml content:\n" - "- bad value1: 10 (in field 'value1')\n" - "- bad value2: banana (in field 'value2')" + "- value error, Bad value1: 10 (in field 'value1')\n" + "- value error, Bad value2: banana (in field 'value2')" ) assert str(err.value) == expected diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index a6a2a49e..6063863c 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -21,7 +21,7 @@ import pytest import pytest_check from craft_application.errors import CraftValidationError, PartsLifecycleError -from pydantic import BaseModel, conint +from pydantic import BaseModel @pytest.mark.parametrize( @@ -69,7 +69,7 @@ def test_parts_lifecycle_error_from_os_error( def test_validation_error_from_pydantic(): class Model(BaseModel): - gt_int: conint(gt=42) # pyright: ignore [reportInvalidTypeForm] + gt_int: int = pydantic.Field(gt=42) a_float: float data = { @@ -86,8 +86,8 @@ class Model(BaseModel): expected = textwrap.dedent( """ Bad myfile.yaml content: - - ensure this value is greater than 42 (in field 'gt_int') - - value is not a valid float (in field 'a_float') + - input should be greater than 42 (in field 'gt_int') + - input should be a valid number, unable to parse string as a number (in field 'a_float') """ ).strip() diff --git a/tests/unit/util/test_snap_config.py b/tests/unit/util/test_snap_config.py index d38d643b..de34cf85 100644 --- a/tests/unit/util/test_snap_config.py +++ b/tests/unit/util/test_snap_config.py @@ -87,7 +87,7 @@ def test_unmarshal_invalid_provider_error(): assert str(raised.value) == ( "Bad snap config content:\n" - "- unexpected value; permitted: 'lxd', 'multipass' (in field 'provider')" + "- input should be 'lxd' or 'multipass' (in field 'provider')" ) @@ -98,7 +98,7 @@ def test_unmarshal_extra_data_error(): assert str(raised.value) == ( "Bad snap config content:\n" - "- extra field 'test' not permitted in top-level configuration" + "- extra inputs are not permitted (in field 'test')" ) From a130b31cb77948a6600b2f85dbd9a9543c519095 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 2 Aug 2024 17:43:15 -0300 Subject: [PATCH 02/82] fix: correctly convert root-level errors The 'loc' is empty in Pydantic v2 apparently. Also drop Project.transform_pydantic_error() because with the new model the messages are correctly generated by the field validator. --- craft_application/models/project.py | 21 ----------- craft_application/util/error_formatting.py | 2 +- tests/unit/test_errors.py | 44 +++++++++++++++++----- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/craft_application/models/project.py b/craft_application/models/project.py index e869a460..63754fbd 100644 --- a/craft_application/models/project.py +++ b/craft_application/models/project.py @@ -28,13 +28,10 @@ from craft_cli import emit from craft_providers import bases from craft_providers.errors import BaseConfigurationError -from typing_extensions import override from craft_application import errors from craft_application.models import base from craft_application.models.constraints import ( - MESSAGE_INVALID_NAME, - MESSAGE_INVALID_VERSION, ProjectName, ProjectTitle, SingleEntryList, @@ -345,21 +342,3 @@ def _validate_devel_base( ) return build_base - - @override - @classmethod - def transform_pydantic_error(cls, error: pydantic.ValidationError) -> None: - errors_to_messages: dict[tuple[str, str], str] = { - ("version", "value_error.str.regex"): MESSAGE_INVALID_VERSION, - ("name", "value_error.str.regex"): MESSAGE_INVALID_NAME, - } - - base.CraftBaseModel.transform_pydantic_error(error) - - for error_dict in error.errors(): - loc_and_type = (str(error_dict["loc"][0]), error_dict["type"]) - if message := errors_to_messages.get(loc_and_type): - # Note that unfortunately, Pydantic 1.x does not have the - # "input" key in the error dict, so we can't put the original - # value in the error message. - error_dict["msg"] = message diff --git a/craft_application/util/error_formatting.py b/craft_application/util/error_formatting.py index 8f878013..5bc4ff7d 100644 --- a/craft_application/util/error_formatting.py +++ b/craft_application/util/error_formatting.py @@ -65,7 +65,7 @@ def format_pydantic_error(loc: Iterable[str | int], message: str) -> str: return f"- extra field {field_name!r} not permitted in {location} configuration" if message == "the list has duplicated items": return f"- duplicate {field_name!r} entry not permitted in {location} configuration" - if field_path == "__root__": + if field_path in ("__root__", ""): return f"- {message}" return f"- {message} (in field {field_path!r})" diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index 6063863c..b918d8a4 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -22,6 +22,7 @@ import pytest_check from craft_application.errors import CraftValidationError, PartsLifecycleError from pydantic import BaseModel +from typing_extensions import Self @pytest.mark.parametrize( @@ -67,21 +68,26 @@ def test_parts_lifecycle_error_from_os_error( assert actual == expected +class Model(BaseModel): + gt_int: int = pydantic.Field(gt=42) + a_float: float + b_int: int = 0 + + @pydantic.model_validator(mode="after") + def b_smaller_gt(self) -> Self: + if self.b_int >= self.gt_int: + raise ValueError("'b_int' must be smaller than 'gt_int'") + return self + + def test_validation_error_from_pydantic(): - class Model(BaseModel): - gt_int: int = pydantic.Field(gt=42) - a_float: float - - data = { - "gt_int": 21, - "a_float": "not a float", - } + data = {"gt_int": 21, "a_float": "not a float"} try: Model(**data) except pydantic.ValidationError as e: err = CraftValidationError.from_pydantic(e, file_name="myfile.yaml") else: # pragma: no cover - pytest.fail("Model failed to validate!") + pytest.fail("Model failed to fail to validate!") expected = textwrap.dedent( """ @@ -93,3 +99,23 @@ class Model(BaseModel): message = str(err) assert message == expected + + +def test_validation_error_from_pydantic_model(): + data = {"gt_int": 100, "a_float": 1.0, "b_int": 3000} + try: + Model(**data) + except pydantic.ValidationError as e: + err = CraftValidationError.from_pydantic(e, file_name="myfile.yaml") + else: # pragma: no cover + pytest.fail("Model failed to fail to validate!") + + expected = textwrap.dedent( + """ + Bad myfile.yaml content: + - value error, 'b_int' must be smaller than 'gt_int' + """ + ).strip() + + message = str(err) + assert message == expected From d49e539e99be3bc613b7c1fd9ca410d7869dcb60 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 5 Aug 2024 11:22:03 -0300 Subject: [PATCH 03/82] fix: _populate_platforms() returns dicts of dicts The problem is this: if `_populate_platforms()` returns a dict of Platform objects, it breaks field_validators() for BuildPlanner/Project subclasses (in applications) which expect the input to be the "raw" yaml data. --- craft_application/models/project.py | 11 ++++++----- tests/unit/models/test_project.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/craft_application/models/project.py b/craft_application/models/project.py index 63754fbd..7b24929e 100644 --- a/craft_application/models/project.py +++ b/craft_application/models/project.py @@ -120,7 +120,7 @@ def _validate_platform_set( return values -def _populate_platforms(platforms: dict[str, Platform]) -> dict[str, Platform]: +def _populate_platforms(platforms: dict[str, Any]) -> dict[str, Any]: """Populate empty platform entries. :param platforms: The platform data. @@ -130,9 +130,10 @@ def _populate_platforms(platforms: dict[str, Platform]) -> dict[str, Platform]: for platform_label, platform in platforms.items(): if not platform: # populate "empty" platforms entries from the platform's name - platforms[platform_label] = Platform( - build_on=[platform_label], build_for=[platform_label] - ) + platforms[platform_label] = { + "build-on": [platform_label], + "build-for": [platform_label], + } return platforms @@ -153,7 +154,7 @@ class BuildPlanner(base.CraftBaseModel, metaclass=abc.ABCMeta): @pydantic.field_validator("platforms", mode="before") @classmethod - def _populate_platforms(cls, platforms: dict[str, Platform]) -> dict[str, Platform]: + def _populate_platforms(cls, platforms: dict[str, Any]) -> dict[str, Any]: """Populate empty platform entries.""" return _populate_platforms(platforms) diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index f8fd3a19..9ad0c3a4 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -29,7 +29,6 @@ DEVEL_BASE_WARNING, BuildInfo, BuildPlanner, - Platform, Project, constraints, ) @@ -616,10 +615,14 @@ def test_get_build_plan_all_with_other_platforms(platforms): def test_get_build_plan_build_on_all(): """`build-on: all` is not allowed.""" with pytest.raises(pydantic.ValidationError) as raised: - BuildPlanner( - base="ubuntu@24.04", - platforms={"arm64": Platform(build_on=["all"], build_for=["s390x"])}, - build_base=None, + BuildPlanner.model_validate( + { + "base": "ubuntu@24.04", + "platforms": { + "arm64": {"build-on": ["all"], "build-for": ["s390x"]}, + }, + "build_base": None, + } ) assert "'all' cannot be used for 'build-on'" in str(raised.value) From 3d23ac0efcda9d110b57b1b180b13c558d698625 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Tue, 6 Aug 2024 10:16:09 -0300 Subject: [PATCH 04/82] feat: make "get_validator_by_regex()" public It's useful for downstream applications that have different regexes (e.g. rockcraft). --- craft_application/models/__init__.py | 2 ++ craft_application/models/constraints.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/craft_application/models/__init__.py b/craft_application/models/__init__.py index 9b841cf7..0646a81f 100644 --- a/craft_application/models/__init__.py +++ b/craft_application/models/__init__.py @@ -22,6 +22,7 @@ SummaryStr, UniqueStrList, VersionStr, + get_validator_by_regex, ) from craft_application.models.grammar import ( GrammarAwareProject, @@ -54,4 +55,5 @@ "SummaryStr", "UniqueStrList", "VersionStr", + "get_validator_by_regex", ] diff --git a/craft_application/models/constraints.py b/craft_application/models/constraints.py index 6bef5c71..3f3eeaf7 100644 --- a/craft_application/models/constraints.py +++ b/craft_application/models/constraints.py @@ -33,7 +33,7 @@ def _validate_list_is_unique(value: list[T]) -> list[T]: raise ValueError(f"duplicate values in list: {dupes}") -def _get_validator_by_regex( +def get_validator_by_regex( regex: re.Pattern[str], error_msg: str ) -> Callable[[str], str]: """Get a string validator by regular expression with a known error message. @@ -99,7 +99,7 @@ def validate(value: str) -> str: ProjectName = Annotated[ str, pydantic.BeforeValidator( - _get_validator_by_regex(_PROJECT_NAME_COMPILED_REGEX, MESSAGE_INVALID_NAME) + get_validator_by_regex(_PROJECT_NAME_COMPILED_REGEX, MESSAGE_INVALID_NAME) ), pydantic.Field( min_length=1, @@ -169,7 +169,7 @@ def validate(value: str) -> str: VersionStr = Annotated[ str, pydantic.BeforeValidator( - _get_validator_by_regex(_VERSION_STR_COMPILED_REGEX, MESSAGE_INVALID_VERSION) + get_validator_by_regex(_VERSION_STR_COMPILED_REGEX, MESSAGE_INVALID_VERSION) ), pydantic.Field( max_length=32, From 2f8dd7c0642aa473d08a647dbd7881c8258c30b2 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Tue, 6 Aug 2024 10:40:50 -0300 Subject: [PATCH 05/82] feat: add to_yaml_string() This is just a shortcut for when you want the yaml but not the file. --- craft_application/models/base.py | 4 ++++ tests/unit/models/test_project.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/craft_application/models/base.py b/craft_application/models/base.py index 4ac8d815..08e30438 100644 --- a/craft_application/models/base.py +++ b/craft_application/models/base.py @@ -89,6 +89,10 @@ def to_yaml_file(self, path: pathlib.Path) -> None: with path.open("wt") as file: util.dump_yaml(self.marshal(), stream=file) + def to_yaml_string(self) -> str: + """Return this model as a YAML string.""" + return util.dump_yaml(self.marshal()) + @classmethod def transform_pydantic_error(cls, error: pydantic.ValidationError) -> None: """Modify, in-place, validation errors generated by Pydantic. diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index 9ad0c3a4..a86dc430 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -224,13 +224,14 @@ def test_from_yaml_data_failure(project_file, error_class): ("full_project", PROJECTS_DIR / "full_project.yaml"), ], ) -def test_to_yaml_file(project_fixture, expected_file, tmp_path, request): +def test_to_yaml(project_fixture, expected_file, tmp_path, request): project = request.getfixturevalue(project_fixture) actual_file = tmp_path / "out.yaml" project.to_yaml_file(actual_file) assert actual_file.read_text() == expected_file.read_text() + assert actual_file.read_text() == project.to_yaml_string() def test_effective_base_is_base(full_project): From 1325427d9386961bac90e424c11d53c308cd5ea0 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Tue, 6 Aug 2024 11:27:36 -0300 Subject: [PATCH 06/82] feat: validate each part individually The benefits: - We can report errors in all parts, instead of raising an exception on the first error; - The error messages contain the name of the failing part. --- craft_application/models/project.py | 19 ++++++++++--------- tests/unit/models/test_project.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/craft_application/models/project.py b/craft_application/models/project.py index 7b24929e..6ff87594 100644 --- a/craft_application/models/project.py +++ b/craft_application/models/project.py @@ -237,6 +237,12 @@ def _validate_package_repository(repository: dict[str, Any]) -> dict[str, Any]: return repository +def _validate_part(part: dict[str, Any]) -> dict[str, Any]: + """Verify each part (craft-parts will re-validate this).""" + craft_parts.validate_part(part) + return part + + class Project(base.CraftBaseModel): """Craft Application project definition.""" @@ -257,7 +263,10 @@ class Project(base.CraftBaseModel): adopt_info: str | None = None - parts: dict[str, dict[str, Any]] # parts are handled by craft-parts + parts: dict[ # parts are handled by craft-parts + str, + Annotated[dict[str, Any], pydantic.BeforeValidator(_validate_part)], + ] package_repositories: ( list[ @@ -268,14 +277,6 @@ class Project(base.CraftBaseModel): | None ) = None - @pydantic.field_validator("parts", mode="before") - @classmethod - def _validate_parts(cls, parts: dict[str, Any]) -> dict[str, Any]: - """Verify each part (craft-parts will re-validate this).""" - for part in parts.values(): - craft_parts.validate_part(part) - return parts - @pydantic.field_validator("platforms", mode="before") @classmethod def _populate_platforms(cls, platforms: dict[str, Platform]) -> dict[str, Platform]: diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index a86dc430..1a17087c 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -16,6 +16,7 @@ """Tests for BaseProject""" import copy import pathlib +import re import textwrap from textwrap import dedent @@ -627,3 +628,19 @@ def test_get_build_plan_build_on_all(): ) assert "'all' cannot be used for 'build-on'" in str(raised.value) + + +def test_invalid_part_error(basic_project_dict): + """Check that the part name is included in the error message.""" + basic_project_dict["parts"] = { + "p1": {"plugin": "badplugin"}, + "p2": {"plugin": "nil", "bad-key": 1}, + } + expected = textwrap.dedent( + """\ + Bad bla.yaml content: + - value error, plugin not registered: 'badplugin' (in field 'parts.p1') + - extra inputs are not permitted (in field 'parts.p2.bad-key')""" + ) + with pytest.raises(CraftValidationError, match=re.escape(expected)): + Project.from_yaml_data(basic_project_dict, filepath=pathlib.Path("bla.yaml")) From 5b1a78b37ca1ee7520f6867a0dd669a5a041a0d3 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Tue, 6 Aug 2024 14:34:25 -0300 Subject: [PATCH 07/82] chore: drop 'value error, ' prefix from validation errors This is dropped in format_pydantic_errors(), which means that it mostly affects CraftValidationErrors and from_yaml_data(). This restores the Pydantic v1 UX. --- craft_application/util/error_formatting.py | 1 + tests/unit/models/test_base.py | 4 ++-- tests/unit/models/test_project.py | 6 +++--- tests/unit/test_application.py | 4 ++-- tests/unit/test_errors.py | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/craft_application/util/error_formatting.py b/craft_application/util/error_formatting.py index 5bc4ff7d..c6fdbf5b 100644 --- a/craft_application/util/error_formatting.py +++ b/craft_application/util/error_formatting.py @@ -110,6 +110,7 @@ def _format_pydantic_error_message(msg: str) -> str: """Format pydantic's error message field.""" # Replace shorthand "str" with "string". msg = msg.replace("str type expected", "string type expected") + msg = msg.removeprefix("Value error, ") if msg: msg = msg[0].lower() + msg[1:] return msg diff --git a/tests/unit/models/test_base.py b/tests/unit/models/test_base.py index a65ecd51..2d9390a8 100644 --- a/tests/unit/models/test_base.py +++ b/tests/unit/models/test_base.py @@ -53,8 +53,8 @@ def test_model_reference_slug_errors(): expected = ( "Bad testcraft.yaml content:\n" - "- value error, Bad value1 value (in field 'value1')\n" - "- value error, Bad value2 value (in field 'value2')" + "- bad value1 value (in field 'value1')\n" + "- bad value2 value (in field 'value2')" ) assert str(err.value) == expected assert err.value.doc_slug == "/mymodel.html" diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index 1a17087c..502a1fbc 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -343,7 +343,7 @@ def test_devel_base_error(): dedent( f""" Bad testcraft.yaml content: - - value error, A development build-base must be used when base is 'ubuntu@{expected_devel}' + - a development build-base must be used when base is 'ubuntu@{expected_devel}' """ ).strip() ) @@ -376,7 +376,7 @@ def test_invalid_field_message( full_expected_message = textwrap.dedent( f""" Bad myproject.yaml content: - - value error, {expected_message} (in field '{field_name}') + - {expected_message} (in field '{field_name}') """ ).strip() @@ -639,7 +639,7 @@ def test_invalid_part_error(basic_project_dict): expected = textwrap.dedent( """\ Bad bla.yaml content: - - value error, plugin not registered: 'badplugin' (in field 'parts.p1') + - plugin not registered: 'badplugin' (in field 'parts.p1') - extra inputs are not permitted (in field 'parts.p2.bad-key')""" ) with pytest.raises(CraftValidationError, match=re.escape(expected)): diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 0a85d79e..b8da1cd2 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -2055,8 +2055,8 @@ def test_build_planner_errors(tmp_path, monkeypatch, fake_services): expected = ( "Bad testcraft.yaml content:\n" - "- value error, Bad value1: 10 (in field 'value1')\n" - "- value error, Bad value2: banana (in field 'value2')" + "- bad value1: 10 (in field 'value1')\n" + "- bad value2: banana (in field 'value2')" ) assert str(err.value) == expected diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index b918d8a4..4edd335f 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -113,7 +113,7 @@ def test_validation_error_from_pydantic_model(): expected = textwrap.dedent( """ Bad myfile.yaml content: - - value error, 'b_int' must be smaller than 'gt_int' + - 'b_int' must be smaller than 'gt_int' """ ).strip() From fb24feaf15f842eeed3e63af884d9a9b1e1368ec Mon Sep 17 00:00:00 2001 From: Callahan Kovacs Date: Wed, 7 Aug 2024 13:22:44 -0500 Subject: [PATCH 08/82] build(deps): point to craft-providers feature/2.0 branch Signed-off-by: Callahan Kovacs --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f5043ca9..fe4c7473 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ dependencies = [ "craft-cli>=2.6.0", "craft-grammar@git+https://github.com/canonical/craft-grammar@feature/pydantic-2", "craft-parts@git+https://github.com/canonical/craft-parts@feature/2.0", - "craft-providers@git+https://github.com/canonical/craft-providers@feature/2.0-git-modules", + "craft-providers@git+https://github.com/canonical/craft-providers@feature/2.0", "snap-helpers>=0.4.2", "platformdirs>=3.10", "pydantic~=2.0", From 03bf7c5524d3821da713c5c6a62e507f13cb88a0 Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Thu, 8 Aug 2024 14:11:05 -0400 Subject: [PATCH 09/82] feat: add validators that handle licenses (#402) Added SpdxLicenseStr, ProprietaryLicenseStr and LicenseStr which is union of the previous two. Signed-off-by: Dariusz Duda --- craft_application/models/constraints.py | 50 ++++++++++- pyproject.toml | 1 + tests/unit/models/test_constraints.py | 106 +++++++++++++++++++++++- 3 files changed, 155 insertions(+), 2 deletions(-) diff --git a/craft_application/models/constraints.py b/craft_application/models/constraints.py index 3f3eeaf7..edec9a5f 100644 --- a/craft_application/models/constraints.py +++ b/craft_application/models/constraints.py @@ -14,12 +14,15 @@ # You should have received a copy of the GNU Lesser General Public License along # with this program. If not, see . """Constrained pydantic types for *craft applications.""" + import collections import re from collections.abc import Callable -from typing import Annotated, TypeVar +from typing import Annotated, Literal, TypeVar +import license_expression # type: ignore[import] import pydantic +from pydantic_core import PydanticCustomError T = TypeVar("T") Tv = TypeVar("Tv") @@ -193,3 +196,48 @@ def validate(value: str) -> str: Applications may use a different set of constraints if necessary, but ideally they will retain this same constraint. """ + + +def _parse_spdx_license(value: str) -> license_expression.LicenseExpression: + licensing = license_expression.get_spdx_licensing() + if ( + lic := licensing.parse( # pyright: ignore[reportUnknownMemberType] + value, validate=True + ) + ) is not None: + return lic + raise ValueError + + +def _validate_spdx_license(value: str) -> str: + """Ensure the provided licence is a valid SPDX licence.""" + try: + _ = _parse_spdx_license(value) + except (license_expression.ExpressionError, ValueError): + raise PydanticCustomError( + "not_spdx_license", + "License '{wrong_license}' not valid. It must be in SPDX format.", + {"wrong_license": value}, + ) from None + else: + return value + + +SpdxLicenseStr = Annotated[ + str, + pydantic.AfterValidator(_validate_spdx_license), + pydantic.Field( + title="License", + description="SPDX license string.", + examples=[ + "GPL-3.0", + "MIT", + "LGPL-3.0-or-later", + "GPL-3.0+ and MIT", + ], + ), +] + +ProprietaryLicenseStr = Literal["proprietary"] + +LicenseStr = SpdxLicenseStr | ProprietaryLicenseStr diff --git a/pyproject.toml b/pyproject.toml index fe4c7473..6bf2cd05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "snap-helpers>=0.4.2", "platformdirs>=3.10", "pydantic~=2.0", + "license-expression>=30.0.0", # "pydantic-yaml<1.0", # Pygit2 and libgit2 need to match versions. # Further info: https://www.pygit2.org/install.html#version-numbers diff --git a/tests/unit/models/test_constraints.py b/tests/unit/models/test_constraints.py index e8300fe2..c4a06013 100644 --- a/tests/unit/models/test_constraints.py +++ b/tests/unit/models/test_constraints.py @@ -14,13 +14,21 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . """Tests for project model.""" + import re from string import ascii_letters, ascii_lowercase, digits +from typing import cast import pydantic.errors import pytest from craft_application.models import constraints -from craft_application.models.constraints import ProjectName, VersionStr +from craft_application.models.constraints import ( + LicenseStr, + ProjectName, + ProprietaryLicenseStr, + SpdxLicenseStr, + VersionStr, +) from hypothesis import given, strategies ALPHA_NUMERIC = [*ascii_letters, *digits] @@ -207,4 +215,100 @@ def test_invalid_version_str(version): _VersionStrModel(version=str(version)) +# endregion +# region SpdxLicenseStr tests + +_VALID_SPDX_LICENCES = [ + "MIT", + "GPL-3.0", + "GPL-3.0+", + "GPL-3.0+ and MIT", + "LGPL-3.0+ or BSD-3-Clause", +] + + +@pytest.fixture(params=_VALID_SPDX_LICENCES) +def valid_spdx_license_str(request: pytest.FixtureRequest) -> str: + return cast(str, request.param) + + +class _SpdxLicenseStrModel(pydantic.BaseModel): + license: SpdxLicenseStr + + +def test_spdx_license_str_valid(valid_spdx_license_str: str) -> None: + model = _SpdxLicenseStrModel(license=valid_spdx_license_str) + assert model.license == valid_spdx_license_str + + +@pytest.mark.parametrize("license_str", ["Copyright 1990", "Proprietary"]) +def test_spdx_license_str_invalid(license_str): + with pytest.raises(pydantic.ValidationError) as validation_error: + _ = _SpdxLicenseStrModel(license=license_str) + assert validation_error.match( + f"License '{license_str}' not valid. It must be in SPDX format.", + ) + + +def test_spdx_parser_with_none(): + from craft_application.models.constraints import _validate_spdx_license + + val = None + with pytest.raises( + ValueError, match=f"License '{val}' not valid. It must be in SPDX format." + ): + _validate_spdx_license(val) # pyright: ignore[reportArgumentType] + + +# endregion +# region ProprietaryLicenseStr tests +class _ProprietaryLicenseStrModel(pydantic.BaseModel): + license: ProprietaryLicenseStr + + +def test_proprietary_str_valid(): + model = _ProprietaryLicenseStrModel(license="proprietary") + assert model.license == "proprietary" + + +def test_proprietary_str_invalid(): + with pytest.raises(pydantic.ValidationError) as validation_error: + _ = _ProprietaryLicenseStrModel( + license="non-proprietary" # pyright: ignore[reportArgumentType] + ) + assert validation_error.match("Input should be 'proprietary'") + + +# endregion +# region LicenseStr tests +class _LicenseStrModel(pydantic.BaseModel): + license: LicenseStr + + +@pytest.mark.parametrize( + "license_str", + [*_VALID_SPDX_LICENCES, "proprietary"], +) +def test_license_str_valid(license_str): + model = _LicenseStrModel(license=license_str) + assert model.license == license_str + + +@pytest.mark.parametrize("license_str", ["Copyright 1990", "Proprietary"]) +def test_license_str_invalid(license_str): + with pytest.raises(pydantic.ValidationError) as validation_error: + _ = _LicenseStrModel(license=license_str) + assert validation_error.match( + f"License '{license_str}' not valid. It must be in SPDX format.", + ) + + +def test_license_str_invalid_literal(): + with pytest.raises(pydantic.ValidationError) as validation_error: + _ = _LicenseStrModel( + license="non-proprietary" # pyright: ignore[reportArgumentType] + ) + assert validation_error.match("Input should be 'proprietary'") + + # endregion From 7ab4e6c3dd62e93f577ac4116f75eedfaebaa8a4 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 8 Aug 2024 15:06:44 -0400 Subject: [PATCH 10/82] chore: set correct dependencies --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bf2cd05..1990bfaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,11 @@ name = "craft-application" description = "A framework for *craft applications." dynamic = ["version", "readme"] dependencies = [ - "craft-archives@git+https://github.com/canonical/craft-archives@feature/2.0", + "craft-archives>=2.0.0", "craft-cli>=2.6.0", - "craft-grammar@git+https://github.com/canonical/craft-grammar@feature/pydantic-2", - "craft-parts@git+https://github.com/canonical/craft-parts@feature/2.0", - "craft-providers@git+https://github.com/canonical/craft-providers@feature/2.0", + "craft-grammar>=2.0.0", + "craft-parts>=2.0.0", + "craft-providers>=2.0.0", "snap-helpers>=0.4.2", "platformdirs>=3.10", "pydantic~=2.0", From 25ea79ccdb88bde2807032a43e197bfe55d7c88b Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 9 Aug 2024 15:40:41 -0400 Subject: [PATCH 11/82] style(type): fix typing issues --- craft_application/models/project.py | 4 +++- craft_application/services/provider.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/craft_application/models/project.py b/craft_application/models/project.py index 6ff87594..550f00e1 100644 --- a/craft_application/models/project.py +++ b/craft_application/models/project.py @@ -309,7 +309,9 @@ def _providers_base(cls, base: str) -> craft_providers.bases.BaseAlias | None: """ try: name, channel = base.split("@") - return craft_providers.bases.get_base_alias((name, channel)) + return craft_providers.bases.get_base_alias( + craft_providers.bases.BaseName(name, channel) + ) except (ValueError, BaseConfigurationError) as err: raise ValueError(f"Unknown base {base!r}") from err diff --git a/craft_application/services/provider.py b/craft_application/services/provider.py index 38607d32..a359d930 100644 --- a/craft_application/services/provider.py +++ b/craft_application/services/provider.py @@ -141,7 +141,7 @@ def instance( def get_base( self, - base_name: bases.BaseName | tuple[str, str], + base_name: bases.BaseName, *, instance_name: str, **kwargs: bool | str | pathlib.Path | None, @@ -164,7 +164,7 @@ def get_base( # this only applies to our Buildd images (i.e.; Ubuntu) self.packages.extend(["gpg", "dirmngr"]) return base_class( - alias=alias, # pyright: ignore[reportArgumentType] craft-providers annotations are loose. + alias=alias, # type: ignore[arg-type] compatibility_tag=f"{self._app.name}-{base_class.compatibility_tag}", hostname=instance_name, snaps=self.snaps, From a00bf9b64ab8255644b40d4013d3e75b02419872 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 9 Aug 2024 16:41:43 -0400 Subject: [PATCH 12/82] docs: add 4.0.0 release to changelog --- docs/reference/changelog.rst | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 34d3f94c..02f520f0 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,37 @@ Changelog ********* +4.0.0 (2024-Aug-09) +------------------- + +Breaking changes +================ + +This release migrates to pydantic 2. +Most exit codes use constants from the ``os`` module. (This makes +craft-application 4 only compatible with Windows when using Python 3.11+.) + +Models +====== +Add constrained string fields that check for SPDX license strings or the +license string "proprietary". + +CraftBaseModel now includes a ``to_yaml_string`` method. + +Custom regex-based validators can be built with +``models.get_validator_by_regex``. These can be used to make a better error +message than the pydantic default. + +Git +=== + +The ``git`` submodule under ``launchpad`` is now its own module and can clone +repositories and add remotes. + + +For a complete list of commits, check out the `4.0.0`_ release on GitHub. + + 3.2.0 (2024-Jul-07) ------------------- @@ -16,7 +47,7 @@ Documentation Add a how-to guide for using partitions. -For a complete list of commits, check out the `3.2.0`_ release on GitHb. +For a complete list of commits, check out the `3.2.0`_ release on GitHub. 3.1.0 (2024-Jul-05) ------------------- @@ -144,3 +175,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _3.0.0: https://github.com/canonical/craft-application/releases/tag/3.0.0 .. _3.1.0: https://github.com/canonical/craft-application/releases/tag/3.1.0 .. _3.2.0: https://github.com/canonical/craft-application/releases/tag/3.2.0 +.. _4.0.0: https://github.com/canonical/craft-application/releases/tag/4.0.0 From 947e1b4ba6afff364135fbb971493621f1db4f59 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Tue, 13 Aug 2024 10:26:55 -0400 Subject: [PATCH 13/82] feat(lifecycle): break out _get_build_for method (#414) Fixes #394 --- craft_application/services/lifecycle.py | 27 ++++++---- tests/unit/conftest.py | 8 ++- tests/unit/services/test_lifecycle.py | 66 +++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/craft_application/services/lifecycle.py b/craft_application/services/lifecycle.py index b1498a9a..86eaa51c 100644 --- a/craft_application/services/lifecycle.py +++ b/craft_application/services/lifecycle.py @@ -151,6 +151,23 @@ def setup(self) -> None: self._lcm = self._init_lifecycle_manager() callbacks.register_post_step(self.post_prime, step_list=[Step.PRIME]) + def _get_build_for(self) -> str: + """Get the ``build_for`` architecture for craft-parts. + + The default behaviour is as follows: + 1. If the build plan's ``build_for`` is ``all``, use the host architecture. + 2. If it's anything else, use that. + 3. If it's undefined, use the host architecture. + """ + # Note: we fallback to the host's architecture here if the build plan + # is empty just to be able to create the LifecycleManager; this will + # correctly fail later on when run() is called (but not necessarily when + # something else like clean() is called). + # We also use the host arch if the build-for is 'all' + if self._build_plan and self._build_plan[0].build_for != "all": + return self._build_plan[0].build_for + return util.get_host_architecture() + def _init_lifecycle_manager(self) -> LifecycleManager: """Create and return the Lifecycle manager. @@ -160,15 +177,7 @@ def _init_lifecycle_manager(self) -> LifecycleManager: emit.debug(f"Initialising lifecycle manager in {self._work_dir}") emit.trace(f"Lifecycle: {repr(self)}") - # Note: we fallback to the host's architecture here if the build plan - # is empty just to be able to create the LifecycleManager; this will - # correctly fail later on when run() is called (but not necessarily when - # something else like clean() is called). - # We also use the host arch if the build-for is 'all' - if self._build_plan and self._build_plan[0].build_for != "all": - build_for = self._build_plan[0].build_for - else: - build_for = util.get_host_architecture() + build_for = self._get_build_for() if self._project.package_repositories: self._manager_kwargs["package_repositories"] = ( diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5f9ee5f9..75b6527a 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -19,7 +19,13 @@ from unittest import mock import pytest -from craft_application import services +from craft_application import services, util + + +@pytest.fixture(params=["amd64", "arm64", "riscv64"]) +def fake_host_architecture(monkeypatch, request) -> str: + monkeypatch.setattr(util, "get_host_architecture", lambda: request.param) + return request.param @pytest.fixture() diff --git a/tests/unit/services/test_lifecycle.py b/tests/unit/services/test_lifecycle.py index 315c6c97..092b7391 100644 --- a/tests/unit/services/test_lifecycle.py +++ b/tests/unit/services/test_lifecycle.py @@ -27,6 +27,7 @@ import pytest_check from craft_application import errors, models, util from craft_application.errors import InvalidParameterError, PartsLifecycleError +from craft_application.models.project import BuildInfo from craft_application.services import lifecycle from craft_application.util import repositories from craft_parts import ( @@ -351,6 +352,71 @@ def test_get_primed_stage_packages(lifecycle_service): assert pkgs == ["pkg1", "pkg2"] +@pytest.mark.parametrize( + ("build_plan", "expected"), + [ + ([], None), + ( + [ + BuildInfo( + "my-platform", + build_on="any", + build_for="all", + base=bases.BaseName("ubuntu", "24.04"), + ) + ], + None, + ), + ( + [ + BuildInfo( + "my-platform", + build_on="any", + build_for="amd64", + base=bases.BaseName("ubuntu", "24.04"), + ) + ], + "amd64", + ), + ( + [ + BuildInfo( + "my-platform", + build_on="any", + build_for="arm64", + base=bases.BaseName("ubuntu", "24.04"), + ) + ], + "arm64", + ), + ( + [ + BuildInfo( + "my-platform", + build_on="any", + build_for="riscv64", + base=bases.BaseName("ubuntu", "24.04"), + ) + ], + "riscv64", + ), + ], +) +def test_get_build_for( + fake_host_architecture, + fake_parts_lifecycle: lifecycle.LifecycleService, + build_plan: list[BuildInfo], + expected: str | None, +): + if expected is None: + expected = fake_host_architecture + fake_parts_lifecycle._build_plan = build_plan + + actual = fake_parts_lifecycle._get_build_for() + + assert actual == expected + + @pytest.mark.parametrize( "actions", [ From dbee926277d3038d5a08b42ab8342f05bce16fd3 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 14 Aug 2024 09:49:33 -0400 Subject: [PATCH 14/82] fix: allow using a snap from the snap store (#417) If the app isn't running from a snap, default to using the stable channel for the snap store snap, with an option to change channels using the CRAFT_SNAP_CHANNEL environment variable. --- craft_application/services/provider.py | 11 ++++-- tests/unit/services/test_provider.py | 50 ++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/craft_application/services/provider.py b/craft_application/services/provider.py index a359d930..7ca9e51c 100644 --- a/craft_application/services/provider.py +++ b/craft_application/services/provider.py @@ -74,8 +74,7 @@ def __init__( self._work_dir = work_dir self._build_plan = build_plan self.snaps: list[Snap] = [] - if install_snap: - self.snaps.append(Snap(name=app.name, channel=None, classic=True)) + self._install_snap = install_snap self.environment: dict[str, str | None] = {self.managed_mode_env_var: "1"} self.packages: list[str] = [] # this is a private attribute because it may not reflect the actual @@ -94,6 +93,14 @@ def setup(self) -> None: if name in os.environ: self.environment[name] = os.getenv(name) + if self._install_snap: + channel = ( + None + if util.is_running_from_snap(self._app.name) + else os.getenv("CRAFT_SNAP_CHANNEL", "latest/stable") + ) + self.snaps.append(Snap(name=self._app.name, channel=channel, classic=True)) + @contextlib.contextmanager def instance( self, diff --git a/tests/unit/services/test_provider.py b/tests/unit/services/test_provider.py index 2e08d303..cd89d08c 100644 --- a/tests/unit/services/test_provider.py +++ b/tests/unit/services/test_provider.py @@ -31,12 +31,55 @@ @pytest.mark.parametrize( - ("install_snap", "snaps"), - [(True, [Snap(name="testcraft", channel=None, classic=True)]), (False, [])], + ("install_snap", "environment", "snaps"), + [ + (True, {}, [Snap(name="testcraft", channel="latest/stable", classic=True)]), + ( + True, + {"CRAFT_SNAP_CHANNEL": "something"}, + [Snap(name="testcraft", channel="something", classic=True)], + ), + ( + True, + {"SNAP_NAME": "testcraft", "SNAP": "/snap/testcraft/x1"}, + [Snap(name="testcraft", channel=None, classic=True)], + ), + ( + True, + { + "SNAP_NAME": "testcraft", + "SNAP": "/snap/testcraft/x1", + "CRAFT_SNAP_CHANNEL": "something", + }, + [Snap(name="testcraft", channel=None, classic=True)], + ), + (False, {}, []), + (False, {"CRAFT_SNAP_CHANNEL": "something"}, []), + ( + False, + { + "SNAP_NAME": "testcraft", + "SNAP": "/snap/testcraft/x1", + "CRAFT_SNAP_CHANNEL": "something", + }, + [], + ), + ], ) def test_install_snap( - app_metadata, fake_project, fake_build_plan, fake_services, install_snap, snaps + monkeypatch, + app_metadata, + fake_project, + fake_build_plan, + fake_services, + install_snap, + environment, + snaps, ): + monkeypatch.delenv("SNAP", raising=False) + monkeypatch.delenv("CRAFT_SNAP_CHANNEL", raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) service = provider.ProviderService( app_metadata, fake_services, @@ -45,6 +88,7 @@ def test_install_snap( build_plan=fake_build_plan, install_snap=install_snap, ) + service.setup() assert service.snaps == snaps From db9f21b6ea18c717709181c6fac3a1c7347436f8 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 14 Aug 2024 12:49:15 -0400 Subject: [PATCH 15/82] ci: update renovate config from starbase (#409) --- .github/renovate.json5 | 65 ++++++++++++++++++++------- .github/workflows/check-renovate.yaml | 40 +++++++++++++++++ 2 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/check-renovate.yaml diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 9c4bd555..8342105f 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,20 +1,22 @@ { // Configuration file for RenovateBot: https://docs.renovatebot.com/configuration-options - extends: ["config:base"], + extends: ["config:recommended", ":semanticCommitTypeAll(build)"], labels: ["dependencies"], // For convenient searching in GitHub + baseBranches: ["$default", "/^hotfix\\/.*/"], pip_requirements: { fileMatch: ["^tox.ini$", "(^|/)requirements([\\w-]*)\\.txt$", "^.pre-commit-config.yaml$"] }, packageRules: [ { // Internal package minor patch updates get top priority, with auto-merging - groupName: "internal package patch releases", + groupName: "internal package minor releases", matchPackagePatterns: ["^craft-.*"], matchUpdateTypes: ["minor", "patch", "pin", "digest"], prPriority: 10, automerge: true, minimumReleaseAge: "0 seconds", schedule: ["at any time"], + matchBaseBranches: ["$default"], // Only do minor releases on main }, { // Same as above, but for hotfix branches, only for patch, and without auto-merging. @@ -24,13 +26,13 @@ prPriority: 10, minimumReleaseAge: "0 seconds", schedule: ["at any time"], - matchBaseBranches: ["/^hotfix/.*/"], // All hotfix branches + matchBaseBranches: ["/^hotfix\\/.*/"], // All hotfix branches }, { // Automerge patches, pin changes and digest changes. // Also groups these changes together. groupName: "bugfixes", - excludePackagePrefixes: ["lint", "types"], + excludeDepPatterns: ["lint/.*", "types/.*"], matchUpdateTypes: ["patch", "pin", "digest"], prPriority: 3, // Patches should go first! automerge: true @@ -38,9 +40,10 @@ { // Update all internal packages in one higher-priority PR groupName: "internal packages", - matchPackagePrefixes: ["craft-", "snap-"], - matchLanguages: ["python"], - prPriority: 2 + matchDepPatterns: ["craft-.*", "snap-.*"], + matchCategories: ["python"], + prPriority: 2, + matchBaseBranches: ["$default"], // Not for hotfix branches }, { // GitHub Actions are higher priority to update than most dependencies since they don't tend to break things. @@ -59,10 +62,32 @@ // Minor changes can be grouped and automerged for dev dependencies, but are also deprioritised. groupName: "development dependencies (non-major)", groupSlug: "dev-dependencies", - matchPackagePrefixes: [ - "dev", - "lint", - "types" + matchDepPatterns: [ + "dev/.*", + "lint/.*", + "types/.*" + ], + matchPackagePatterns: [ + // Brought from charmcraft. May not be complete. + // This helps group dependencies in requirements-dev.txt files. + "^(.*/)?autoflake$", + "^(.*/)?black$", + "^(.*/)?codespell$", + "^(.*/)?coverage$", + "^(.*/)?flake8$", + "^(.*/)?hypothesis$", + "^(.*/)?mypy$", + "^(.*/)?pycodestyle$", + "^(.*/)?docstyle$", + "^(.*/)?pyfakefs$", + "^(.*/)?pyflakes$", + "^(.*/)?pylint$", + "^(.*/)?pytest", + "^(.*/)?responses$", + "^(.*/)?ruff$", + "^(.*/)?twine$", + "^(.*/)?tox$", + "^(.*/)?types-", ], matchUpdateTypes: ["minor", "patch", "pin", "digest"], prPriority: -1, @@ -74,12 +99,14 @@ groupSlug: "doc-dependencies", matchPackageNames: ["Sphinx", "furo"], matchPackagePatterns: ["[Ss]phinx.*$"], - matchPackagePrefixes: ["docs"], + matchDepPatterns: ["docs/.*"], + matchBaseBranches: ["$default"], // Not for hotfix branches }, { // Other major dependencies get deprioritised below minor dev dependencies. matchUpdateTypes: ["major"], - prPriority: -2 + prPriority: -2, + matchBaseBranches: ["$default"], // Not for hotfix branches }, { // Major dev dependencies are stone last, but grouped. @@ -87,19 +114,22 @@ groupSlug: "dev-dependencies", matchDepTypes: ["devDependencies"], matchUpdateTypes: ["major"], - prPriority: -3 + prPriority: -3, + matchBaseBranches: ["$default"], // Not for hotfix branches }, { // Pyright makes regular breaking changes in patch releases, so we separate these // and do them independently. matchPackageNames: ["pyright", "types/pyright"], - prPriority: -4 + prPriority: -4, + matchBaseBranches: ["$default"], // Not for hotfix branches } ], - regexManagers: [ + customManagers: [ { // tox.ini can get updates too if we specify for each package. fileMatch: ["tox.ini"], + customType: "regex", depTypeTemplate: "devDependencies", matchStrings: [ "# renovate: datasource=(?\\S+)\n\\s+(?.*?)(\\[[\\w]*\\])*[=><]=?(?.*?)\n" @@ -108,6 +138,7 @@ { // .pre-commit-config.yaml version updates fileMatch: [".pre-commit-config.yaml"], + customType: "regex", datasourceTemplate: "pypi", depTypeTemplate: "lint", matchStrings: [ @@ -122,7 +153,7 @@ prCreation: "not-pending", // Wait until status checks have completed before raising the PR prNotPendingHours: 4, // ...unless the status checks have been running for 4+ hours. prHourlyLimit: 1, // No more than 1 PR per hour. - stabilityDays: 2, // Wait 2 days from release before updating. + minimumReleaseAge: "2 days", automergeStrategy: "squash", // Squash & rebase when auto-merging. semanticCommitType: "build" // use `build` as commit header type (i.e. `build(deps): `) } diff --git a/.github/workflows/check-renovate.yaml b/.github/workflows/check-renovate.yaml new file mode 100644 index 00000000..7d13cfbf --- /dev/null +++ b/.github/workflows/check-renovate.yaml @@ -0,0 +1,40 @@ +name: Renovate check +on: + pull_request: + paths: + - ".github/workflows/check-renovate.yaml" + - ".github/renovate.json5" + + # Allows triggering the workflow manually from the Actions tab + workflow_dispatch: + inputs: + enable_ssh_access: + type: boolean + description: 'Enable ssh access' + required: false + default: false + +jobs: + renovate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install node + uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Install renovate + run: npm install --global renovate + - name: Enable ssh access + uses: mxschmitt/action-tmate@v3 + if: ${{ inputs.enable_ssh_access }} + with: + limit-access-to-actor: true + - name: Check renovate config + run: renovate-config-validator .github/renovate.json5 + - name: Renovate dry-run + run: renovate --dry-run --autodiscover + env: + RENOVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RENOVATE_USE_BASE_BRANCH_CONFIG: ${{ github.ref }} From 499fa410dccff3fa4d9352c12e84136fe6622683 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 14 Aug 2024 13:01:03 -0400 Subject: [PATCH 16/82] build(deps): update development and build dependencies (#418) --- pyproject.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1990bfaa..0f2868a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,10 +48,10 @@ remote = [ "launchpadlib>=1.10.16", ] dev = [ - "coverage[toml]==7.4.4", + "coverage[toml]==7.6.1", "hypothesis>=6.0", "pyfakefs~=5.3", - "pytest==8.1.1", + "pytest==8.3.2", "pytest-check==2.3.1", "pytest-cov==5.0.0", "pytest-mock==3.14.0", @@ -69,7 +69,7 @@ lint = [ ] types = [ "mypy[reports]==1.9.0", - "pyright==1.1.359", + "pyright==1.1.376", "types-requests", "types-urllib3", ] @@ -84,8 +84,8 @@ apt = [ [build-system] requires = [ - "setuptools==70.1.0", - "setuptools_scm[toml]>=7.1" + "setuptools==72.2.0", + "setuptools_scm[toml]>=8.1" ] build-backend = "setuptools.build_meta" From 20a1e10000e66a19696589f9322d892425ea3110 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 14 Aug 2024 14:18:35 -0400 Subject: [PATCH 17/82] docs: add 4.1.0 changelog entry (#421) --- docs/reference/changelog.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 02f520f0..474d5fb8 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,24 @@ Changelog ********* +4.1.0 (2024-Aug-14) +------------------- + +Application +=========== + +If an app isn't running from snap, the installed app will install the snap +in the provider using the channel in the ``CRAFT_SNAP_CHANNEL`` environment +variable, defaulting to ``latest/stable`` if none is set. + +Services +======== + +The ``LifecycleService`` now breaks out a ``_get_build_for`` method for +apps to override if necessary. + +For a complete list of commits, check out the `4.1.0`_ release on GitHub. + 4.0.0 (2024-Aug-09) ------------------- @@ -176,3 +194,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _3.1.0: https://github.com/canonical/craft-application/releases/tag/3.1.0 .. _3.2.0: https://github.com/canonical/craft-application/releases/tag/3.2.0 .. _4.0.0: https://github.com/canonical/craft-application/releases/tag/4.0.0 +.. _4.1.0: https://github.com/canonical/craft-application/releases/tag/4.1.0 From 2f56f14b42eb9728d3a53b0bf7127af1a05af8c8 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Mon, 19 Aug 2024 08:53:48 -0400 Subject: [PATCH 18/82] style(lint): fix ruff 0.6.0 linting errors (#423) Run using `ruff check --fix` --- tests/conftest.py | 28 +++++++++---------- tests/integration/conftest.py | 6 ++--- tests/integration/test_application.py | 8 +++--- tests/unit/commands/test_base.py | 6 ++--- tests/unit/commands/test_lifecycle.py | 2 +- tests/unit/conftest.py | 4 +-- tests/unit/git/test_git.py | 6 ++--- tests/unit/launchpad/conftest.py | 6 ++--- tests/unit/launchpad/models/test_base.py | 2 +- tests/unit/models/test_project.py | 8 +++--- tests/unit/remote/conftest.py | 2 +- tests/unit/remote/test_utils.py | 2 +- tests/unit/services/conftest.py | 8 +++--- tests/unit/services/test_lifecycle.py | 2 +- tests/unit/services/test_provider.py | 2 +- tests/unit/services/test_remotebuild.py | 8 +++--- tests/unit/services/test_service_factory.py | 2 +- tests/unit/test_application.py | 30 ++++++++++----------- tests/unit/test_secrets.py | 2 +- tests/unit/util/test_retry.py | 2 +- tests/unit/util/test_snap_config.py | 4 +-- 21 files changed, 70 insertions(+), 70 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fe56e6d5..d46e8ffe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,7 +40,7 @@ def _create_fake_build_plan(num_infos: int = 1) -> list[models.BuildInfo]: return [models.BuildInfo("foo", arch, arch, base)] * num_infos -@pytest.fixture() +@pytest.fixture def features(request) -> dict[str, bool]: """Fixture that controls the enabled features. @@ -69,7 +69,7 @@ def default_app_metadata() -> craft_application.AppMetadata: ) -@pytest.fixture() +@pytest.fixture def app_metadata(features) -> craft_application.AppMetadata: with pytest.MonkeyPatch.context() as m: m.setattr(metadata, "version", lambda _: "3.14159") @@ -82,7 +82,7 @@ def app_metadata(features) -> craft_application.AppMetadata: ) -@pytest.fixture() +@pytest.fixture def app_metadata_docs(features) -> craft_application.AppMetadata: with pytest.MonkeyPatch.context() as m: m.setattr(metadata, "version", lambda _: "3.14159") @@ -95,7 +95,7 @@ def app_metadata_docs(features) -> craft_application.AppMetadata: ) -@pytest.fixture() +@pytest.fixture def fake_project() -> models.Project: arch = util.get_host_architecture() return models.Project( @@ -116,13 +116,13 @@ def fake_project() -> models.Project: ) -@pytest.fixture() +@pytest.fixture def fake_build_plan(request) -> list[models.BuildInfo]: num_infos = getattr(request, "param", 1) return _create_fake_build_plan(num_infos) -@pytest.fixture() +@pytest.fixture def full_build_plan(mocker) -> list[models.BuildInfo]: """A big build plan with multiple bases and build-for targets.""" host_arch = util.get_host_architecture() @@ -142,7 +142,7 @@ def full_build_plan(mocker) -> list[models.BuildInfo]: return build_plan -@pytest.fixture() +@pytest.fixture def enable_partitions() -> Iterator[craft_parts.Features]: """Enable the partitions feature in craft_parts for the relevant test.""" enable_overlay = craft_parts.Features().enable_overlay @@ -152,7 +152,7 @@ def enable_partitions() -> Iterator[craft_parts.Features]: craft_parts.Features.reset() -@pytest.fixture() +@pytest.fixture def enable_overlay() -> Iterator[craft_parts.Features]: """Enable the overlay feature in craft_parts for the relevant test.""" if not os.getenv("CI") and not shutil.which("fuse-overlayfs"): @@ -164,7 +164,7 @@ def enable_overlay() -> Iterator[craft_parts.Features]: craft_parts.Features.reset() -@pytest.fixture() +@pytest.fixture def lifecycle_service( app_metadata, fake_project, fake_services, fake_build_plan, mocker, tmp_path ) -> services.LifecycleService: @@ -194,7 +194,7 @@ def lifecycle_service( return service -@pytest.fixture() +@pytest.fixture def request_service(app_metadata, fake_services) -> services.RequestService: """A working version of the requests service.""" return services.RequestService(app=app_metadata, services=fake_services) @@ -208,7 +208,7 @@ def emitter_verbosity(request): emit.set_mode(reset_verbosity) -@pytest.fixture() +@pytest.fixture def fake_provider_service_class(fake_build_plan): class FakeProviderService(services.ProviderService): def __init__( @@ -229,7 +229,7 @@ def __init__( return FakeProviderService -@pytest.fixture() +@pytest.fixture def fake_package_service_class(): class FakePackageService(services.PackageService): def pack( @@ -247,7 +247,7 @@ def metadata(self) -> models.BaseMetadata: return FakePackageService -@pytest.fixture() +@pytest.fixture def fake_lifecycle_service_class(tmp_path, fake_build_plan): class FakeLifecycleService(services.LifecycleService): def __init__( @@ -272,7 +272,7 @@ def __init__( return FakeLifecycleService -@pytest.fixture() +@pytest.fixture def fake_services( app_metadata, fake_project, fake_lifecycle_service_class, fake_package_service_class ): diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a6ae6c48..dfe7a9d7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -42,7 +42,7 @@ def pytest_runtest_setup(item: pytest.Item): pytest.skip("multipass not installed") -@pytest.fixture() +@pytest.fixture def provider_service(app_metadata, fake_project, fake_build_plan, fake_services): """Provider service with install snap disabled for integration tests""" return provider.ProviderService( @@ -63,7 +63,7 @@ def anonymous_remote_build_service(default_app_metadata): return service -@pytest.fixture() +@pytest.fixture def snap_safe_tmp_path(): """A temporary path accessible to snap-confined craft providers. @@ -87,7 +87,7 @@ def snap_safe_tmp_path(): yield pathlib.Path(temp_dir) -@pytest.fixture() +@pytest.fixture def pretend_jammy(mocker) -> None: """Pretend we're running on jammy. Used for tests that use destructive mode.""" fake_host = bases.BaseName(name="ubuntu", version="22.04") diff --git a/tests/integration/test_application.py b/tests/integration/test_application.py index 7da91574..c3c2d01e 100644 --- a/tests/integration/test_application.py +++ b/tests/integration/test_application.py @@ -39,7 +39,7 @@ def _pre_run(self, dispatcher: craft_cli.Dispatcher) -> None: self.project_dir = pathlib.Path.cwd() -@pytest.fixture() +@pytest.fixture def create_app(app_metadata, fake_package_service_class): def _inner(): # Create a factory without a project, to simulate a real application use @@ -52,7 +52,7 @@ def _inner(): return _inner -@pytest.fixture() +@pytest.fixture def app(create_app): return create_app() @@ -327,7 +327,7 @@ def test_global_environment( ) -@pytest.fixture() +@pytest.fixture def setup_secrets_project(create_app, monkeypatch, tmp_path): """Test the use of build secrets in destructive mode.""" @@ -348,7 +348,7 @@ def _inner(*, destructive_mode: bool): return _inner -@pytest.fixture() +@pytest.fixture def check_secrets_output(tmp_path, capsys): def _inner(): prime_dir = tmp_path / "prime" diff --git a/tests/unit/commands/test_base.py b/tests/unit/commands/test_base.py index a8fa9336..8b94bae0 100644 --- a/tests/unit/commands/test_base.py +++ b/tests/unit/commands/test_base.py @@ -23,7 +23,7 @@ from typing_extensions import override -@pytest.fixture() +@pytest.fixture def fake_command(app_metadata, fake_services): class FakeCommand(base.AppCommand): _run_managed = True @@ -85,7 +85,7 @@ def test_needs_project(fake_command, always_load_project): # region Tests for ExtensibleCommand -@pytest.fixture() +@pytest.fixture def fake_extensible_cls(): class FakeExtensibleCommand(base.ExtensibleCommand): name = "fake" @@ -108,7 +108,7 @@ def _run( return FakeExtensibleCommand -@pytest.fixture() +@pytest.fixture def fake_extensible_child(fake_extensible_cls): class FakeChild(fake_extensible_cls): name = "child" diff --git a/tests/unit/commands/test_lifecycle.py b/tests/unit/commands/test_lifecycle.py index 7c08030c..5be67dea 100644 --- a/tests/unit/commands/test_lifecycle.py +++ b/tests/unit/commands/test_lifecycle.py @@ -492,7 +492,7 @@ def test_pack_run_wrong_step(app_metadata, fake_services): assert exc_info.value.args[0] == "Step name wrong-command passed to pack command." -@pytest.fixture() +@pytest.fixture def mock_subprocess_run(mocker): return mocker.patch.object(subprocess, "run") diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 75b6527a..9ad49687 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -28,7 +28,7 @@ def fake_host_architecture(monkeypatch, request) -> str: return request.param -@pytest.fixture() +@pytest.fixture def provider_service( app_metadata, fake_project, fake_build_plan, fake_services, tmp_path ): @@ -41,7 +41,7 @@ def provider_service( ) -@pytest.fixture() +@pytest.fixture def mock_services(app_metadata, fake_project, fake_package_service_class): factory = services.ServiceFactory( app_metadata, project=fake_project, PackageClass=fake_package_service_class diff --git a/tests/unit/git/test_git.py b/tests/unit/git/test_git.py index 5b2207b8..f2c0c5a4 100644 --- a/tests/unit/git/test_git.py +++ b/tests/unit/git/test_git.py @@ -34,7 +34,7 @@ ) -@pytest.fixture() +@pytest.fixture def empty_working_directory(tmp_path) -> Iterator[Path]: cwd = pathlib.Path.cwd() @@ -46,7 +46,7 @@ def empty_working_directory(tmp_path) -> Iterator[Path]: os.chdir(cwd) -@pytest.fixture() +@pytest.fixture def empty_repository(empty_working_directory) -> Path: subprocess.run(["git", "init"], check=True) return cast(Path, empty_working_directory) @@ -611,7 +611,7 @@ def test_check_git_repo_for_remote_build_invalid(empty_working_directory): check_git_repo_for_remote_build(empty_working_directory) -@pytest.fixture() +@pytest.fixture def patched_cloning_process(mocker): return mocker.patch( "craft_parts.utils.os_utils.process_run", diff --git a/tests/unit/launchpad/conftest.py b/tests/unit/launchpad/conftest.py index a94f4495..d5a3ca60 100644 --- a/tests/unit/launchpad/conftest.py +++ b/tests/unit/launchpad/conftest.py @@ -21,12 +21,12 @@ from craft_application.launchpad import Launchpad -@pytest.fixture() +@pytest.fixture def mock_lplib(): return mock.Mock(**{"me.name": "test_user"}) -@pytest.fixture() +@pytest.fixture def mock_lplib_entry(): return mock.MagicMock( __class__=lazr.restfulclient.resource.Entry, @@ -34,6 +34,6 @@ def mock_lplib_entry(): ) -@pytest.fixture() +@pytest.fixture def fake_launchpad(mock_lplib): return Launchpad("testcraft", mock_lplib) diff --git a/tests/unit/launchpad/models/test_base.py b/tests/unit/launchpad/models/test_base.py index 97e47cb5..19c5f93f 100644 --- a/tests/unit/launchpad/models/test_base.py +++ b/tests/unit/launchpad/models/test_base.py @@ -49,7 +49,7 @@ def new(cls, *args: Any, **kwargs: Any) -> Self: raise NotImplementedError -@pytest.fixture() +@pytest.fixture def fake_obj(fake_launchpad, mock_lplib_entry): return FakeLaunchpadObject(fake_launchpad, mock_lplib_entry) diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index 502a1fbc..ab0f21b6 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -38,7 +38,7 @@ PARTS_DICT = {"my-part": {"plugin": "nil"}} -@pytest.fixture() +@pytest.fixture def basic_project(): # pyright doesn't like these types and doesn't have a pydantic plugin like mypy. # Because of this, we need to silence several errors in these constants. @@ -63,13 +63,13 @@ def basic_project(): } -@pytest.fixture() +@pytest.fixture def basic_project_dict(): """Provides a modifiable copy of ``BASIC_PROJECT_DICT``""" return copy.deepcopy(BASIC_PROJECT_DICT) -@pytest.fixture() +@pytest.fixture def full_project(): return Project.model_validate( { @@ -115,7 +115,7 @@ def full_project(): } -@pytest.fixture() +@pytest.fixture def full_project_dict(): """Provides a modifiable copy of ``FULL_PROJECT_DICT``""" return copy.deepcopy(FULL_PROJECT_DICT) diff --git a/tests/unit/remote/conftest.py b/tests/unit/remote/conftest.py index c301a349..f64a3ec5 100644 --- a/tests/unit/remote/conftest.py +++ b/tests/unit/remote/conftest.py @@ -17,7 +17,7 @@ import pytest -@pytest.fixture() +@pytest.fixture def new_dir(tmp_path): """Change to a new temporary directory.""" diff --git a/tests/unit/remote/test_utils.py b/tests/unit/remote/test_utils.py index 112b2b7d..a8897adb 100644 --- a/tests/unit/remote/test_utils.py +++ b/tests/unit/remote/test_utils.py @@ -152,7 +152,7 @@ def test_get_build_id_directory_is_not_a_directory_error(): ################ -@pytest.fixture() +@pytest.fixture def stub_directory_tree(): """Creates a tree of directories and files.""" root_dir = Path("root-dir") diff --git a/tests/unit/services/conftest.py b/tests/unit/services/conftest.py index b526d04b..61eedf5c 100644 --- a/tests/unit/services/conftest.py +++ b/tests/unit/services/conftest.py @@ -50,7 +50,7 @@ def get_mock_callable(**kwargs): return mock.Mock(spec_set=Callable, **kwargs) -@pytest.fixture() +@pytest.fixture def mock_project_entry(): return get_mock_lazr_entry( resource_type="project", @@ -58,7 +58,7 @@ def mock_project_entry(): ) -@pytest.fixture() +@pytest.fixture def mock_git_repository(): return get_mock_lazr_entry( "git_repository", @@ -67,7 +67,7 @@ def mock_git_repository(): ) -@pytest.fixture() +@pytest.fixture def fake_launchpad(app_metadata, mock_git_repository, mock_project_entry): me = mock.Mock(lazr.restfulclient.resource.Entry) me.name = "craft_test_user" @@ -109,7 +109,7 @@ def fake_launchpad(app_metadata, mock_git_repository, mock_project_entry): return launchpad.Launchpad(app_metadata.name, lp) -@pytest.fixture() +@pytest.fixture def remote_build_service(app_metadata, fake_services, fake_launchpad): class FakeRemoteBuildService(services.RemoteBuildService): RecipeClass = launchpad.models.SnapRecipe diff --git a/tests/unit/services/test_lifecycle.py b/tests/unit/services/test_lifecycle.py index 092b7391..2cd882a8 100644 --- a/tests/unit/services/test_lifecycle.py +++ b/tests/unit/services/test_lifecycle.py @@ -58,7 +58,7 @@ def _init_lifecycle_manager(self) -> LifecycleManager: return mock_lcm -@pytest.fixture() +@pytest.fixture def fake_parts_lifecycle( app_metadata, fake_project, fake_services, tmp_path, fake_build_plan ): diff --git a/tests/unit/services/test_provider.py b/tests/unit/services/test_provider.py index cd89d08c..68b20454 100644 --- a/tests/unit/services/test_provider.py +++ b/tests/unit/services/test_provider.py @@ -420,7 +420,7 @@ def test_load_bashrc_missing( ) -@pytest.fixture() +@pytest.fixture def setup_fetch_logs_provider(monkeypatch, provider_service, tmp_path): """Return a function that, when called, mocks the provider_service's instance().""" diff --git a/tests/unit/services/test_remotebuild.py b/tests/unit/services/test_remotebuild.py index 3b095802..fc5037c5 100644 --- a/tests/unit/services/test_remotebuild.py +++ b/tests/unit/services/test_remotebuild.py @@ -34,14 +34,14 @@ ) -@pytest.fixture() +@pytest.fixture def mock_push_url(monkeypatch): push_url = get_mock_callable(return_value=None) monkeypatch.setattr(git.GitRepo, "push_url", push_url) return push_url -@pytest.fixture() +@pytest.fixture def mock_push_url_raises_git_error(monkeypatch): push_url = get_mock_callable( side_effect=git.GitError("Fake push_url error during tests") @@ -50,7 +50,7 @@ def mock_push_url_raises_git_error(monkeypatch): return push_url -@pytest.fixture() +@pytest.fixture def mock_init_raises_git_error(monkeypatch): git_repo_init = get_mock_callable( side_effect=git.GitError("Fake _init_repo error during tests") @@ -59,7 +59,7 @@ def mock_init_raises_git_error(monkeypatch): return git_repo_init -@pytest.fixture() +@pytest.fixture def mock_lp_project(fake_launchpad, mock_project_entry): return launchpad.models.Project(fake_launchpad, mock_project_entry) diff --git a/tests/unit/services/test_service_factory.py b/tests/unit/services/test_service_factory.py index 40bf7f6b..1f4ca25c 100644 --- a/tests/unit/services/test_service_factory.py +++ b/tests/unit/services/test_service_factory.py @@ -22,7 +22,7 @@ from craft_cli import emit -@pytest.fixture() +@pytest.fixture def factory( app_metadata, fake_project, fake_package_service_class, fake_lifecycle_service_class ): diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index b8da1cd2..29871f01 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -391,7 +391,7 @@ def _extra_yaml_transform( return yaml_data -@pytest.fixture() +@pytest.fixture def app(app_metadata, fake_services): return FakeApplication(app_metadata, fake_services) @@ -416,12 +416,12 @@ def get_build_sources(self) -> set[str]: return set() -@pytest.fixture() +@pytest.fixture def fake_plugin(app_metadata, fake_services): return FakePlugin(app_metadata, fake_services) -@pytest.fixture() +@pytest.fixture def mock_dispatcher(monkeypatch): dispatcher = mock.Mock(spec_set=craft_cli.Dispatcher) monkeypatch.setattr("craft_cli.Dispatcher", mock.Mock(return_value=dispatcher)) @@ -1206,7 +1206,7 @@ def test_filter_plan(mocker, plan, platform, build_for, host_arch, result): assert application.filter_plan(plan, platform, build_for, host_arch) == result -@pytest.fixture() +@pytest.fixture def fake_project_file(monkeypatch, tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() @@ -1248,7 +1248,7 @@ def test_work_dir_project_managed(monkeypatch, app_metadata, fake_services): assert project.version == "1.0" -@pytest.fixture() +@pytest.fixture def environment_project(monkeypatch, tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() @@ -1334,7 +1334,7 @@ def test_application_expand_environment(app_metadata, fake_services): ] -@pytest.fixture() +@pytest.fixture def build_secrets_project(monkeypatch, tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() @@ -1508,7 +1508,7 @@ def test_mandatory_adoptable_fields(tmp_path, app_metadata, fake_services): ) -@pytest.fixture() +@pytest.fixture def grammar_project_mini(tmp_path): """A project that builds on amd64 to riscv64 and s390x.""" contents = dedent( @@ -1551,21 +1551,21 @@ def grammar_project_mini(tmp_path): project_file.write_text(contents) -@pytest.fixture() +@pytest.fixture def non_grammar_project_full(tmp_path): """A project that builds on amd64 to riscv64.""" project_file = tmp_path / "testcraft.yaml" project_file.write_text(FULL_PROJECT_YAML) -@pytest.fixture() +@pytest.fixture def grammar_project_full(tmp_path): """A project that builds on amd64 to riscv64 and s390x.""" project_file = tmp_path / "testcraft.yaml" project_file.write_text(FULL_GRAMMAR_PROJECT_YAML) -@pytest.fixture() +@pytest.fixture def non_grammar_build_plan(mocker): """A build plan to build on amd64 to riscv64.""" host_arch = "amd64" @@ -1582,7 +1582,7 @@ def non_grammar_build_plan(mocker): mocker.patch.object(models.BuildPlanner, "get_build_plan", return_value=build_plan) -@pytest.fixture() +@pytest.fixture def grammar_build_plan(mocker): """A build plan to build on amd64 to riscv64 and s390x.""" host_arch = "amd64" @@ -1600,7 +1600,7 @@ def grammar_build_plan(mocker): mocker.patch.object(models.BuildPlanner, "get_build_plan", return_value=build_plan) -@pytest.fixture() +@pytest.fixture def grammar_app_mini( tmp_path, grammar_project_mini, # noqa: ARG001 @@ -1614,7 +1614,7 @@ def grammar_app_mini( return app -@pytest.fixture() +@pytest.fixture def non_grammar_app_full( tmp_path, non_grammar_project_full, # noqa: ARG001 @@ -1628,7 +1628,7 @@ def non_grammar_app_full( return app -@pytest.fixture() +@pytest.fixture def grammar_app_full( tmp_path, grammar_project_full, # noqa: ARG001 @@ -1797,7 +1797,7 @@ def _setup_partitions(self, yaml_data) -> list[str]: return ["default", "mypartition"] -@pytest.fixture() +@pytest.fixture def environment_partitions_project(monkeypatch, tmp_path): project_dir = tmp_path / "project" project_dir.mkdir() diff --git a/tests/unit/test_secrets.py b/tests/unit/test_secrets.py index cb2b4a96..62f0d88a 100644 --- a/tests/unit/test_secrets.py +++ b/tests/unit/test_secrets.py @@ -19,7 +19,7 @@ from craft_application import errors, secrets -@pytest.fixture() +@pytest.fixture def good_yaml_data(): p1_data = { "source": "the source secret is $(HOST_SECRET:echo ${SECRET_1})", diff --git a/tests/unit/util/test_retry.py b/tests/unit/util/test_retry.py index c7c5a59a..e8e05daa 100644 --- a/tests/unit/util/test_retry.py +++ b/tests/unit/util/test_retry.py @@ -33,7 +33,7 @@ def always_raises(*_args, **_kwargs) -> None: raise MyError("raised an error!") -@pytest.fixture() +@pytest.fixture def mocked_sleep(mocker): return mocker.patch.object(time, "sleep") diff --git a/tests/unit/util/test_snap_config.py b/tests/unit/util/test_snap_config.py index de34cf85..cdd65e41 100644 --- a/tests/unit/util/test_snap_config.py +++ b/tests/unit/util/test_snap_config.py @@ -24,14 +24,14 @@ from snaphelpers import SnapCtlError -@pytest.fixture() +@pytest.fixture def mock_config(mocker): return mocker.patch( "craft_application.util.snap_config.SnapConfigOptions", autospec=True ) -@pytest.fixture() +@pytest.fixture def mock_is_running_from_snap(mocker): return mocker.patch( "craft_application.util.snap_config.is_running_from_snap", From df906c31a9d0c42e3cbc3a7a1b07d3395866dfa7 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 23 Aug 2024 17:55:34 -0400 Subject: [PATCH 19/82] fix(yaml): wrap error message for YAML errors (#427) --- craft_application/errors.py | 16 +++++++++++++ craft_application/util/yaml.py | 13 +++++++--- tests/unit/test_errors.py | 44 +++++++++++++++++++++++++++++++++- tests/unit/util/test_yaml.py | 11 +++++++-- 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/craft_application/errors.py b/craft_application/errors.py index 5b91d54f..81da65fb 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -23,6 +23,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING +import yaml from craft_cli import CraftError from craft_providers import bases @@ -42,6 +43,21 @@ class PathInvalidError(CraftError, OSError): """Error that the given path is not usable.""" +class YamlError(CraftError, yaml.YAMLError): + """Craft-cli friendly version of a YAML error.""" + + @classmethod + def from_yaml_error(cls, filename: str, error: yaml.YAMLError) -> Self: + """Convert a pyyaml YAMLError to a craft-application YamlError.""" + message = f"error parsing {filename!r}" + details = str(error) + return cls( + message, + details=details, + resolution=f"Ensure {filename} contains valid YAML", + ) + + class CraftValidationError(CraftError): """Error validating project yaml.""" diff --git a/craft_application/util/yaml.py b/craft_application/util/yaml.py index a649bded..2beac5eb 100644 --- a/craft_application/util/yaml.py +++ b/craft_application/util/yaml.py @@ -17,10 +17,13 @@ from __future__ import annotations import contextlib +import pathlib from typing import TYPE_CHECKING, Any, TextIO, cast, overload import yaml +from craft_application import errors + if TYPE_CHECKING: # pragma: no cover from collections.abc import Hashable @@ -99,9 +102,13 @@ def safe_yaml_load(stream: TextIO) -> Any: # noqa: ANN401 - The YAML could be a :param stream: Any text-like IO object. :returns: A dict object mapping the yaml. """ - # Silencing S506 ("probable use of unsafe loader") because we override it by using - # our own safe loader. - return yaml.load(stream, Loader=_SafeYamlLoader) # noqa: S506 + try: + # Silencing S506 ("probable use of unsafe loader") because we override it by + # using our own safe loader. + return yaml.load(stream, Loader=_SafeYamlLoader) # noqa: S506 + except yaml.YAMLError as error: + filename = pathlib.Path(stream.name).name + raise errors.YamlError.from_yaml_error(filename, error) from error @overload diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index 4edd335f..c7b46432 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -20,11 +20,53 @@ import pydantic import pytest import pytest_check -from craft_application.errors import CraftValidationError, PartsLifecycleError +import yaml +from craft_application.errors import ( + CraftValidationError, + PartsLifecycleError, + YamlError, +) from pydantic import BaseModel from typing_extensions import Self +@pytest.mark.parametrize( + ("original", "expected"), + [ + ( + yaml.YAMLError("I am a thing"), + YamlError( + "error parsing 'something.yaml'", + details="I am a thing", + resolution="Ensure something.yaml contains valid YAML", + ), + ), + ( + yaml.MarkedYAMLError( + problem="I am a thing", + problem_mark=yaml.error.Mark( + name="bork", + index=0, + line=0, + column=0, + buffer="Hello there", + pointer=0, + ), + ), + YamlError( + "error parsing 'something.yaml'", + details='I am a thing\n in "bork", line 1, column 1:\n Hello there\n ^', + resolution="Ensure something.yaml contains valid YAML", + ), + ), + ], +) +def test_yaml_error_from_yaml_error(original, expected): + actual = YamlError.from_yaml_error("something.yaml", original) + + assert actual == expected + + @pytest.mark.parametrize( "err", [ diff --git a/tests/unit/util/test_yaml.py b/tests/unit/util/test_yaml.py index 853a4c95..af4ea812 100644 --- a/tests/unit/util/test_yaml.py +++ b/tests/unit/util/test_yaml.py @@ -18,8 +18,9 @@ import pathlib import pytest +import pytest_check +from craft_application import errors from craft_application.util import yaml -from yaml.error import YAMLError TEST_DIR = pathlib.Path(__file__).parent @@ -39,9 +40,15 @@ def test_safe_yaml_loader_valid(file): ) def test_safe_yaml_loader_invalid(file): with file.open() as f: - with pytest.raises(YAMLError): + with pytest.raises( + errors.YamlError, match=f"error parsing {file.name!r}" + ) as exc_info: yaml.safe_yaml_load(f) + pytest_check.is_in(file.name, exc_info.value.resolution) + pytest_check.is_true(str(exc_info.value.resolution).endswith("contains valid YAML")) + pytest_check.is_in("found", exc_info.value.details) + @pytest.mark.parametrize( ("data", "kwargs", "expected"), From c98819926e0c440f07fcf94208e94cc7025f89ed Mon Sep 17 00:00:00 2001 From: Matt Culler Date: Mon, 26 Aug 2024 17:28:55 -0400 Subject: [PATCH 20/82] feat: tell the user which platforms matched when failing (#428) * feat: tell the user which platforms matched * fix: wrong quotes * tests: add unit test coverage for new error string * chore: formatting * chore: formatting * chore: use humanize_list, always end with period * refactor(tests): explicit result strings --- craft_application/errors.py | 12 ++++++++++-- craft_application/services/lifecycle.py | 2 +- tests/unit/services/test_lifecycle.py | 17 ++++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/craft_application/errors.py b/craft_application/errors.py index 81da65fb..7a8c2b7b 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -27,7 +27,9 @@ from craft_cli import CraftError from craft_providers import bases +from craft_application import models from craft_application.util.error_formatting import format_pydantic_errors +from craft_application.util.string import humanize_list if TYPE_CHECKING: # pragma: no cover import craft_parts @@ -157,8 +159,14 @@ def __init__(self) -> None: class MultipleBuildsError(CraftError): """The build plan contains multiple possible builds.""" - def __init__(self) -> None: - message = "Multiple builds match the current platform." + def __init__(self, matching_builds: list[models.BuildInfo] | None = None) -> None: + message = "Multiple builds match the current platform" + if matching_builds: + message += ": " + humanize_list( + [build.platform for build in matching_builds], + conjunction="and", + ) + message += "." resolution = 'Check the "--platform" and "--build-for" parameters.' super().__init__(message=message, resolution=resolution) diff --git a/craft_application/services/lifecycle.py b/craft_application/services/lifecycle.py index 86eaa51c..6e959b0e 100644 --- a/craft_application/services/lifecycle.py +++ b/craft_application/services/lifecycle.py @@ -425,7 +425,7 @@ def _validate_build_plan(build_plan: list[models.BuildInfo]) -> None: raise errors.EmptyBuildPlanError if len(build_plan) > 1: - raise errors.MultipleBuildsError + raise errors.MultipleBuildsError(matching_builds=build_plan) build_base = build_plan[0].base host_base = util.get_host_base() diff --git a/tests/unit/services/test_lifecycle.py b/tests/unit/services/test_lifecycle.py index 2cd882a8..99e3be54 100644 --- a/tests/unit/services/test_lifecycle.py +++ b/tests/unit/services/test_lifecycle.py @@ -822,11 +822,22 @@ def test_no_builds_error(fake_parts_lifecycle): fake_parts_lifecycle.run("prime") -@pytest.mark.parametrize("fake_build_plan", [2, 3, 4], indirect=True) -def test_multiple_builds_error(fake_parts_lifecycle): +@pytest.mark.parametrize( + ("fake_build_plan", "fake_result_str"), + [ + (2, "'foo' and 'foo'"), + (3, "'foo', 'foo', and 'foo'"), + (4, "'foo', 'foo', 'foo', and 'foo'"), + ], + indirect=["fake_build_plan"], +) +def test_multiple_builds_error(fake_parts_lifecycle, fake_result_str): """Build plan contains more than 1 item.""" - with pytest.raises(errors.MultipleBuildsError): + with pytest.raises(errors.MultipleBuildsError) as e: fake_parts_lifecycle.run("prime") + assert str(e.value) == ( + f"Multiple builds match the current platform: {fake_result_str}." + ) @pytest.mark.parametrize( From fac7490193563b50ae032d603065f75bd501a5f6 Mon Sep 17 00:00:00 2001 From: Matt Culler Date: Tue, 27 Aug 2024 14:00:30 -0400 Subject: [PATCH 21/82] docs(changelog): add release notes for 4.1.1 --- docs/reference/changelog.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 474d5fb8..469723c6 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,18 @@ Changelog ********* +4.1.1 (2024-Aug-27) +------------------- + +Application +=========== + +* When a build fails due to matching multiple platforms, those matching + platforms will be specified in the error message. +* Show nicer error messages for invalid YAML files. + +For a complete list of commits, check out the `4.1.1`_ release on GitHub. + 4.1.0 (2024-Aug-14) ------------------- @@ -195,3 +207,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _3.2.0: https://github.com/canonical/craft-application/releases/tag/3.2.0 .. _4.0.0: https://github.com/canonical/craft-application/releases/tag/4.0.0 .. _4.1.0: https://github.com/canonical/craft-application/releases/tag/4.1.0 +.. _4.1.1: https://github.com/canonical/craft-application/releases/tag/4.1.1 From 1509cec9af705c762b06725bf867ef1d9e1d822b Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 28 Aug 2024 14:03:46 -0400 Subject: [PATCH 22/82] docs: add environment variables reference (#431) --- docs/reference/environment-variables.rst | 111 +++++++++++++++++++++++ docs/reference/index.rst | 1 + 2 files changed, 112 insertions(+) create mode 100644 docs/reference/environment-variables.rst diff --git a/docs/reference/environment-variables.rst b/docs/reference/environment-variables.rst new file mode 100644 index 00000000..0fb29d8f --- /dev/null +++ b/docs/reference/environment-variables.rst @@ -0,0 +1,111 @@ +********************* +Environment variables +********************* + +Applications built on craft-application have several environment variables that +can configure their behaviour. They and the behaviour they modify are listed +below. + +Variables passed to managed builders +------------------------------------ + +Several environment variables from the host environment are passed to the +managed build environment. While an application may adjust these by adjusting +the ``environment`` dictionary attached to the ``ProviderService``, +craft-application will by default forward the ``http_proxy``, ``https_proxy`` +and ``no_proxy`` environment variables from the host. + +Supported variables +------------------- + +These variables are explicitly supported for user configuration. + +.. _env-var-craft-build-environment: + +``CRAFT_BUILD_ENVIRONMENT`` +=========================== + +If the value is ``host``, allows an environment to tell a craft application +to run directly on the host rather than in managed mode. This method is +roughly equivalent to using ``--destructive-mode``, but is designed for +configurations where the application is already being run in an appropriate +container or VM, such as +`Snapcraft rocks `_ or +when controlled by a CI system such as `Launchpad `_. + +**CAUTION**: Setting the build environment is only recommended if you are +aware of the exact packages needed to reproduce the build containers created +by the app. + +``CRAFT_BUILD_FOR`` +=================== + +Sets the default architecture to build for. Overridden by ``--build-for`` in +lifecycle commands. + +``CRAFT_PLATFORM`` +================== + +Sets the default platform to build. Overridden by ``--platform`` in lifecycle +commands. + +``CRAFT_SNAP_CHANNEL`` +====================== + +Overrides the default channel that a craft application's snap is installed from +if the manager instance is not running as a snap. If unset, the application +will be installed from the ``latest/stable`` channel. If the application is +running from a snap, this variable is ignored and the same snap used on +the host system is injected into the managed builder. + +``CRAFT_VERBOSITY_LEVEL`` +========================= + +Set the verbosity level for the application. Valid values are: ``quiet``, +``brief``, ``verbose``, ``debug`` and ``trace``. This is overridden by the +``--quiet``, ``--verbose`` or ``--verbosity={value}`` global command options. + +Development variables +--------------------- + +The following variables exist to help developers writing applications using +craft-application more easily debug their code: + +``CRAFT_DEBUG`` +=============== + +Controls whether the application is in debug mode. If this variable is set to +``1``, general exceptions will not be caught, instead showing a traceback on +the command line. This is normally only useful for developers working on +craft-application or an app that uses the framework, as a traceback is always +written to the log file as well. + +``CRAFT_LAUNCHPAD_INSTANCE`` +============================ + +For remote builds, allows the user to set an alternative launchpad instance. +Accepts any string that can be used as the ``service_root`` value in +`Launchpadlib `_. + +Unsupported variables +--------------------- + +The following variables cause behaviour changes in craft-application, but +should not be set except by craft-application itself. + +``CRAFT_LXD_REMOTE`` +==================== + +If using LXD, the application will start containers in the configured remote +rather than ``local``. + +**CAUTION:** Using non-default remotes is experimental and not recommended at +this time. + +``CRAFT_MANAGED_MODE`` +====================== + +Alerts the application that it is running in managed mode. This should only be +set by craft-application when creating a provider. Systems designed to wrap +craft applications may use the :ref:`env-var-craft-build-environment` +environment variable to make the app run on the host. diff --git a/docs/reference/index.rst b/docs/reference/index.rst index b188a0d4..1e7796c6 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -7,6 +7,7 @@ Reference :maxdepth: 1 changelog + environment-variables platforms Indices and tables From de48adc3950e21bcec4f14e95608a59dbd97bb82 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 29 Aug 2024 16:01:00 -0400 Subject: [PATCH 23/82] fix: provide more info in YAML errors (#433) --- craft_application/errors.py | 2 ++ tests/unit/test_errors.py | 2 +- tests/unit/util/test_yaml.py | 25 ++++++++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/craft_application/errors.py b/craft_application/errors.py index 7a8c2b7b..8c85516a 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -52,6 +52,8 @@ class YamlError(CraftError, yaml.YAMLError): def from_yaml_error(cls, filename: str, error: yaml.YAMLError) -> Self: """Convert a pyyaml YAMLError to a craft-application YamlError.""" message = f"error parsing {filename!r}" + if isinstance(error, yaml.MarkedYAMLError): + message += f": {error.problem}" details = str(error) return cls( message, diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index c7b46432..830efd5d 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -54,7 +54,7 @@ ), ), YamlError( - "error parsing 'something.yaml'", + "error parsing 'something.yaml': I am a thing", details='I am a thing\n in "bork", line 1, column 1:\n Hello there\n ^', resolution="Ensure something.yaml contains valid YAML", ), diff --git a/tests/unit/util/test_yaml.py b/tests/unit/util/test_yaml.py index af4ea812..a717d8ab 100644 --- a/tests/unit/util/test_yaml.py +++ b/tests/unit/util/test_yaml.py @@ -41,7 +41,7 @@ def test_safe_yaml_loader_valid(file): def test_safe_yaml_loader_invalid(file): with file.open() as f: with pytest.raises( - errors.YamlError, match=f"error parsing {file.name!r}" + errors.YamlError, match=f"error parsing {file.name!r}: " ) as exc_info: yaml.safe_yaml_load(f) @@ -50,6 +50,29 @@ def test_safe_yaml_loader_invalid(file): pytest_check.is_in("found", exc_info.value.details) +@pytest.mark.parametrize( + ("yaml_text", "error_msg"), + [ + ( + "thing: \nthing:\n", + "error parsing 'testcraft.yaml': found duplicate key 'thing'", + ), + ( + "{{unhashable}}:", + "error parsing 'testcraft.yaml': found unhashable key", + ), + ], +) +def test_safe_yaml_loader_specific_error(yaml_text: str, error_msg: str): + f = io.StringIO(yaml_text) + f.name = "testcraft.yaml" + + with pytest.raises(errors.YamlError) as exc_info: + yaml.safe_yaml_load(f) + + assert exc_info.value.args[0] == error_msg + + @pytest.mark.parametrize( ("data", "kwargs", "expected"), [ From a49a3ea988371dbe018b218f53e8f1c737af2e0d Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 29 Aug 2024 21:39:07 -0400 Subject: [PATCH 24/82] feat(provider): use standard library to get proxies (#426) Fixes #422 --- craft_application/services/provider.py | 10 +++-- tests/unit/services/test_provider.py | 53 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/craft_application/services/provider.py b/craft_application/services/provider.py index 7ca9e51c..fe12c012 100644 --- a/craft_application/services/provider.py +++ b/craft_application/services/provider.py @@ -22,6 +22,8 @@ import pathlib import pkgutil import sys +import urllib.request +from collections.abc import Generator, Iterable from pathlib import Path from typing import TYPE_CHECKING @@ -36,8 +38,6 @@ from craft_application.util import platforms, snap_config if TYPE_CHECKING: # pragma: no cover - from collections.abc import Generator - import craft_providers from craft_application import models @@ -45,7 +45,7 @@ from craft_application.services import ServiceFactory -DEFAULT_FORWARD_ENVIRONMENT_VARIABLES = ("http_proxy", "https_proxy", "no_proxy") +DEFAULT_FORWARD_ENVIRONMENT_VARIABLES: Iterable[str] = () class ProviderService(base.ProjectService): @@ -93,6 +93,10 @@ def setup(self) -> None: if name in os.environ: self.environment[name] = os.getenv(name) + for scheme, value in urllib.request.getproxies().items(): + self.environment[f"{scheme.lower()}_proxy"] = value + self.environment[f"{scheme.upper()}_PROXY"] = value + if self._install_snap: channel = ( None diff --git a/tests/unit/services/test_provider.py b/tests/unit/services/test_provider.py index 68b20454..17f71814 100644 --- a/tests/unit/services/test_provider.py +++ b/tests/unit/services/test_provider.py @@ -30,6 +30,59 @@ from craft_providers.actions.snap_installer import Snap +@pytest.mark.parametrize( + ("given_environment", "expected_environment"), + [ + ({}, {}), + ({"http_proxy": "thing"}, {"http_proxy": "thing", "HTTP_PROXY": "thing"}), + ({"HTTP_PROXY": "thing"}, {"http_proxy": "thing", "HTTP_PROXY": "thing"}), + ({"ssh_proxy": "thing"}, {"ssh_proxy": "thing", "SSH_PROXY": "thing"}), + ({"no_proxy": "thing"}, {"no_proxy": "thing", "NO_PROXY": "thing"}), + ({"NO_PROXY": "thing"}, {"no_proxy": "thing", "NO_PROXY": "thing"}), + # Special case handled by upstream: + # https://docs.python.org/3/library/urllib.request.html#urllib.request.getproxies + ( + { + "REQUEST_METHOD": "GET", + "HTTP_PROXY": "thing", + }, + {}, + ), + ( # But lower-case http_proxy is still allowed + { + "REQUEST_METHOD": "GET", + "http_proxy": "thing", + }, + {"http_proxy": "thing", "HTTP_PROXY": "thing"}, + ), + ], +) +def test_setup_proxy_environment( + monkeypatch: pytest.MonkeyPatch, + app_metadata, + fake_services, + fake_project, + fake_build_plan, + given_environment: dict[str, str], + expected_environment: dict[str, str], +): + for var, value in given_environment.items(): + monkeypatch.setenv(var, value) + + expected_environment |= {"CRAFT_MANAGED_MODE": "1"} + + service = provider.ProviderService( + app_metadata, + fake_services, + project=fake_project, + work_dir=pathlib.Path(), + build_plan=fake_build_plan, + ) + service.setup() + + assert service.environment == expected_environment + + @pytest.mark.parametrize( ("install_snap", "environment", "snaps"), [ From 5e7560e4993855331f70a1671103c92b116e0114 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 2 Sep 2024 16:43:30 -0300 Subject: [PATCH 25/82] fix: fail managed runs if the build plan is empty This fix addresses cases like, for example, a project that only builds on `amd64` being built on `riscv64`. Previously the call to run_managed() would loop over the (empty) build plan and then finish "successfully". This new version will raise an EmptyBuildPlanError indicating to the user that they should check the project's 'platforms' declaration and the command-line parameters. Fixes #225 --- craft_application/application.py | 4 +++- craft_application/errors.py | 7 +++++-- tests/unit/test_application.py | 8 ++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index 89471684..14ed25eb 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -336,8 +336,10 @@ def is_managed(self) -> bool: def run_managed(self, platform: str | None, build_for: str | None) -> None: """Run the application in a managed instance.""" - extra_args: dict[str, Any] = {} + if not self._build_plan: + raise errors.EmptyBuildPlanError + extra_args: dict[str, Any] = {} for build_info in self._build_plan: if platform and platform != build_info.platform: continue diff --git a/craft_application/errors.py b/craft_application/errors.py index 8c85516a..cb209a18 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -152,8 +152,11 @@ class EmptyBuildPlanError(CraftError): """The build plan filtered out all possible builds.""" def __init__(self) -> None: - message = "No build matches the current platform." - resolution = 'Check the "--platform" and "--build-for" parameters.' + message = "No build matches the current execution environment." + resolution = ( + "Check the project's 'platforms' declaration, and the " + "'--platform' and '--build-for' parameters." + ) super().__init__(message=message, resolution=resolution) diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 29871f01..2c0ea3d8 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -606,6 +606,14 @@ def test_run_managed_specified_platform(app, fake_project): assert mock.call(info1, work_dir=mock.ANY) not in mock_provider.instance.mock_calls +def test_run_managed_empty_plan(app, fake_project): + app.set_project(fake_project) + + app._build_plan = [] + with pytest.raises(errors.EmptyBuildPlanError): + app.run_managed(None, None) + + @pytest.mark.parametrize( ("managed", "error", "exit_code", "message"), [ From ab55ec81729acb34680760ebe70f948529ff3d08 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Wed, 4 Sep 2024 17:47:09 -0300 Subject: [PATCH 26/82] feat: support shell/shell-after and debug in 'pack' (#436) `--shell` and `--shell-after` are implemented for a) consistency with the other lifecycle commands, and b) because it makes sense to want to inspect the build environment at packing time, considering that there are packing-related steps, like writing the metadata file, that the users don't have a way to inspect otherwise (short of examining the final artefact). `--debug` is improved to actually shell into the build environment when the packing itself failed, as opposed to the previous behavior of only debugging failures that happen during the lifecycle steps. Fixes #430 --------- Co-authored-by: Alex Lowe --- craft_application/commands/lifecycle.py | 31 +++++++-- tests/unit/commands/test_lifecycle.py | 93 ++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/craft_application/commands/lifecycle.py b/craft_application/commands/lifecycle.py index f5754394..4c67108e 100644 --- a/craft_application/commands/lifecycle.py +++ b/craft_application/commands/lifecycle.py @@ -367,13 +367,32 @@ def _run( """Run the pack command.""" if step_name not in ("pack", None): raise RuntimeError(f"Step name {step_name} passed to pack command.") + + shell = getattr(parsed_args, "shell", False) + shell_after = getattr(parsed_args, "shell_after", False) + debug = getattr(parsed_args, "debug", False) + + # Prevent the steps in the prime command from using `--shell` or `--shell-after` + parsed_args.shell = False + parsed_args.shell_after = False + super()._run(parsed_args, step_name="prime") self._run_post_prime_steps() + if shell: + _launch_shell() + return + emit.progress("Packing...") - packages = self._services.package.pack( - self._services.lifecycle.prime_dir, parsed_args.output - ) + try: + packages = self._services.package.pack( + self._services.lifecycle.prime_dir, parsed_args.output + ) + except Exception as err: + if debug: + emit.progress(str(err), permanent=True) + _launch_shell() + raise if not packages: emit.progress("No packages created.", permanent=True) @@ -383,10 +402,8 @@ def _run( package_names = ", ".join(pkg.name for pkg in packages) emit.progress(f"Packed: {package_names}", permanent=True) - @staticmethod - @override - def _should_add_shell_args() -> bool: - return False + if shell_after: + _launch_shell() class CleanCommand(_BaseLifecycleCommand): diff --git a/tests/unit/commands/test_lifecycle.py b/tests/unit/commands/test_lifecycle.py index 5be67dea..7a643e54 100644 --- a/tests/unit/commands/test_lifecycle.py +++ b/tests/unit/commands/test_lifecycle.py @@ -414,6 +414,7 @@ def test_clean_run_managed( @pytest.mark.parametrize(("build_env_dict", "build_env_args"), BUILD_ENV_COMMANDS) +@pytest.mark.parametrize(("shell_dict", "shell_args"), SHELL_PARAMS) @pytest.mark.parametrize(("debug_dict", "debug_args"), DEBUG_PARAMS) @pytest.mark.parametrize("output_arg", [".", "/"]) def test_pack_fill_parser( @@ -421,6 +422,8 @@ def test_pack_fill_parser( mock_services, build_env_dict, build_env_args, + shell_dict, + shell_args, debug_dict, debug_args, output_arg, @@ -430,6 +433,7 @@ def test_pack_fill_parser( "platform": None, "build_for": None, "output": pathlib.Path(output_arg), + **shell_dict, **debug_dict, **build_env_dict, } @@ -438,7 +442,9 @@ def test_pack_fill_parser( command.fill_parser(parser) args_dict = vars( - parser.parse_args([*build_env_args, *debug_args, f"--output={output_arg}"]) + parser.parse_args( + [*build_env_args, *shell_args, *debug_args, f"--output={output_arg}"] + ) ) assert args_dict == expected @@ -531,6 +537,34 @@ def test_shell( mock_subprocess_run.assert_called_once_with(["bash"], check=False) +def test_shell_pack( + app_metadata, + fake_services, + mocker, + mock_subprocess_run, +): + parsed_args = argparse.Namespace(shell=True) + mock_lifecycle_run = mocker.patch.object(fake_services.lifecycle, "run") + mock_pack = mocker.patch.object(fake_services.package, "pack") + mocker.patch.object( + fake_services.lifecycle.project_info, "execution_finished", return_value=True + ) + command = PackCommand( + { + "app": app_metadata, + "services": fake_services, + } + ) + command.run(parsed_args) + + # Must run the lifecycle + mock_lifecycle_run.assert_called_once_with(step_name="prime") + + # Must call the shell instead of packing + mock_subprocess_run.assert_called_once_with(["bash"], check=False) + assert not mock_pack.called + + @pytest.mark.parametrize("command_cls", MANAGED_LIFECYCLE_COMMANDS) def test_shell_after( app_metadata, fake_services, mocker, mock_subprocess_run, command_cls @@ -554,6 +588,33 @@ def test_shell_after( mock_subprocess_run.assert_called_once_with(["bash"], check=False) +def test_shell_after_pack( + app_metadata, + fake_services, + mocker, + mock_subprocess_run, +): + parsed_args = argparse.Namespace(shell_after=True, output=pathlib.Path()) + mock_lifecycle_run = mocker.patch.object(fake_services.lifecycle, "run") + mock_pack = mocker.patch.object(fake_services.package, "pack") + mocker.patch.object( + fake_services.lifecycle.project_info, "execution_finished", return_value=True + ) + command = PackCommand( + { + "app": app_metadata, + "services": fake_services, + } + ) + command.run(parsed_args) + + # Must run the lifecycle + mock_lifecycle_run.assert_called_once_with(step_name="prime") + # Must pack, and then shell + mock_pack.assert_called_once_with(fake_services.lifecycle.prime_dir, pathlib.Path()) + mock_subprocess_run.assert_called_once_with(["bash"], check=False) + + @pytest.mark.parametrize("command_cls", [*MANAGED_LIFECYCLE_COMMANDS, PackCommand]) def test_debug(app_metadata, fake_services, mocker, mock_subprocess_run, command_cls): parsed_args = argparse.Namespace(parts=None, debug=True) @@ -574,3 +635,33 @@ def test_debug(app_metadata, fake_services, mocker, mock_subprocess_run, command command.run(parsed_args) mock_subprocess_run.assert_called_once_with(["bash"], check=False) + + +def test_debug_pack( + app_metadata, + fake_services, + mocker, + mock_subprocess_run, +): + """Same as test_debug(), but checking when the error happens when packing.""" + parsed_args = argparse.Namespace(debug=True, output=pathlib.Path()) + error_message = "Packing failed!" + + # Lifecycle.run() should work + mocker.patch.object(fake_services.lifecycle, "run") + # Package.pack() should fail + mocker.patch.object( + fake_services.package, "pack", side_effect=RuntimeError(error_message) + ) + mocker.patch.object(fake_services.package, "update_project") + command = PackCommand( + { + "app": app_metadata, + "services": fake_services, + } + ) + + with pytest.raises(RuntimeError, match=error_message): + command.run(parsed_args) + + mock_subprocess_run.assert_called_once_with(["bash"], check=False) From d69789871fb93285ce677f5fa8a427465a1ce7a1 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 5 Sep 2024 16:07:48 -0400 Subject: [PATCH 27/82] fix(platform): automatically vectorise architectures (#442) This is a fix for https://github.com/canonical/charmcraft/issues/1874 The spec (ST105) allows | in both build-on and build-for on all craft apps. This change makes the Platform model accept a string value, but doesn't specify that in the schema. This means text editors using the schema will still suggest converting the build-on and build-for values to lists. --- craft_application/models/project.py | 8 +++++++ tests/unit/models/test_project.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/craft_application/models/project.py b/craft_application/models/project.py index 550f00e1..81f1acfe 100644 --- a/craft_application/models/project.py +++ b/craft_application/models/project.py @@ -93,6 +93,14 @@ class Platform(base.CraftBaseModel): build_on: UniqueList[str] | None = pydantic.Field(min_length=1) build_for: SingleEntryList[str] | None = None + @pydantic.field_validator("build_on", "build_for", mode="before") + @classmethod + def _vectorise_architectures(cls, values: str | list[str]) -> list[str]: + """Convert string build-on and build-for to lists.""" + if isinstance(values, str): + return [values] + return values + @pydantic.field_validator("build_on", "build_for", mode="after") @classmethod def _validate_architectures(cls, values: list[str]) -> list[str]: diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index ab0f21b6..ca20a2aa 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -30,9 +30,11 @@ DEVEL_BASE_WARNING, BuildInfo, BuildPlanner, + Platform, Project, constraints, ) +from craft_application.util import platforms PROJECTS_DIR = pathlib.Path(__file__).parent / "project_models" PARTS_DICT = {"my-part": {"plugin": "nil"}} @@ -121,6 +123,38 @@ def full_project_dict(): return copy.deepcopy(FULL_PROJECT_DICT) +@pytest.mark.parametrize( + ("incoming", "expected"), + [ + *( + pytest.param( + {"build-on": arch, "build-for": arch}, + Platform(build_on=[arch], build_for=[arch]), + id=arch, + ) + for arch in platforms._ARCH_TRANSLATIONS_DEB_TO_PLATFORM + ), + *( + pytest.param( + {"build-on": arch}, + Platform(build_on=[arch]), + id=f"build-on-only-{arch}", + ) + for arch in platforms._ARCH_TRANSLATIONS_DEB_TO_PLATFORM + ), + pytest.param( + {"build-on": "amd64", "build-for": "riscv64"}, + Platform(build_on=["amd64"], build_for=["riscv64"]), + id="cross-compile", + ), + ], +) +def test_platform_vectorise_architectures(incoming, expected): + platform = Platform.model_validate(incoming) + + assert platform == expected + + @pytest.mark.parametrize( ("project_fixture", "project_dict"), [("basic_project", BASIC_PROJECT_DICT), ("full_project", FULL_PROJECT_DICT)], From 2fe26ce01a3130423190ebd8a5e35388fef29222 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 2 Sep 2024 16:43:30 -0300 Subject: [PATCH 28/82] fix: fail managed runs if the build plan is empty This fix addresses cases like, for example, a project that only builds on `amd64` being built on `riscv64`. Previously the call to run_managed() would loop over the (empty) build plan and then finish "successfully". This new version will raise an EmptyBuildPlanError indicating to the user that they should check the project's 'platforms' declaration and the command-line parameters. Fixes #225 --- craft_application/application.py | 4 +++- craft_application/errors.py | 7 +++++-- tests/unit/test_application.py | 8 ++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index 89471684..14ed25eb 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -336,8 +336,10 @@ def is_managed(self) -> bool: def run_managed(self, platform: str | None, build_for: str | None) -> None: """Run the application in a managed instance.""" - extra_args: dict[str, Any] = {} + if not self._build_plan: + raise errors.EmptyBuildPlanError + extra_args: dict[str, Any] = {} for build_info in self._build_plan: if platform and platform != build_info.platform: continue diff --git a/craft_application/errors.py b/craft_application/errors.py index 7a8c2b7b..e6c55eb7 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -150,8 +150,11 @@ class EmptyBuildPlanError(CraftError): """The build plan filtered out all possible builds.""" def __init__(self) -> None: - message = "No build matches the current platform." - resolution = 'Check the "--platform" and "--build-for" parameters.' + message = "No build matches the current execution environment." + resolution = ( + "Check the project's 'platforms' declaration, and the " + "'--platform' and '--build-for' parameters." + ) super().__init__(message=message, resolution=resolution) diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 29871f01..2c0ea3d8 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -606,6 +606,14 @@ def test_run_managed_specified_platform(app, fake_project): assert mock.call(info1, work_dir=mock.ANY) not in mock_provider.instance.mock_calls +def test_run_managed_empty_plan(app, fake_project): + app.set_project(fake_project) + + app._build_plan = [] + with pytest.raises(errors.EmptyBuildPlanError): + app.run_managed(None, None) + + @pytest.mark.parametrize( ("managed", "error", "exit_code", "message"), [ From 365cf2f59baa7828b9a46cf48d8c59b8b3831521 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 29 Aug 2024 16:01:00 -0400 Subject: [PATCH 29/82] fix: provide more info in YAML errors (#433) --- craft_application/errors.py | 2 ++ tests/unit/test_errors.py | 2 +- tests/unit/util/test_yaml.py | 25 ++++++++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/craft_application/errors.py b/craft_application/errors.py index e6c55eb7..cb209a18 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -52,6 +52,8 @@ class YamlError(CraftError, yaml.YAMLError): def from_yaml_error(cls, filename: str, error: yaml.YAMLError) -> Self: """Convert a pyyaml YAMLError to a craft-application YamlError.""" message = f"error parsing {filename!r}" + if isinstance(error, yaml.MarkedYAMLError): + message += f": {error.problem}" details = str(error) return cls( message, diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index c7b46432..830efd5d 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -54,7 +54,7 @@ ), ), YamlError( - "error parsing 'something.yaml'", + "error parsing 'something.yaml': I am a thing", details='I am a thing\n in "bork", line 1, column 1:\n Hello there\n ^', resolution="Ensure something.yaml contains valid YAML", ), diff --git a/tests/unit/util/test_yaml.py b/tests/unit/util/test_yaml.py index af4ea812..a717d8ab 100644 --- a/tests/unit/util/test_yaml.py +++ b/tests/unit/util/test_yaml.py @@ -41,7 +41,7 @@ def test_safe_yaml_loader_valid(file): def test_safe_yaml_loader_invalid(file): with file.open() as f: with pytest.raises( - errors.YamlError, match=f"error parsing {file.name!r}" + errors.YamlError, match=f"error parsing {file.name!r}: " ) as exc_info: yaml.safe_yaml_load(f) @@ -50,6 +50,29 @@ def test_safe_yaml_loader_invalid(file): pytest_check.is_in("found", exc_info.value.details) +@pytest.mark.parametrize( + ("yaml_text", "error_msg"), + [ + ( + "thing: \nthing:\n", + "error parsing 'testcraft.yaml': found duplicate key 'thing'", + ), + ( + "{{unhashable}}:", + "error parsing 'testcraft.yaml': found unhashable key", + ), + ], +) +def test_safe_yaml_loader_specific_error(yaml_text: str, error_msg: str): + f = io.StringIO(yaml_text) + f.name = "testcraft.yaml" + + with pytest.raises(errors.YamlError) as exc_info: + yaml.safe_yaml_load(f) + + assert exc_info.value.args[0] == error_msg + + @pytest.mark.parametrize( ("data", "kwargs", "expected"), [ From c8ccf47191354293b28e368e76cb6021038245fc Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 5 Sep 2024 16:19:49 -0400 Subject: [PATCH 30/82] docs: add 4.1.2 to changelog --- docs/reference/changelog.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 469723c6..2b5be99d 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,22 @@ Changelog ********* +4.1.2 (2024-Sep-05) +------------------- + +Application +=========== + +- Managed runs now fail if the build plan is empty. +- Error message tweaks for invalid YAML files. + +Models +====== + +- Platform models now correctly accept non-vectorised architectures. + +For a complete list of commits, check out the `4.1.2`_ release on GitHub. + 4.1.1 (2024-Aug-27) ------------------- @@ -208,3 +224,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _4.0.0: https://github.com/canonical/craft-application/releases/tag/4.0.0 .. _4.1.0: https://github.com/canonical/craft-application/releases/tag/4.1.0 .. _4.1.1: https://github.com/canonical/craft-application/releases/tag/4.1.1 +.. _4.1.2: https://github.com/canonical/craft-application/releases/tag/4.1.2 From bbb1d72b20af081c6c374ba85c1adc4c0a58c37c Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 5 Sep 2024 21:23:12 -0400 Subject: [PATCH 31/82] feat: replace ServiceFactory.set_kwargs with ServiceFactory.update_kwargs (#440) --- craft_application/application.py | 4 +- craft_application/services/service_factory.py | 26 +++++++++- tests/unit/services/test_service_factory.py | 47 ++++++++++++++++++- 3 files changed, 73 insertions(+), 4 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index 14ed25eb..e98ec14c 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -223,14 +223,14 @@ def _configure_services(self, provider_name: str | None) -> None: Any child classes that override this must either call this directly or must provide a valid ``project`` to ``self.services``. """ - self.services.set_kwargs( + self.services.update_kwargs( "lifecycle", cache_dir=self.cache_dir, work_dir=self._work_dir, build_plan=self._build_plan, partitions=self._partitions, ) - self.services.set_kwargs( + self.services.update_kwargs( "provider", work_dir=self._work_dir, build_plan=self._build_plan, diff --git a/craft_application/services/service_factory.py b/craft_application/services/service_factory.py index d00e4be6..79b85222 100644 --- a/craft_application/services/service_factory.py +++ b/craft_application/services/service_factory.py @@ -15,6 +15,7 @@ from __future__ import annotations import dataclasses +import warnings from typing import TYPE_CHECKING, Any from craft_application import models, services @@ -61,9 +62,32 @@ def set_kwargs( service: str, **kwargs: Any, # noqa: ANN401 this is intentionally duck-typed. ) -> None: - """Set up the keyword arguments to pass to a particular service class.""" + """Set up the keyword arguments to pass to a particular service class. + + PENDING DEPRECATION: use update_kwargs instead + """ + warnings.warn( + PendingDeprecationWarning( + "ServiceFactory.set_kwargs is pending deprecation. Use update_kwargs instead." + ), + stacklevel=2, + ) self._service_kwargs[service] = kwargs + def update_kwargs( + self, + service: str, + **kwargs: Any, # noqa: ANN401 this is intentionally duck-typed. + ) -> None: + """Update the keyword arguments to pass to a particular service class. + + This works like ``dict.update()``, overwriting already-set values. + + :param service: the name of the service (e.g. "lifecycle") + :param kwargs: keyword arguments to set. + """ + self._service_kwargs.setdefault(service, {}).update(kwargs) + def __getattr__(self, service: str) -> services.AppService: """Instantiate a service class. diff --git a/tests/unit/services/test_service_factory.py b/tests/unit/services/test_service_factory.py index 1f4ca25c..223296e8 100644 --- a/tests/unit/services/test_service_factory.py +++ b/tests/unit/services/test_service_factory.py @@ -74,7 +74,8 @@ def __new__(cls, *args, **kwargs): app_metadata, project=fake_project, PackageClass=MockPackageService ) - factory.set_kwargs("package", **kwargs) + with pytest.warns(PendingDeprecationWarning): + factory.set_kwargs("package", **kwargs) check.equal(factory.package, MockPackageService.mock_class.return_value) with check: @@ -83,6 +84,50 @@ def __new__(cls, *args, **kwargs): ) +@pytest.mark.parametrize( + ("first_kwargs", "second_kwargs", "expected"), + [ + ({}, {}, {}), + ( + {"arg_1": None}, + {"arg_b": "something"}, + {"arg_1": None, "arg_b": "something"}, + ), + ( + {"overridden": False}, + {"overridden": True}, + {"overridden": True}, + ), + ], +) +def test_update_kwargs( + app_metadata, + fake_project, + fake_package_service_class, + first_kwargs, + second_kwargs, + expected, +): + class MockPackageService(fake_package_service_class): + mock_class = mock.Mock(return_value=mock.Mock(spec_set=services.PackageService)) + + def __new__(cls, *args, **kwargs): + return cls.mock_class(*args, **kwargs) + + factory = services.ServiceFactory( + app_metadata, project=fake_project, PackageClass=MockPackageService + ) + + factory.update_kwargs("package", **first_kwargs) + factory.update_kwargs("package", **second_kwargs) + + pytest_check.is_(factory.package, MockPackageService.mock_class.return_value) + with pytest_check.check(): + MockPackageService.mock_class.assert_called_once_with( + app=app_metadata, services=factory, project=fake_project, **expected + ) + + def test_getattr_cached_service(monkeypatch, check, factory): mock_getattr = mock.Mock(wraps=factory.__getattr__) monkeypatch.setattr(services.ServiceFactory, "__getattr__", mock_getattr) From 4d7bc64d502135c2327834f393392b40f4e5d385 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Thu, 5 Sep 2024 23:27:00 -0400 Subject: [PATCH 32/82] feat: add a configuration service (#416) Adds a configuration service that can be used to get application configuration. CRAFT_* environment variables are only used if the config item is known to craft-application (not for app-specific config). Co-authored-by: Dariusz Duda --- craft_application/__init__.py | 2 + craft_application/_config.py | 37 +++ craft_application/application.py | 24 +- craft_application/commands/lifecycle.py | 2 - craft_application/launchpad/__init__.py | 2 + craft_application/services/__init__.py | 2 + craft_application/services/config.py | 198 +++++++++++++++ craft_application/services/provider.py | 12 +- craft_application/services/remotebuild.py | 6 +- craft_application/services/service_factory.py | 2 + pyproject.toml | 1 + tests/conftest.py | 28 ++- tests/integration/test_application.py | 3 +- tests/unit/services/test_config.py | 227 ++++++++++++++++++ tests/unit/test_application.py | 3 + 15 files changed, 527 insertions(+), 22 deletions(-) create mode 100644 craft_application/_config.py create mode 100644 craft_application/services/config.py create mode 100644 tests/unit/services/test_config.py diff --git a/craft_application/__init__.py b/craft_application/__init__.py index b7bd325b..8848d94b 100644 --- a/craft_application/__init__.py +++ b/craft_application/__init__.py @@ -25,6 +25,7 @@ ProviderService, ServiceFactory, ) +from craft_application._config import ConfigModel try: from ._version import __version__ @@ -42,6 +43,7 @@ "AppFeatures", "AppMetadata", "AppService", + "ConfigModel", "models", "ProjectService", "LifecycleService", diff --git a/craft_application/_config.py b/craft_application/_config.py new file mode 100644 index 00000000..c9619e12 --- /dev/null +++ b/craft_application/_config.py @@ -0,0 +1,37 @@ +# This file is part of craft-application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +"""Configuration model for craft applications.""" +from __future__ import annotations + +import craft_cli +import pydantic + + +class ConfigModel(pydantic.BaseModel): + """A configuration model for a craft application.""" + + verbosity_level: craft_cli.EmitterMode = craft_cli.EmitterMode.BRIEF + debug: bool = False + build_environment: str | None = None + secrets: str + + platform: str | None = None + build_for: str | None = None + + parallel_build_count: int + max_parallel_build_count: int + lxd_remote: str = "local" + launchpad_instance: str = "production" diff --git a/craft_application/application.py b/craft_application/application.py index e98ec14c..3a65cfb6 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -17,6 +17,7 @@ from __future__ import annotations +import argparse import importlib import os import pathlib @@ -35,7 +36,7 @@ from craft_parts.plugins.plugins import PluginType from platformdirs import user_cache_path -from craft_application import commands, errors, grammar, models, secrets, util +from craft_application import _config, commands, errors, grammar, models, secrets, util from craft_application.errors import PathInvalidError from craft_application.models import BuildInfo, GrammarAwareProject @@ -79,6 +80,7 @@ class AppMetadata: features: AppFeatures = AppFeatures() project_variables: list[str] = field(default_factory=lambda: ["version"]) mandatory_adoptable_fields: list[str] = field(default_factory=lambda: ["version"]) + ConfigModel: type[_config.ConfigModel] = _config.ConfigModel ProjectClass: type[models.Project] = models.Project BuildPlannerClass: type[models.BuildPlanner] = models.BuildPlanner @@ -431,7 +433,7 @@ def _get_dispatcher(self) -> craft_cli.Dispatcher: f"Internal error while loading {self.app.name}: {err!r}" ) ) - if os.getenv("CRAFT_DEBUG") == "1": + if self.services.config.get("debug"): raise sys.exit(os.EX_SOFTWARE) @@ -493,6 +495,17 @@ def _pre_run(self, dispatcher: craft_cli.Dispatcher) -> None: resolution="Ensure the path entered is correct.", ) + def get_arg_or_config( + self, parsed_args: argparse.Namespace, item: str + ) -> Any: # noqa: ANN401 + """Get a configuration option that could be overridden by a command argument. + + :param parsed_args: The argparse Namespace to check. + :param item: the name of the namespace or config item. + :returns: the requested value. + """ + return getattr(parsed_args, item, self.services.config.get(item)) + def run( # noqa: PLR0912,PLR0915 (too many branches, too many statements) self, ) -> int: @@ -508,8 +521,9 @@ def run( # noqa: PLR0912,PLR0915 (too many branches, too many statements) commands.AppCommand, dispatcher.load_command(self.app_config), ) - platform = getattr(dispatcher.parsed_args(), "platform", None) - build_for = getattr(dispatcher.parsed_args(), "build_for", None) + parsed_args = dispatcher.parsed_args() + platform = self.get_arg_or_config(parsed_args, "platform") + build_for = self.get_arg_or_config(parsed_args, "build_for") # Some commands (e.g. remote build) can allow multiple platforms # or build-fors, comma-separated. In these cases, we create the @@ -574,7 +588,7 @@ def run( # noqa: PLR0912,PLR0915 (too many branches, too many statements) craft_cli.CraftError(f"{self.app.name} internal error: {err!r}"), cause=err, ) - if os.getenv("CRAFT_DEBUG") == "1": + if self.services.config.get("debug"): raise return_code = os.EX_SOFTWARE else: diff --git a/craft_application/commands/lifecycle.py b/craft_application/commands/lifecycle.py index 4c67108e..2d77285c 100644 --- a/craft_application/commands/lifecycle.py +++ b/craft_application/commands/lifecycle.py @@ -147,14 +147,12 @@ def _fill_parser(self, parser: argparse.ArgumentParser) -> None: "--platform", type=str, metavar="name", - default=os.getenv("CRAFT_PLATFORM"), help="Set platform to build for", ) group.add_argument( "--build-for", type=str, metavar="arch", - default=os.getenv("CRAFT_BUILD_FOR"), help="Set architecture to build for", ) diff --git a/craft_application/launchpad/__init__.py b/craft_application/launchpad/__init__.py index 8ff4d4cc..f548d08a 100644 --- a/craft_application/launchpad/__init__.py +++ b/craft_application/launchpad/__init__.py @@ -22,6 +22,7 @@ from .errors import LaunchpadError from .launchpad import Launchpad from .models import LaunchpadObject, RecipeType, Recipe, SnapRecipe, CharmRecipe +from .util import Architecture __all__ = [ "errors", @@ -32,4 +33,5 @@ "Recipe", "SnapRecipe", "CharmRecipe", + "Architecture", ] diff --git a/craft_application/services/__init__.py b/craft_application/services/__init__.py index 331c56fe..76d6704c 100644 --- a/craft_application/services/__init__.py +++ b/craft_application/services/__init__.py @@ -16,6 +16,7 @@ """Service classes for the business logic of various categories of command.""" from craft_application.services.base import AppService, ProjectService +from craft_application.services.config import ConfigService from craft_application.services.lifecycle import LifecycleService from craft_application.services.package import PackageService from craft_application.services.provider import ProviderService @@ -26,6 +27,7 @@ __all__ = [ "AppService", "ProjectService", + "ConfigService", "LifecycleService", "PackageService", "ProviderService", diff --git a/craft_application/services/config.py b/craft_application/services/config.py new file mode 100644 index 00000000..3fb04bbb --- /dev/null +++ b/craft_application/services/config.py @@ -0,0 +1,198 @@ +# This file is part of craft-application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +"""Configuration service.""" +from __future__ import annotations + +import abc +import contextlib +import enum +import os +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, TypeVar, cast, final + +import pydantic +import pydantic_core +import snaphelpers +from craft_cli import emit +from typing_extensions import override + +from craft_application import _config, application, util +from craft_application.services import base + +if TYPE_CHECKING: + from craft_application.services.service_factory import ServiceFactory + + +T = TypeVar("T") + + +class ConfigHandler(abc.ABC): + """An abstract class for configuration handlers.""" + + def __init__(self, app: application.AppMetadata) -> None: + self._app = app + + @abc.abstractmethod + def get_raw(self, item: str) -> Any: # noqa: ANN401 + """Get the string value for a configuration item. + + :param item: the name of the configuration item. + :returns: The raw value of the item. + :raises: KeyError if the item cannot be found. + """ + + +@final +class AppEnvironmentHandler(ConfigHandler): + """Configuration handler to get values from app-specific environment variables.""" + + def __init__(self, app: application.AppMetadata) -> None: + super().__init__(app) + self._environ_prefix = f"{app.name.upper()}" + + @override + def get_raw(self, item: str) -> str: + return os.environ[f"{self._environ_prefix}_{item.upper()}"] + + +@final +class CraftEnvironmentHandler(ConfigHandler): + """Configuration handler to get values from CRAFT environment variables.""" + + def __init__(self, app: application.AppMetadata) -> None: + super().__init__(app) + self._fields = _config.ConfigModel.model_fields + + @override + def get_raw(self, item: str) -> str: + # Ensure that CRAFT_* env vars can only be used for configuration items + # known to craft-application. + if item not in self._fields: + raise KeyError(f"{item!r} not a general craft-application config item.") + + return os.environ[f"CRAFT_{item.upper()}"] + + +class SnapConfigHandler(ConfigHandler): + """Configuration handler that gets values from snap.""" + + def __init__(self, app: application.AppMetadata) -> None: + super().__init__(app) + try: + self._snap = snaphelpers.SnapConfig() + except KeyError: + raise OSError("Not running as a snap.") + + @override + def get_raw(self, item: str) -> Any: + snap_item = item.replace("_", "-") + try: + return self._snap.get(snap_item) + except snaphelpers.UnknownConfigKey as exc: + raise KeyError(f"unknown snap config item: {item!r}") from exc + + +@final +class DefaultConfigHandler(ConfigHandler): + """Configuration handler for getting default values.""" + + def __init__(self, app: application.AppMetadata) -> None: + super().__init__(app) + self._config_model = app.ConfigModel + self._cache: dict[str, str] = {} + + @override + def get_raw(self, item: str) -> Any: + if item in self._cache: + return self._cache[item] + + field = self._config_model.model_fields[item] + if field.default is not pydantic_core.PydanticUndefined: + self._cache[item] = field.default + return field.default + if field.default_factory is not None: + default = field.default_factory() + self._cache[item] = default + return default + + raise KeyError(f"config item {item!r} has no default value.") + + +class ConfigService(base.AppService): + """Application-wide configuration access.""" + + _handlers: list[ConfigHandler] + + def __init__( + self, + app: application.AppMetadata, + services: ServiceFactory, + *, + extra_handlers: Iterable[type[ConfigHandler]] = (), + ) -> None: + super().__init__(app, services) + self._extra_handlers = extra_handlers + self._default_handler = DefaultConfigHandler(self._app) + + @override + def setup(self) -> None: + super().setup() + self._handlers = [ + AppEnvironmentHandler(self._app), + CraftEnvironmentHandler(self._app), + *(handler(self._app) for handler in self._extra_handlers), + ] + try: + snap_handler = SnapConfigHandler(self._app) + except OSError: + emit.debug( + "App is not running as a snap - snap config handler not created." + ) + else: + self._handlers.append(snap_handler) + + def get(self, item: str) -> Any: # noqa: ANN401 + """Get the given configuration item.""" + if item not in self._app.ConfigModel.model_fields: + raise KeyError(r"unknown config item: {item!r}") + field_info = self._app.ConfigModel.model_fields[item] + + for handler in self._handlers: + try: + value = handler.get_raw(item) + except KeyError: + continue + else: + break + else: + return self._default_handler.get_raw(item) + + return self._convert_type(value, field_info.annotation) # type: ignore[arg-type,return-value] + + def _convert_type(self, value: str, field_type: type[T]) -> T: + """Convert the value to the appropriate type.""" + if isinstance(field_type, type): + if issubclass(field_type, str): + return cast(T, field_type(value)) + if issubclass(field_type, bool): + return cast(T, util.strtobool(value)) + if issubclass(field_type, enum.Enum): + with contextlib.suppress(KeyError): + return cast(T, field_type[value]) + with contextlib.suppress(KeyError): + return cast(T, field_type[value.upper()]) + field_adapter = pydantic.TypeAdapter(field_type) + return field_adapter.validate_strings(value) diff --git a/craft_application/services/provider.py b/craft_application/services/provider.py index fe12c012..a42c508e 100644 --- a/craft_application/services/provider.py +++ b/craft_application/services/provider.py @@ -210,12 +210,12 @@ def get_provider(self, name: str | None = None) -> craft_providers.Provider: emit.debug(f"Using provider {name!r} passed as an argument.") chosen_provider: str = name - # (2) get the provider from the environment (CRAFT_BUILD_ENVIRONMENT), - elif env_provider := os.getenv("CRAFT_BUILD_ENVIRONMENT"): - emit.debug(f"Using provider {env_provider!r} from environment.") - chosen_provider = env_provider + # (2) get the provider from build_environment + elif provider := self._services.config.get("build_environment"): + emit.debug(f"Using provider {provider!r} from system configuration.") + chosen_provider = provider - # (3) use provider specified with snap configuration, + # (3) use provider specified in snap configuration elif snap_provider := self._get_provider_from_snap_config(): emit.debug(f"Using provider {snap_provider!r} from snap config.") chosen_provider = snap_provider @@ -289,7 +289,7 @@ def _get_provider_by_name(self, name: str) -> craft_providers.Provider: def _get_lxd_provider(self) -> LXDProvider: """Get the LXD provider for this manager.""" - lxd_remote = os.getenv("CRAFT_LXD_REMOTE", "local") + lxd_remote = self._services.config.get("lxd_remote") return LXDProvider(lxd_project=self._app.name, lxd_remote=lxd_remote) def _get_multipass_provider(self) -> MultipassProvider: diff --git a/craft_application/services/remotebuild.py b/craft_application/services/remotebuild.py index a58b7dee..e7645ac8 100644 --- a/craft_application/services/remotebuild.py +++ b/craft_application/services/remotebuild.py @@ -47,10 +47,6 @@ DEFAULT_POLL_INTERVAL = 30 -def _get_launchpad_instance(default: str = "production") -> str: - return os.getenv("CRAFT_LAUNCHPAD_INSTANCE", default) - - class RemoteBuildService(base.AppService): """Abstract service for performing remote builds.""" @@ -254,7 +250,7 @@ def _get_lp_client(self) -> launchpad.Launchpad: with craft_cli.emit.pause(): return launchpad.Launchpad.login( f"{self._app.name}/{self._app.version}", - root=_get_launchpad_instance(), + root=self._services.config.get("launchpad_instance"), credentials_file=credentials_filepath, ) diff --git a/craft_application/services/service_factory.py b/craft_application/services/service_factory.py index 79b85222..9d01a9da 100644 --- a/craft_application/services/service_factory.py +++ b/craft_application/services/service_factory.py @@ -42,6 +42,7 @@ class ServiceFactory: ProviderClass: type[services.ProviderService] = services.ProviderService RemoteBuildClass: type[services.RemoteBuildService] = services.RemoteBuildService RequestClass: type[services.RequestService] = services.RequestService + ConfigClass: type[services.ConfigService] = services.ConfigService project: models.Project | None = None @@ -53,6 +54,7 @@ class ServiceFactory: provider: services.ProviderService = None # type: ignore[assignment] remote_build: services.RemoteBuildService = None # type: ignore[assignment] request: services.RequestService = None # type: ignore[assignment] + config: services.ConfigService = None # type: ignore[assignment] def __post_init__(self) -> None: self._service_kwargs: dict[str, dict[str, Any]] = {} diff --git a/pyproject.toml b/pyproject.toml index 0f2868a0..df16f3e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ dev = [ "pytest-cov==5.0.0", "pytest-mock==3.14.0", "pytest-rerunfailures==14.0", + "pytest-subprocess~=1.5.2", "pytest-time>=0.3.1", # Pin requests because of https://github.com/msabramo/requests-unixsocket/issues/73 "requests<2.32.0", diff --git a/tests/conftest.py b/tests/conftest.py index d46e8ffe..93387265 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,8 +24,9 @@ import craft_application import craft_parts +import pydantic import pytest -from craft_application import application, models, services, util +from craft_application import application, launchpad, models, services, util from craft_cli import EmitterMode, emit from craft_providers import bases @@ -58,19 +59,39 @@ def test_with_build_secrets(...) return features +class FakeConfigModel(craft_application.ConfigModel): + + my_str: str + my_int: int + my_bool: bool + my_default_str: str = "default" + my_default_int: int = -1 + my_default_bool: bool = True + my_default_factory: dict[str, str] = pydantic.Field( + default_factory=lambda: {"dict": "yes"} + ) + my_arch: launchpad.Architecture + + +@pytest.fixture(scope="session") +def fake_config_model() -> type[FakeConfigModel]: + return FakeConfigModel + + @pytest.fixture(scope="session") -def default_app_metadata() -> craft_application.AppMetadata: +def default_app_metadata(fake_config_model) -> craft_application.AppMetadata: with pytest.MonkeyPatch.context() as m: m.setattr(metadata, "version", lambda _: "3.14159") return craft_application.AppMetadata( "testcraft", "A fake app for testing craft-application", source_ignore_patterns=["*.snap", "*.charm", "*.starcraft"], + ConfigModel=fake_config_model, ) @pytest.fixture -def app_metadata(features) -> craft_application.AppMetadata: +def app_metadata(features, fake_config_model) -> craft_application.AppMetadata: with pytest.MonkeyPatch.context() as m: m.setattr(metadata, "version", lambda _: "3.14159") return craft_application.AppMetadata( @@ -79,6 +100,7 @@ def app_metadata(features) -> craft_application.AppMetadata: source_ignore_patterns=["*.snap", "*.charm", "*.starcraft"], features=craft_application.AppFeatures(**features), docs_url="www.craft-app.com/docs/{version}", + ConfigModel=fake_config_model, ) diff --git a/tests/integration/test_application.py b/tests/integration/test_application.py index c3c2d01e..62b6c333 100644 --- a/tests/integration/test_application.py +++ b/tests/integration/test_application.py @@ -197,6 +197,7 @@ def test_version(capsys, monkeypatch, app): assert captured.out == "testcraft 3.14159\n" +@pytest.mark.usefixtures("emitter") def test_non_lifecycle_command_does_not_require_project(monkeypatch, app): """Run a command without having a project instance shall not fail.""" monkeypatch.setattr("sys.argv", ["testcraft", "nothing"]) @@ -428,7 +429,7 @@ def test_lifecycle_error_logging(monkeypatch, tmp_path, create_app): assert parts_message in log_contents -@pytest.mark.usefixtures("pretend_jammy") +@pytest.mark.usefixtures("pretend_jammy", "emitter") def test_runtime_error_logging(monkeypatch, tmp_path, create_app, mocker): monkeypatch.chdir(tmp_path) shutil.copytree(INVALID_PROJECTS_DIR / "build-error", tmp_path, dirs_exist_ok=True) diff --git a/tests/unit/services/test_config.py b/tests/unit/services/test_config.py new file mode 100644 index 00000000..85d0c41f --- /dev/null +++ b/tests/unit/services/test_config.py @@ -0,0 +1,227 @@ +# This file is part of craft-application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +"""Unit tests for the configuration service.""" + +import itertools +import json +import string +import subprocess +from collections.abc import Iterator + +import craft_application +import craft_cli +import pytest +import pytest_subprocess +from craft_application import launchpad +from craft_application.services import config +from hypothesis import given, strategies + +CRAFT_APPLICATION_TEST_ENTRY_VALUES = [ + *( + ("verbosity_level", mode.name.lower(), mode) + for mode in craft_cli.messages.EmitterMode + ), + *(("verbosity_level", mode.name, mode) for mode in craft_cli.messages.EmitterMode), + ("debug", "true", True), + ("debug", "false", False), + ("build_environment", "host", "host"), + ("secrets", "Cara likes Butterscotch.", "Cara likes Butterscotch."), + ("platform", "laptop", "laptop"), + ("platform", "mainframe", "mainframe"), + ("build_for", "riscv64", "riscv64"), + ("build_for", "s390x", "s390x"), + *(("parallel_build_count", str(i), i) for i in range(10)), + *(("max_parallel_build_count", str(i), i) for i in range(10)), +] +APP_SPECIFIC_TEST_ENTRY_VALUES = [ + ("my_str", "some string", "some string"), + ("my_int", "1", 1), + ("my_int", "2", 2), + ("my_bool", "true", True), + ("my_bool", "false", False), + ("my_default_str", "something", "something"), + ("my_default_int", "4294967296", 2**32), + ("my_bool", "1", True), + ("my_arch", "riscv64", launchpad.Architecture.RISCV64), +] +TEST_ENTRY_VALUES = CRAFT_APPLICATION_TEST_ENTRY_VALUES + APP_SPECIFIC_TEST_ENTRY_VALUES + + +@pytest.fixture(scope="module") +def app_environment_handler(default_app_metadata) -> config.AppEnvironmentHandler: + return config.AppEnvironmentHandler(default_app_metadata) + + +@pytest.fixture(scope="module") +def craft_environment_handler(default_app_metadata) -> config.CraftEnvironmentHandler: + return config.CraftEnvironmentHandler(default_app_metadata) + + +@pytest.fixture(scope="module") +def snap_config_handler(default_app_metadata) -> Iterator[config.SnapConfigHandler]: + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setenv("SNAP", "/snap/testcraft/x1") + monkeypatch.setenv("SNAP_COMMON", "/") + monkeypatch.setenv("SNAP_DATA", "/") + monkeypatch.setenv("SNAP_REAL_HOME", "/") + monkeypatch.setenv("SNAP_USER_COMMON", "/") + monkeypatch.setenv("SNAP_USER_DATA", "/") + monkeypatch.setenv("SNAP_INSTANCE_NAME", "testcraft") + monkeypatch.setenv("SNAP_INSTANCE_KEY", "") + yield config.SnapConfigHandler(default_app_metadata) + + +@pytest.fixture(scope="module") +def default_config_handler(default_app_metadata) -> config.DefaultConfigHandler: + return config.DefaultConfigHandler(default_app_metadata) + + +@given( + item=strategies.text(alphabet=string.ascii_letters + "_", min_size=1), + content=strategies.text( + alphabet=strategies.characters(categories=["L", "M", "N", "P", "S", "Z"]) + ), +) +def test_app_environment_handler(app_environment_handler, item: str, content: str): + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setenv(f"TESTCRAFT_{item.upper()}", content) + + assert app_environment_handler.get_raw(item) == content + + +@given( + item=strategies.sampled_from(list(craft_application.ConfigModel.model_fields)), + content=strategies.text( + alphabet=strategies.characters(categories=["L", "M", "N", "P", "S", "Z"]) + ), +) +def test_craft_environment_handler(craft_environment_handler, item: str, content: str): + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setenv(f"CRAFT_{item.upper()}", content) + + assert craft_environment_handler.get_raw(item) == content + + +@pytest.mark.parametrize(("item", "content", "_"), CRAFT_APPLICATION_TEST_ENTRY_VALUES) +@pytest.mark.usefixtures("_") +def test_craft_environment_handler_success( + monkeypatch, craft_environment_handler, item: str, content: str +): + monkeypatch.setenv(f"CRAFT_{item.upper()}", content) + + assert craft_environment_handler.get_raw(item) == content + + +@pytest.mark.parametrize(("item", "content", "_"), APP_SPECIFIC_TEST_ENTRY_VALUES) +@pytest.mark.usefixtures("_") +def test_craft_environment_handler_error( + monkeypatch, craft_environment_handler, item: str, content: str +): + monkeypatch.setenv(f"CRAFT_{item.upper()}", content) + + with pytest.raises(KeyError): + assert craft_environment_handler.get_raw(item) == content + + +@given( + item=strategies.text(alphabet=string.ascii_letters + "_", min_size=1), + content=strategies.text( + alphabet=strategies.characters(categories=["L", "M", "N", "P", "S", "Z"]) + ), +) +def test_snap_config_handler(snap_config_handler, item: str, content: str): + snap_item = item.replace("_", "-") + with pytest_subprocess.FakeProcess.context() as fp, pytest.MonkeyPatch.context() as mp: + mp.setattr("snaphelpers._ctl.Popen", subprocess.Popen) + fp.register( + ["/usr/bin/snapctl", "get", "-d", snap_item], + stdout=json.dumps({snap_item: content}), + ) + assert snap_config_handler.get_raw(item) == content + + +@pytest.mark.parametrize( + ("item", "expected"), + [ + ("verbosity_level", craft_cli.EmitterMode.BRIEF), + ("debug", False), + ("lxd_remote", "local"), + ("launchpad_instance", "production"), + # Application model items with defaults + ("my_default_str", "default"), + ("my_default_int", -1), + ("my_default_bool", True), + ("my_default_factory", {"dict": "yes"}), + ], +) +def test_default_config_handler_success(default_config_handler, item, expected): + assert default_config_handler.get_raw(item) == expected + + +@pytest.mark.parametrize( + ("item", "environment_variables", "expected"), + [ + ("verbosity_level", {}, craft_cli.EmitterMode.BRIEF), + *( + ("verbosity_level", {"TESTCRAFT_VERBOSITY_LEVEL": mode.name}, mode) + for mode in craft_cli.EmitterMode + ), + *( + ("verbosity_level", {"TESTCRAFT_VERBOSITY_LEVEL": mode.name.lower()}, mode) + for mode in craft_cli.EmitterMode + ), + *( + ("verbosity_level", {"CRAFT_VERBOSITY_LEVEL": mode.name}, mode) + for mode in craft_cli.EmitterMode + ), + *( + ("verbosity_level", {"CRAFT_VERBOSITY_LEVEL": mode.name.lower()}, mode) + for mode in craft_cli.EmitterMode + ), + *( + ("debug", {var: value}, True) + for var, value in itertools.product( + ["CRAFT_DEBUG", "TESTCRAFT_DEBUG"], ["true", "1", "yes", "Y"] + ) + ), + *( + ("debug", {var: value}, False) + for var, value in itertools.product( + ["CRAFT_DEBUG", "TESTCRAFT_DEBUG"], ["false", "0", "no", "N"] + ) + ), + *( + ("parallel_build_count", {var: str(value)}, value) + for var, value in itertools.product( + ["CRAFT_PARALLEL_BUILD_COUNT", "TESTCRAFT_PARALLEL_BUILD_COUNT"], + range(10), + ) + ), + ], +) +def test_config_service_converts_type( + monkeypatch: pytest.MonkeyPatch, + fake_process: pytest_subprocess.FakeProcess, + fake_services, + item: str, + environment_variables: dict[str, str], + expected, +): + monkeypatch.setattr("snaphelpers._ctl.Popen", subprocess.Popen) + for key, value in environment_variables.items(): + monkeypatch.setenv(key, value) + fake_process.register(["/usr/bin/snapctl", fake_process.any()], stdout="{}") + assert fake_services.config.get(item) == expected diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 2c0ea3d8..6059636c 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -628,6 +628,7 @@ def test_run_managed_empty_plan(app, fake_project): ), ], ) +@pytest.mark.usefixtures("emitter") def test_get_dispatcher_error( monkeypatch, check, capsys, app, mock_dispatcher, managed, error, exit_code, message ): @@ -919,6 +920,7 @@ def test_run_success_managed_inside_managed( ), ], ) +@pytest.mark.usefixtures("emitter") def test_run_error( monkeypatch, capsys, @@ -988,6 +990,7 @@ def test_run_error_with_docs_url( @pytest.mark.parametrize("error", [KeyError(), ValueError(), Exception()]) +@pytest.mark.usefixtures("emitter") def test_run_error_debug(monkeypatch, mock_dispatcher, app, fake_project, error): app.set_project(fake_project) mock_dispatcher.load_command.side_effect = error From 45ec282634a255129097090406a4cad2833e4982 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 09:49:29 -0400 Subject: [PATCH 33/82] build(deps): update dependency pytest-check to v2.4.1 (#446) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index df16f3e8..5b869131 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ dev = [ "hypothesis>=6.0", "pyfakefs~=5.3", "pytest==8.3.2", - "pytest-check==2.3.1", + "pytest-check==2.4.1", "pytest-cov==5.0.0", "pytest-mock==3.14.0", "pytest-rerunfailures==14.0", From 2837322e2a00b0b074145364a55df504425bacba Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 6 Sep 2024 09:55:36 -0400 Subject: [PATCH 34/82] chore: update pre-commit hooks (#441) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a9b3a40..7570dc9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -14,17 +14,17 @@ repos: - id: mixed-line-ending - repo: https://github.com/charliermarsh/ruff-pre-commit # renovate: datasource=pypi;depName=ruff - rev: "v0.0.267" + rev: "v0.6.3" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black # renovate: datasource=pypi;depName=black - rev: "23.3.0" + rev: "24.8.0" hooks: - id: black - repo: https://github.com/adrienverge/yamllint.git # renovate: datasource=pypi;depName=yamllint - rev: "v1.31.0" + rev: "v1.35.1" hooks: - id: yamllint From 260bd46c5d586c2a3730a5a161b80b7ba7f51fb4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:01:39 -0400 Subject: [PATCH 35/82] build(deps): update dependency sphinx-autobuild to v2024.9.3 (#449) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5b869131..28a459db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ types = [ ] docs = [ "canonical-sphinx~=0.1", - "sphinx-autobuild==2024.4.16", + "sphinx-autobuild==2024.9.3", "sphinx-lint==0.9.1", ] apt = [ From d96da4e50a2e467cb54566b9c73759bb5b0574a4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:32:52 -0400 Subject: [PATCH 36/82] build(deps): update dependency mypy to v1.11.2 (#448) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 28a459db..1782c897 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ lint = [ "yamllint==1.35.1" ] types = [ - "mypy[reports]==1.9.0", + "mypy[reports]==1.11.2", "pyright==1.1.376", "types-requests", "types-urllib3", From f8513c889e178ae5bd5b01cd5a6e177b47f1d43e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:39:52 -0400 Subject: [PATCH 37/82] build(deps): update dependency setuptools to v74 (#447) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1782c897..4c3186d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ apt = [ [build-system] requires = [ - "setuptools==72.2.0", + "setuptools==74.1.1", "setuptools_scm[toml]>=8.1" ] build-backend = "setuptools.build_meta" From dc1928f2e129cfa06df67ad86f2ad430e136dfc1 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 6 Sep 2024 16:41:12 -0400 Subject: [PATCH 38/82] fix(models): coerce numbers to strings by default (#450) --- craft_application/models/base.py | 1 + tests/unit/models/test_base.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/craft_application/models/base.py b/craft_application/models/base.py index 08e30438..cd261f77 100644 --- a/craft_application/models/base.py +++ b/craft_application/models/base.py @@ -38,6 +38,7 @@ class CraftBaseModel(pydantic.BaseModel): extra="forbid", populate_by_name=True, alias_generator=alias_generator, + coerce_numbers_to_str=True, ) def marshal(self) -> dict[str, str | list[str] | dict[str, Any]]: diff --git a/tests/unit/models/test_base.py b/tests/unit/models/test_base.py index 2d9390a8..da7067c6 100644 --- a/tests/unit/models/test_base.py +++ b/tests/unit/models/test_base.py @@ -19,6 +19,7 @@ import pydantic import pytest from craft_application import errors, models +from hypothesis import given, strategies from overrides import override @@ -58,3 +59,22 @@ def test_model_reference_slug_errors(): ) assert str(err.value) == expected assert err.value.doc_slug == "/mymodel.html" + + +class CoerceModel(models.CraftBaseModel): + + stringy: str + + +@given( + strategies.one_of( + strategies.integers(), + strategies.floats(), + strategies.decimals(), + strategies.text(), + ) +) +def test_model_coerces_to_strings(value): + result = CoerceModel.model_validate({"stringy": value}) + + assert result.stringy == str(value) From 244b52cb34e491901d274a836281bfd06b0b69af Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Mon, 9 Sep 2024 17:28:54 -0400 Subject: [PATCH 39/82] tests: re-enable almalinux tests (#445) --- tests/integration/services/test_provider.py | 1 + tests/unit/services/test_provider.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/tests/integration/services/test_provider.py b/tests/integration/services/test_provider.py index 68f5c85f..65d663e0 100644 --- a/tests/integration/services/test_provider.py +++ b/tests/integration/services/test_provider.py @@ -36,6 +36,7 @@ ), pytest.param(("ubuntu", "24.04"), id="ubuntu_lts"), pytest.param(("ubuntu", "22.04"), id="ubuntu_old_lts"), + pytest.param(("almalinux", "9"), id="almalinux_9"), ], ) @pytest.mark.parametrize( diff --git a/tests/unit/services/test_provider.py b/tests/unit/services/test_provider.py index 17f71814..8c3abd8f 100644 --- a/tests/unit/services/test_provider.py +++ b/tests/unit/services/test_provider.py @@ -337,7 +337,9 @@ def test_get_provider_from_platform( ("base_name", "base_class", "alias"), [ (("ubuntu", "devel"), bases.BuilddBase, bases.BuilddBaseAlias.DEVEL), + (("ubuntu", "24.04"), bases.BuilddBase, bases.BuilddBaseAlias.NOBLE), (("ubuntu", "22.04"), bases.BuilddBase, bases.BuilddBaseAlias.JAMMY), + (("ubuntu", "20.04"), bases.BuilddBase, bases.BuilddBaseAlias.FOCAL), ], ) def test_get_base_buildd( @@ -374,7 +376,11 @@ def test_get_base_packages(provider_service): "base_name", [ ("ubuntu", "devel"), + ("ubuntu", "24.10"), + ("ubuntu", "24.04"), ("ubuntu", "22.04"), + ("ubuntu", "20.04"), + ("almalinux", "9"), ], ) def test_instance( From 2bda2579de8a4f5733486914cd4d9c9383fbcb3e Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Mon, 9 Sep 2024 18:31:46 -0400 Subject: [PATCH 40/82] chore(test): put ids on some hard-to-trigger tests (#451) --- tests/integration/test_application.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_application.py b/tests/integration/test_application.py index 62b6c333..47606046 100644 --- a/tests/integration/test_application.py +++ b/tests/integration/test_application.py @@ -98,14 +98,16 @@ def app(create_app): @pytest.mark.parametrize( ("argv", "stdout", "stderr", "exit_code"), [ - (["help"], "", BASIC_USAGE, 0), - (["--help"], "", BASIC_USAGE, 0), - (["-h"], "", BASIC_USAGE, 0), - (["--version"], VERSION_INFO, "", 0), - (["-V"], VERSION_INFO, "", 0), - (["-q", "--version"], "", "", 0), - (["--invalid-parameter"], "", BASIC_USAGE, 64), - (["non-command"], "", INVALID_COMMAND, 64), + pytest.param(["help"], "", BASIC_USAGE, 0, id="help"), + pytest.param(["--help"], "", BASIC_USAGE, 0, id="--help"), + pytest.param(["-h"], "", BASIC_USAGE, 0, id="-h"), + pytest.param(["--version"], VERSION_INFO, "", 0, id="--version"), + pytest.param(["-V"], VERSION_INFO, "", 0, id="-V"), + pytest.param(["-q", "--version"], "", "", 0, id="-q--version"), + pytest.param( + ["--invalid-parameter"], "", BASIC_USAGE, 64, id="--invalid-parameter" + ), + pytest.param(["non-command"], "", INVALID_COMMAND, 64, id="non-command"), ], ) def test_special_inputs(capsys, monkeypatch, app, argv, stdout, stderr, exit_code): From a8c7876b2e8f9efd837e106a8d16aba50f829abe Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Wed, 11 Sep 2024 19:52:27 -0300 Subject: [PATCH 41/82] docs: add changelog for 4.2.0 (#457) * docs: add changelog for 4.2.0 * docs: review changelog 4.2.0 * Update docs/reference/changelog.rst Co-authored-by: Michael DuBelko * move command entry to subheader --------- Co-authored-by: Michael DuBelko --- docs/reference/changelog.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 2b5be99d..a7267d9e 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,24 @@ Changelog ********* +4.2.0 (2024-Sep-12) +------------------- + +Application +=========== + +- Add a configuration service to unify handling of command line arguments, + environment variables, snap configurations, and so on. +- Use the standard library to retrieve the host's proxies. + +Commands +======== + +- Properly support ``--shell``, ``--shell-after`` and ``--debug`` on the + ``pack`` command. + +For a complete list of commits, check out the `4.2.0`_ release on GitHub. + 4.1.2 (2024-Sep-05) ------------------- @@ -225,3 +243,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _4.1.0: https://github.com/canonical/craft-application/releases/tag/4.1.0 .. _4.1.1: https://github.com/canonical/craft-application/releases/tag/4.1.1 .. _4.1.2: https://github.com/canonical/craft-application/releases/tag/4.1.2 +.. _4.2.0: https://github.com/canonical/craft-application/releases/tag/4.2.0 From 22909dbb6caa53dd47a94e094ee0da76e3a1a51c Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 10 Jun 2024 17:17:21 -0300 Subject: [PATCH 42/82] feat: add initial FetchService skeleton (#7) This sets the expected API that clients (the Application) will use: - Creating a fetch-service is done in setup(); - Sessions are created with create_session(), which receives a managed instance because we'll need it to figure out the network gateway; - Sessions are destroyed with teardown_session(); - Stopping the fetch-service is done with shutdown(). --- craft_application/application.py | 13 +++ craft_application/services/__init__.py | 2 + craft_application/services/fetch.py | 77 +++++++++++++++++ craft_application/services/service_factory.py | 2 + tests/conftest.py | 30 +++++++ tests/unit/test_application.py | 31 +------ tests/unit/test_application_fetch.py | 83 +++++++++++++++++++ 7 files changed, 209 insertions(+), 29 deletions(-) create mode 100644 craft_application/services/fetch.py create mode 100644 tests/unit/test_application_fetch.py diff --git a/craft_application/application.py b/craft_application/application.py index 3a65cfb6..aef40f9b 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -65,6 +65,9 @@ class AppFeatures: build_secrets: bool = False """Support for build-time secrets.""" + fetch_service: bool = False + """Support for the fetch service.""" + @final @dataclass(frozen=True) @@ -370,6 +373,10 @@ def run_managed(self, platform: str | None, build_for: str | None) -> None: with self.services.provider.instance( build_info, work_dir=self._work_dir ) as instance: + if self.app.features.fetch_service: + session_env = self.services.fetch.create_session(instance) + env.update(session_env) + cmd = [self.app.name, *sys.argv[1:]] craft_cli.emit.debug( f"Executing {cmd} in instance location {instance_path} with {extra_args}." @@ -387,6 +394,12 @@ def run_managed(self, platform: str | None, build_for: str | None) -> None: raise craft_providers.ProviderError( f"Failed to execute {self.app.name} in instance." ) from exc + finally: + if self.app.features.fetch_service: + self.services.fetch.teardown_session() + + if self.app.features.fetch_service: + self.services.fetch.shutdown() def configure(self, global_args: dict[str, Any]) -> None: """Configure the application using any global arguments.""" diff --git a/craft_application/services/__init__.py b/craft_application/services/__init__.py index 76d6704c..accbd923 100644 --- a/craft_application/services/__init__.py +++ b/craft_application/services/__init__.py @@ -17,6 +17,7 @@ from craft_application.services.base import AppService, ProjectService from craft_application.services.config import ConfigService +from craft_application.services.fetch import FetchService from craft_application.services.lifecycle import LifecycleService from craft_application.services.package import PackageService from craft_application.services.provider import ProviderService @@ -26,6 +27,7 @@ __all__ = [ "AppService", + "FetchService", "ProjectService", "ConfigService", "LifecycleService", diff --git a/craft_application/services/fetch.py b/craft_application/services/fetch.py new file mode 100644 index 00000000..c0f23885 --- /dev/null +++ b/craft_application/services/fetch.py @@ -0,0 +1,77 @@ +# This file is part of craft-application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +"""Service class to communicate with the fetch-service.""" + +import craft_providers +from typing_extensions import override + +from craft_application import services + + +class FetchService(services.AppService): + """Service class that handles communication with the fetch-service. + + This Service is able to spawn a fetch-service instance and create sessions + to be used in managed runs. The general usage flow is this: + + - Initialise a fetch-service via setup() (done automatically by the service + factory); + - For each managed execution: + - Create a new session with create_session(), passing the new managed + instance; + - Teardown/close the session with teardown_session(); + - Stop the fetch-service via shutdown(). + """ + + @override + def setup(self) -> None: + """Start the fetch-service process with proper arguments.""" + # Things to do here: + # - Figure out available ports + # - Spawn the service (from the snap?) + + def create_session( + self, + instance: craft_providers.Executor, # noqa: ARG002 (unused-method-argument) + ) -> dict[str, str]: + """Create a new session. + + :return: The environment variables that must be used by any process + that will use the new session. + """ + # Things to do here (from the prototype): + # To create the session: + # - Create a session (POST), store session data + # - Find the gateway for the network used by the instance (need API) + # - Return http_proxy/https_proxy with session data + # + # To configure the instance: + # - Install bundled certificate + # - Configure snap proxy + # - Clean APT's cache + return {} + + def teardown_session(self) -> None: + """Teardown and cleanup a previously-created session.""" + # Things to do here (from the prototype): + # - Dump service status (where?) + # - Revoke session token + # - Dump session report + # - Delete session + # - Clean session resources? + + def shutdown(self) -> None: + """Stop the fetch-service.""" diff --git a/craft_application/services/service_factory.py b/craft_application/services/service_factory.py index 9d01a9da..d7c3cf4f 100644 --- a/craft_application/services/service_factory.py +++ b/craft_application/services/service_factory.py @@ -43,6 +43,7 @@ class ServiceFactory: RemoteBuildClass: type[services.RemoteBuildService] = services.RemoteBuildService RequestClass: type[services.RequestService] = services.RequestService ConfigClass: type[services.ConfigService] = services.ConfigService + FetchClass: type[services.FetchService] = services.FetchService project: models.Project | None = None @@ -55,6 +56,7 @@ class ServiceFactory: remote_build: services.RemoteBuildService = None # type: ignore[assignment] request: services.RequestService = None # type: ignore[assignment] config: services.ConfigService = None # type: ignore[assignment] + fetch: services.FetchService = None # type: ignore[assignment] def __post_init__(self) -> None: self._service_kwargs: dict[str, dict[str, Any]] = {} diff --git a/tests/conftest.py b/tests/conftest.py index 93387265..4e1d0adc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,7 @@ from craft_application import application, launchpad, models, services, util from craft_cli import EmitterMode, emit from craft_providers import bases +from typing_extensions import override if TYPE_CHECKING: # pragma: no cover from collections.abc import Iterator @@ -304,3 +305,32 @@ def fake_services( PackageClass=fake_package_service_class, LifecycleClass=fake_lifecycle_service_class, ) + + +class FakeApplication(application.Application): + """An application class explicitly for testing. Adds some convenient test hooks.""" + + platform: str = "unknown-platform" + build_on: str = "unknown-build-on" + build_for: str | None = "unknown-build-for" + + def set_project(self, project): + self._Application__project = project + + @override + def _extra_yaml_transform( + self, + yaml_data: dict[str, Any], + *, + build_on: str, + build_for: str | None, + ) -> dict[str, Any]: + self.build_on = build_on + self.build_for = build_for + + return yaml_data + + +@pytest.fixture() +def app(app_metadata, fake_services): + return FakeApplication(app_metadata, fake_services) diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 6059636c..869a693b 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -56,6 +56,8 @@ from craft_providers import bases from overrides import override +from tests.conftest import FakeApplication + EMPTY_COMMAND_GROUP = craft_cli.CommandGroup("FakeCommands", []) BASIC_PROJECT_YAML = """ name: myproject @@ -367,35 +369,6 @@ def test_app_metadata_default_mandatory_adoptable_fields(): assert app.mandatory_adoptable_fields == ["version"] -class FakeApplication(application.Application): - """An application class explicitly for testing. Adds some convenient test hooks.""" - - platform: str = "unknown-platform" - build_on: str = "unknown-build-on" - build_for: str | None = "unknown-build-for" - - def set_project(self, project): - self._Application__project = project - - @override - def _extra_yaml_transform( - self, - yaml_data: dict[str, Any], - *, - build_on: str, - build_for: str | None, - ) -> dict[str, Any]: - self.build_on = build_on - self.build_for = build_for - - return yaml_data - - -@pytest.fixture -def app(app_metadata, fake_services): - return FakeApplication(app_metadata, fake_services) - - class FakePlugin(craft_parts.plugins.Plugin): def __init__(self, properties, part_info): pass diff --git a/tests/unit/test_application_fetch.py b/tests/unit/test_application_fetch.py new file mode 100644 index 00000000..2ed9cf9d --- /dev/null +++ b/tests/unit/test_application_fetch.py @@ -0,0 +1,83 @@ +# This file is part of craft_application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with this program. If not, see . +"""Unit tests for the interaction between the Application and the FetchService.""" +from unittest import mock + +import craft_providers +import pytest +from craft_application import services +from craft_application.util import get_host_architecture +from typing_extensions import override + + +class FakeFetchService(services.FetchService): + """Fake FetchService that tracks calls""" + + calls: list[str] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.calls = [] + + @override + def setup(self) -> None: + self.calls.append("setup") + + @override + def create_session( + self, + instance: craft_providers.Executor, # (unused-method-argument) + ) -> dict[str, str]: + self.calls.append("create_session") + return {} + + @override + def teardown_session(self) -> None: + self.calls.append("teardown_session") + + @override + def shutdown(self) -> None: + self.calls.append("shutdown") + + +@pytest.mark.enable_features("fetch_service") +@pytest.mark.parametrize("fake_build_plan", [2], indirect=True) +def test_run_managed_fetch_service(app, fake_project, fake_build_plan): + """Test that the application calls the correct FetchService methods.""" + mock_provider = mock.MagicMock(spec_set=services.ProviderService) + app.services.provider = mock_provider + app.project = fake_project + + expected_build_infos = 2 + assert len(fake_build_plan) == expected_build_infos + app._build_plan = fake_build_plan + + app.services.FetchClass = FakeFetchService + + app.run_managed(None, get_host_architecture()) + + fetch_service = app.services.fetch + assert fetch_service.calls == [ + # One call to setup + "setup", + # Two pairs of create/teardown sessions, for two builds + "create_session", + "teardown_session", + "create_session", + "teardown_session", + # One call to shut down + "shutdown", + ] From ff76bb5e438cc6aebb502c353dcc7543e110d342 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Thu, 13 Jun 2024 15:23:26 -0300 Subject: [PATCH 43/82] feat: start the fetch-service (#8) Add a set of utility functions so that FetchService.setup() will spawn a fetch-service if necessary. These utility functions include a function to get the service's (json) status, and a function to tell whether the service is up. Use the $SNAP_USER_COMMON location for the fetch-service snap as a base for the 'config' and 'spool' subdirectories. --- .github/workflows/tests.yaml | 7 + craft_application/errors.py | 4 + craft_application/fetch.py | 170 +++++++++++++++++++++++ craft_application/services/fetch.py | 33 ++++- tests/integration/services/test_fetch.py | 90 ++++++++++++ tests/unit/test_application_fetch.py | 8 +- tests/unit/test_fetch.py | 117 ++++++++++++++++ 7 files changed, 419 insertions(+), 10 deletions(-) create mode 100644 craft_application/fetch.py create mode 100644 tests/integration/services/test_fetch.py create mode 100644 tests/unit/test_fetch.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a402e28b..8801ac2a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -115,6 +115,10 @@ jobs: channel: latest/stable - name: Configure environment run: | + echo "::group::Begin snap install" + echo "Installing snaps in the background while running apt and pip..." + sudo snap install --no-wait --channel=beta fetch-service + echo "::endgroup::" echo "::group::apt-get" sudo apt update sudo apt-get install -y libapt-pkg-dev @@ -123,6 +127,9 @@ jobs: python -m pip install tox echo "::endgroup::" mkdir -p results + echo "::group::Wait for snap to complete" + snap watch --last=install + echo "::endgroup::" - name: Setup Tox environments run: tox run -e integration-${{ matrix.python }} --notest - name: Integration tests diff --git a/craft_application/errors.py b/craft_application/errors.py index cb209a18..0c4a74af 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -242,3 +242,7 @@ def __init__( # (too many arguments) reportable=reportable, retcode=retcode, ) + + +class FetchServiceError(CraftError): + """Errors related to the fetch-service.""" diff --git a/craft_application/fetch.py b/craft_application/fetch.py new file mode 100644 index 00000000..5e648179 --- /dev/null +++ b/craft_application/fetch.py @@ -0,0 +1,170 @@ +# This file is part of craft_application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with this program. If not, see . +"""Utilities to interact with the fetch-service.""" +import contextlib +import pathlib +import subprocess +from dataclasses import dataclass +from typing import Any, cast + +import requests +from requests.auth import HTTPBasicAuth + +from craft_application import errors +from craft_application.util import retry + + +@dataclass(frozen=True) +class FetchServiceConfig: + """Dataclass for the ports that a fetch-service instance uses.""" + + proxy: int + """The proxy port, to be passed to the applications to be proxied.""" + control: int + """The control port, to create/terminate sessions, get status, etc.""" + username: str + """The username for auth.""" + password: str + """The password for auth.""" + + @property + def auth(self) -> str: + """Authentication in user:passwd format.""" + return f"{self.username}:{self.password}" + + +_FETCH_BINARY = "/snap/bin/fetch-service" + +_DEFAULT_CONFIG = FetchServiceConfig( + proxy=13444, + control=13555, + username="craft", + password="craft", # noqa: S106 (hardcoded-password-func-arg) +) + + +def is_service_online() -> bool: + """Whether the fetch-service is up and listening.""" + try: + status = get_service_status() + except errors.FetchServiceError: + return False + return "uptime" in status + + +def get_service_status() -> dict[str, Any]: + """Get the JSON status of the fetch-service. + + :raises errors.FetchServiceError: if a connection error happens. + """ + headers = { + "Content-type": "application/json", + "Accept": "application/json", + } + auth = HTTPBasicAuth(_DEFAULT_CONFIG.username, _DEFAULT_CONFIG.password) + try: + response = requests.get( + f"http://localhost:{_DEFAULT_CONFIG.control}/status", + auth=auth, + headers=headers, + timeout=0.1, + ) + response.raise_for_status() + except requests.RequestException as err: + message = f"Error querying fetch-service status: {str(err)}" + raise errors.FetchServiceError(message) + + return cast(dict[str, Any], response.json()) + + +def start_service() -> subprocess.Popen[str] | None: + """Start the fetch-service with default ports and auth.""" + if is_service_online(): + # Nothing to do, service is already up. + return None + + cmd = [_FETCH_BINARY] + + env = {"FETCH_SERVICE_AUTH": _DEFAULT_CONFIG.auth} + + # Add the ports + cmd.append(f"--control-port={_DEFAULT_CONFIG.control}") + cmd.append(f"--proxy-port={_DEFAULT_CONFIG.proxy}") + + # Set config and spool directories + base_dir = _get_service_base_dir() + + for dir_name in ("config", "spool"): + dir_path = base_dir / dir_name + dir_path.mkdir(exist_ok=True) + cmd.append(f"--{dir_name}={dir_path}") + + fetch_process = subprocess.Popen( + cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + + # Wait a bit for the service to come online + with contextlib.suppress(subprocess.TimeoutExpired): + fetch_process.wait(0.1) + + if fetch_process.poll() is not None: + # fetch-service already exited, something is wrong + stdout = "" + if fetch_process.stdout is not None: + stdout = fetch_process.stdout.read() + + if "bind: address already in use" in stdout: + proxy, control = _DEFAULT_CONFIG.proxy, _DEFAULT_CONFIG.control + message = f"fetch-service ports {proxy} and {control} are already in use." + details = None + else: + message = "Error spawning the fetch-service." + details = stdout + raise errors.FetchServiceError(message, details=details) + + status = retry( + "wait for fetch-service to come online", + errors.FetchServiceError, + get_service_status, # pyright: ignore[reportArgumentType] + ) + if "uptime" not in status: + stop_service(fetch_process) + raise errors.FetchServiceError( + f"Fetch service did not start correctly: {status}" + ) + + return fetch_process + + +def stop_service(fetch_process: subprocess.Popen[str]) -> None: + """Stop the fetch-service. + + This function first calls terminate(), and then kill() after a short time. + """ + fetch_process.terminate() + try: + fetch_process.wait(timeout=1.0) + except subprocess.TimeoutExpired: + fetch_process.kill() + + +def _get_service_base_dir() -> pathlib.Path: + """Get the base directory to contain the fetch-service's runtime files.""" + input_line = "sh -c 'echo $SNAP_USER_COMMON'" + output = subprocess.check_output( + ["snap", "run", "--shell", "fetch-service"], text=True, input=input_line + ) + return pathlib.Path(output.strip()) diff --git a/craft_application/services/fetch.py b/craft_application/services/fetch.py index c0f23885..6d8da52f 100644 --- a/craft_application/services/fetch.py +++ b/craft_application/services/fetch.py @@ -14,11 +14,18 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . """Service class to communicate with the fetch-service.""" +from __future__ import annotations + +import subprocess +import typing import craft_providers from typing_extensions import override -from craft_application import services +from craft_application import fetch, services + +if typing.TYPE_CHECKING: + from craft_application.application import AppMetadata class FetchService(services.AppService): @@ -36,12 +43,17 @@ class FetchService(services.AppService): - Stop the fetch-service via shutdown(). """ + _fetch_process: subprocess.Popen[str] | None + + def __init__(self, app: AppMetadata, services: services.ServiceFactory) -> None: + super().__init__(app, services) + self._fetch_process = None + @override def setup(self) -> None: """Start the fetch-service process with proper arguments.""" - # Things to do here: - # - Figure out available ports - # - Spawn the service (from the snap?) + super().setup() + self._fetch_process = fetch.start_service() def create_session( self, @@ -73,5 +85,14 @@ def teardown_session(self) -> None: # - Delete session # - Clean session resources? - def shutdown(self) -> None: - """Stop the fetch-service.""" + def shutdown(self, *, force: bool = False) -> None: + """Stop the fetch-service. + + The default behavior is a no-op; the Application never shuts down the + fetch-service so that it stays up and ready to serve other craft + applications. + + :param force: Whether the fetch-service should be, in fact, stopped. + """ + if force and self._fetch_process: + fetch.stop_service(self._fetch_process) diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py new file mode 100644 index 00000000..8cdb373a --- /dev/null +++ b/tests/integration/services/test_fetch.py @@ -0,0 +1,90 @@ +# This file is part of craft-application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +"""Tests for FetchService.""" +import shutil +import socket + +import pytest +from craft_application import errors, fetch, services + + +@pytest.fixture(autouse=True) +def _set_test_base_dir(mocker): + original = fetch._get_service_base_dir() + test_dir = original / "test" + test_dir.mkdir(exist_ok=False) + mocker.patch.object(fetch, "_get_service_base_dir", return_value=test_dir) + yield + shutil.rmtree(test_dir) + + +@pytest.fixture() +def app_service(app_metadata, fake_services): + fetch_service = services.FetchService(app_metadata, fake_services) + yield fetch_service + fetch_service.shutdown(force=True) + + +def test_start_service(app_service): + assert not fetch.is_service_online() + app_service.setup() + assert fetch.is_service_online() + + +def test_start_service_already_up(app_service, request): + # Create a fetch-service "manually" + fetch_process = fetch.start_service() + assert fetch.is_service_online() + # Ensure its cleaned up when the test is done + if fetch_process is not None: + request.addfinalizer(lambda: fetch.stop_service(fetch_process)) + + app_service.setup() + assert fetch.is_service_online() + + +@pytest.mark.parametrize( + "port", [fetch._DEFAULT_CONFIG.control, fetch._DEFAULT_CONFIG.proxy] +) +def test_start_service_port_taken(app_service, request, port): + # "Occupy" one of the necessary ports manually. + soc = socket.create_server(("localhost", port), reuse_port=True) + request.addfinalizer(soc.close) + + assert not fetch.is_service_online() + + proxy = fetch._DEFAULT_CONFIG.proxy + control = fetch._DEFAULT_CONFIG.control + + expected = f"fetch-service ports {proxy} and {control} are already in use." + with pytest.raises(errors.FetchServiceError, match=expected): + app_service.setup() + + +def test_shutdown_service(app_service): + assert not fetch.is_service_online() + + app_service.setup() + assert fetch.is_service_online() + + # By default, shutdown() without parameters doesn't actually stop the + # fetch-service. + app_service.shutdown() + assert fetch.is_service_online() + + # shutdown(force=True) must stop the fetch-service. + app_service.shutdown(force=True) + assert not fetch.is_service_online() diff --git a/tests/unit/test_application_fetch.py b/tests/unit/test_application_fetch.py index 2ed9cf9d..8caeb62e 100644 --- a/tests/unit/test_application_fetch.py +++ b/tests/unit/test_application_fetch.py @@ -49,8 +49,8 @@ def teardown_session(self) -> None: self.calls.append("teardown_session") @override - def shutdown(self) -> None: - self.calls.append("shutdown") + def shutdown(self, *, force: bool = False) -> None: + self.calls.append(f"shutdown({force})") @pytest.mark.enable_features("fetch_service") @@ -78,6 +78,6 @@ def test_run_managed_fetch_service(app, fake_project, fake_build_plan): "teardown_session", "create_session", "teardown_session", - # One call to shut down - "shutdown", + # One call to shut down (without `force`) + "shutdown(False)", ] diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py new file mode 100644 index 00000000..1b63c5a9 --- /dev/null +++ b/tests/unit/test_fetch.py @@ -0,0 +1,117 @@ +# This file is part of craft-application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +"""Tests for fetch-service-related functions.""" +import subprocess +from unittest.mock import call + +import pytest +import responses +from craft_application import errors, fetch + +CONTROL = fetch._DEFAULT_CONFIG.control +PROXY = fetch._DEFAULT_CONFIG.proxy +AUTH = fetch._DEFAULT_CONFIG.auth + + +@responses.activate +def test_get_service_status_success(): + responses.add( + responses.GET, + f"http://localhost:{CONTROL}/status", + json={"uptime": 10}, + status=200, + ) + status = fetch.get_service_status() + assert status == {"uptime": 10} + + +@responses.activate +def test_get_service_status_failure(): + responses.add( + responses.GET, + f"http://localhost:{CONTROL}/status", + status=404, + ) + expected = "Error querying fetch-service status: 404 Client Error" + with pytest.raises(errors.FetchServiceError, match=expected): + fetch.get_service_status() + + +@pytest.mark.parametrize( + ("status", "json", "expected"), + [ + (200, {"uptime": 10}, True), + (200, {"uptime": 10, "other-key": "value"}, True), + (200, {"other-key": "value"}, False), + (404, {"other-key": "value"}, False), + ], +) +@responses.activate +def test_is_service_online(status, json, expected): + responses.add( + responses.GET, + f"http://localhost:{CONTROL}/status", + status=status, + json=json, + ) + assert fetch.is_service_online() == expected + + +def test_start_service(mocker, tmp_path): + mock_is_online = mocker.patch.object(fetch, "is_service_online", return_value=False) + mock_base_dir = mocker.patch.object( + fetch, "_get_service_base_dir", return_value=tmp_path + ) + mock_get_status = mocker.patch.object( + fetch, "get_service_status", return_value={"uptime": 10} + ) + + mock_popen = mocker.patch.object(subprocess, "Popen") + mock_process = mock_popen.return_value + mock_process.poll.return_value = None + + process = fetch.start_service() + assert process is mock_process + + assert mock_is_online.called + assert mock_base_dir.called + assert mock_get_status.called + + popen_call = mock_popen.mock_calls[0] + assert popen_call == call( + [ + fetch._FETCH_BINARY, + f"--control-port={CONTROL}", + f"--proxy-port={PROXY}", + f"--config={tmp_path/'config'}", + f"--spool={tmp_path/'spool'}", + ], + env={"FETCH_SERVICE_AUTH": AUTH}, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + +def test_start_service_already_up(mocker): + """If the fetch-service is already up then a new process is *not* created.""" + mock_is_online = mocker.patch.object(fetch, "is_service_online", return_value=True) + mock_popen = mocker.patch.object(subprocess, "Popen") + + assert fetch.start_service() is None + + assert mock_is_online.called + assert not mock_popen.called From a047ae41899d2e2e8baae9fe3c4c78420ed70599 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 14 Jun 2024 13:47:08 -0300 Subject: [PATCH 44/82] feat: create/teardown fetch-service sessions (#9) Session credentials are stored to be used (in the near future) when setting up the http/https proxies. When closing the session, the report obtained from the fetch-service is discarded (for now, until we figure out what to do with it). --- craft_application/fetch.py | 88 +++++++++++++++++++----- craft_application/services/fetch.py | 25 ++++--- pyproject.toml | 1 + tests/integration/services/test_fetch.py | 18 +++++ tests/unit/git/test_git.py | 2 +- tests/unit/services/test_fetch.py | 53 ++++++++++++++ tests/unit/test_application_fetch.py | 4 +- tests/unit/test_fetch.py | 59 ++++++++++++++-- tests/unit/test_secrets.py | 2 +- 9 files changed, 220 insertions(+), 32 deletions(-) create mode 100644 tests/unit/services/test_fetch.py diff --git a/craft_application/fetch.py b/craft_application/fetch.py index 5e648179..fefc4fee 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -21,9 +21,11 @@ from typing import Any, cast import requests +from pydantic import Field from requests.auth import HTTPBasicAuth from craft_application import errors +from craft_application.models import CraftBaseModel from craft_application.util import retry @@ -56,6 +58,13 @@ def auth(self) -> str: ) +class SessionData(CraftBaseModel): + """Fetch service session data.""" + + session_id: str = Field(alias="id") + token: str + + def is_service_online() -> bool: """Whether the fetch-service is up and listening.""" try: @@ -70,23 +79,7 @@ def get_service_status() -> dict[str, Any]: :raises errors.FetchServiceError: if a connection error happens. """ - headers = { - "Content-type": "application/json", - "Accept": "application/json", - } - auth = HTTPBasicAuth(_DEFAULT_CONFIG.username, _DEFAULT_CONFIG.password) - try: - response = requests.get( - f"http://localhost:{_DEFAULT_CONFIG.control}/status", - auth=auth, - headers=headers, - timeout=0.1, - ) - response.raise_for_status() - except requests.RequestException as err: - message = f"Error querying fetch-service status: {str(err)}" - raise errors.FetchServiceError(message) - + response = _service_request("get", "status") return cast(dict[str, Any], response.json()) @@ -161,6 +154,67 @@ def stop_service(fetch_process: subprocess.Popen[str]) -> None: fetch_process.kill() +def create_session() -> SessionData: + """Create a new fetch-service session. + + :return: a SessionData object containing the session's id and token. + """ + data = _service_request("post", "session", json={}).json() + + return SessionData.unmarshal(data=data) + + +def teardown_session(session_data: SessionData) -> dict[str, Any]: + """Stop and cleanup a running fetch-service session. + + :param SessionData: the data of a previously-created session. + :return: A dict containing the session's report (the contents and format + of this dict are still subject to change). + """ + session_id = session_data.session_id + session_token = session_data.token + + # Revoke token + _revoke_data = _service_request( + "delete", f"session/{session_id}/token", json={"token": session_token} + ).json() + + # Get session report + session_report = _service_request("get", f"session/{session_id}", json={}).json() + + # Delete session + _service_request("delete", f"session/{session_id}") + + # Delete session resources + _service_request("delete", f"resources/{session_id}") + + return cast(dict[str, Any], session_report) + + +def _service_request( + verb: str, endpoint: str, json: dict[str, Any] | None = None +) -> requests.Response: + headers = { + "Content-type": "application/json", + } + auth = HTTPBasicAuth(_DEFAULT_CONFIG.username, _DEFAULT_CONFIG.password) + try: + response = requests.request( + verb, + f"http://localhost:{_DEFAULT_CONFIG.control}/{endpoint}", + auth=auth, + headers=headers, + json=json, # Use defaults + timeout=0.1, + ) + response.raise_for_status() + except requests.RequestException as err: + message = f"Error with fetch-service {verb.upper()}: {str(err)}" + raise errors.FetchServiceError(message) + + return response + + def _get_service_base_dir() -> pathlib.Path: """Get the base directory to contain the fetch-service's runtime files.""" input_line = "sh -c 'echo $SNAP_USER_COMMON'" diff --git a/craft_application/services/fetch.py b/craft_application/services/fetch.py index 6d8da52f..17df57f7 100644 --- a/craft_application/services/fetch.py +++ b/craft_application/services/fetch.py @@ -44,10 +44,12 @@ class FetchService(services.AppService): """ _fetch_process: subprocess.Popen[str] | None + _session_data: fetch.SessionData | None def __init__(self, app: AppMetadata, services: services.ServiceFactory) -> None: super().__init__(app, services) self._fetch_process = None + self._session_data = None @override def setup(self) -> None: @@ -66,7 +68,7 @@ def create_session( """ # Things to do here (from the prototype): # To create the session: - # - Create a session (POST), store session data + # - Create a session (POST), store session data (DONE) # - Find the gateway for the network used by the instance (need API) # - Return http_proxy/https_proxy with session data # @@ -74,16 +76,23 @@ def create_session( # - Install bundled certificate # - Configure snap proxy # - Clean APT's cache + if self._session_data is not None: + raise ValueError( + "create_session() called but there's already a live fetch-service session." + ) + + self._session_data = fetch.create_session() return {} - def teardown_session(self) -> None: + def teardown_session(self) -> dict[str, typing.Any]: """Teardown and cleanup a previously-created session.""" - # Things to do here (from the prototype): - # - Dump service status (where?) - # - Revoke session token - # - Dump session report - # - Delete session - # - Clean session resources? + if self._session_data is None: + raise ValueError( + "teardown_session() called with no live fetch-service session." + ) + report = fetch.teardown_session(self._session_data) + self._session_data = None + return report def shutdown(self, *, force: bool = False) -> None: """Stop the fetch-service. diff --git a/pyproject.toml b/pyproject.toml index 4c3186d9..4572f608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -284,6 +284,7 @@ lint.ignore = [ "ANN", # Ignore type annotations in tests "S101", # Allow assertions in tests "S103", # Allow `os.chmod` setting a permissive mask `0o555` on file or directory + "S105", "S106", "S107", # Allow hardcoded "passwords" in test files. "S108", # Allow Probable insecure usage of temporary file or directory "PLR0913", # Allow many arguments for test functions "PT004", # Allow fixtures that don't return anything to not start with underscores diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index 8cdb373a..b7f11343 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -16,7 +16,9 @@ """Tests for FetchService.""" import shutil import socket +from unittest import mock +import craft_providers import pytest from craft_application import errors, fetch, services @@ -88,3 +90,19 @@ def test_shutdown_service(app_service): # shutdown(force=True) must stop the fetch-service. app_service.shutdown(force=True) assert not fetch.is_service_online() + + +def test_create_teardown_session(app_service): + app_service.setup() + + assert len(fetch.get_service_status()["active-sessions"]) == 0 + + app_service.create_session( + instance=mock.MagicMock(spec_set=craft_providers.Executor) + ) + assert len(fetch.get_service_status()["active-sessions"]) == 1 + + report = app_service.teardown_session() + assert len(fetch.get_service_status()["active-sessions"]) == 0 + + assert "artefacts" in report diff --git a/tests/unit/git/test_git.py b/tests/unit/git/test_git.py index f2c0c5a4..8c6bcd95 100644 --- a/tests/unit/git/test_git.py +++ b/tests/unit/git/test_git.py @@ -549,7 +549,7 @@ def test_push_url_hide_token(url, expected_url, mocker, empty_working_directory) repo.push_url( remote_url=url, remote_branch="test-branch", - token="test-token", # noqa: S106 + token="test-token", ) # token should be hidden in the log output diff --git a/tests/unit/services/test_fetch.py b/tests/unit/services/test_fetch.py new file mode 100644 index 00000000..a615ee4d --- /dev/null +++ b/tests/unit/services/test_fetch.py @@ -0,0 +1,53 @@ +# This file is part of craft-application. +# +# Copyright 2024 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +"""Unit tests for the FetchService. + +Note that most of the fetch-service functionality is already tested either on: +- unit/test_fetch.py, for unit tests of the endpoint calls, or; +- integration/services/test_fetch.py, for full integration tests. + +As such, this module mostly unit-tests error paths coming from wrong usage of +the FetchService class. +""" +import re +from unittest.mock import MagicMock + +import pytest +from craft_application import fetch, services + + +@pytest.fixture() +def fetch_service(app, fake_services): + return services.FetchService(app, fake_services) + + +def test_create_session_already_exists(fetch_service): + fetch_service._session_data = fetch.SessionData(id="id", token="token") + + expected = re.escape( + "create_session() called but there's already a live fetch-service session." + ) + with pytest.raises(ValueError, match=expected): + fetch_service.create_session(instance=MagicMock()) + + +def test_teardown_session_no_session(fetch_service): + expected = re.escape( + "teardown_session() called with no live fetch-service session." + ) + + with pytest.raises(ValueError, match=expected): + fetch_service.teardown_session() diff --git a/tests/unit/test_application_fetch.py b/tests/unit/test_application_fetch.py index 8caeb62e..6d35ff2e 100644 --- a/tests/unit/test_application_fetch.py +++ b/tests/unit/test_application_fetch.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License along # with this program. If not, see . """Unit tests for the interaction between the Application and the FetchService.""" +from typing import Any from unittest import mock import craft_providers @@ -45,8 +46,9 @@ def create_session( return {} @override - def teardown_session(self) -> None: + def teardown_session(self) -> dict[str, Any]: self.calls.append("teardown_session") + return {} @override def shutdown(self, *, force: bool = False) -> None: diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index 1b63c5a9..6ef0b7f9 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -20,13 +20,16 @@ import pytest import responses from craft_application import errors, fetch +from responses import matchers CONTROL = fetch._DEFAULT_CONFIG.control PROXY = fetch._DEFAULT_CONFIG.proxy AUTH = fetch._DEFAULT_CONFIG.auth +assert_requests = responses.activate(assert_all_requests_are_fired=True) -@responses.activate + +@assert_requests def test_get_service_status_success(): responses.add( responses.GET, @@ -38,14 +41,14 @@ def test_get_service_status_success(): assert status == {"uptime": 10} -@responses.activate +@assert_requests def test_get_service_status_failure(): responses.add( responses.GET, f"http://localhost:{CONTROL}/status", status=404, ) - expected = "Error querying fetch-service status: 404 Client Error" + expected = "Error with fetch-service GET: 404 Client Error" with pytest.raises(errors.FetchServiceError, match=expected): fetch.get_service_status() @@ -59,7 +62,7 @@ def test_get_service_status_failure(): (404, {"other-key": "value"}, False), ], ) -@responses.activate +@assert_requests def test_is_service_online(status, json, expected): responses.add( responses.GET, @@ -115,3 +118,51 @@ def test_start_service_already_up(mocker): assert mock_is_online.called assert not mock_popen.called + + +@assert_requests +def test_create_session(): + responses.add( + responses.POST, + f"http://localhost:{CONTROL}/session", + json={"id": "my-session-id", "token": "my-session-token"}, + status=200, + ) + + session_data = fetch.create_session() + + assert session_data.session_id == "my-session-id" + assert session_data.token == "my-session-token" + + +@assert_requests +def test_teardown_session(): + session_data = fetch.SessionData(id="my-session-id", token="my-session-token") + + # Call to delete token + responses.delete( + f"http://localhost:{CONTROL}/session/{session_data.session_id}/token", + match=[matchers.json_params_matcher({"token": session_data.token})], + json={}, + status=200, + ) + # Call to get session report + responses.get( + f"http://localhost:{CONTROL}/session/{session_data.session_id}", + json={}, + status=200, + ) + # Call to delete session + responses.delete( + f"http://localhost:{CONTROL}/session/{session_data.session_id}", + json={}, + status=200, + ) + # Call to delete session resources + responses.delete( + f"http://localhost:{CONTROL}/resources/{session_data.session_id}", + json={}, + status=200, + ) + + fetch.teardown_session(session_data) diff --git a/tests/unit/test_secrets.py b/tests/unit/test_secrets.py index 62f0d88a..22db8525 100644 --- a/tests/unit/test_secrets.py +++ b/tests/unit/test_secrets.py @@ -100,7 +100,7 @@ def test_secrets_cache(mocker, monkeypatch): spied_run.assert_called_once_with("echo ${SECRET_1}") -_SECRET = "$(HOST_SECRET:echo ${GIT_VERSION})" # noqa: S105 (this is not a password) +_SECRET = "$(HOST_SECRET:echo ${GIT_VERSION})" # (this is not a password) @pytest.mark.parametrize( From bdb374c72f4b78ff450340c6b497953636d60e66 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 17 Jun 2024 10:18:47 -0300 Subject: [PATCH 45/82] feat: configure build instances for fetch-service (#11) This configuration happens when creating the session. Currently we: - Install the local certificate (added to this repo) - Configure pip, snapd and apt - Refresh apt's package listings to ensure all installations are traceable by the fetch-service This initial implementation *always* refreshes apt listings, even if the project build won't use them (e.g. no stage- or build- packages). In the future we'll be smarter about this. The implementation also only supports LXD instances, just because we need to figure out the instance's gateway address and only the LXD work for that has been done so far. In the future we can support all instance types with a cleaner craft-providers API. --- craft_application/certs/local-ca.crt | 34 ++++++ craft_application/fetch.py | 147 ++++++++++++++++++++++- craft_application/services/fetch.py | 17 +-- tests/integration/services/test_fetch.py | 51 +++++++- tests/unit/test_fetch.py | 87 +++++++++++++- 5 files changed, 316 insertions(+), 20 deletions(-) create mode 100644 craft_application/certs/local-ca.crt diff --git a/craft_application/certs/local-ca.crt b/craft_application/certs/local-ca.crt new file mode 100644 index 00000000..62653dae --- /dev/null +++ b/craft_application/certs/local-ca.crt @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF9DCCA9ygAwIBAgIJAODqYUwoVjJkMA0GCSqGSIb3DQEBCwUAMIGOMQswCQYD +VQQGEwJJTDEPMA0GA1UECAwGQ2VudGVyMQwwCgYDVQQHDANMb2QxEDAOBgNVBAoM +B0dvUHJveHkxEDAOBgNVBAsMB0dvUHJveHkxGjAYBgNVBAMMEWdvcHJveHkuZ2l0 +aHViLmlvMSAwHgYJKoZIhvcNAQkBFhFlbGF6YXJsQGdtYWlsLmNvbTAeFw0xNzA0 +MDUyMDAwMTBaFw0zNzAzMzEyMDAwMTBaMIGOMQswCQYDVQQGEwJJTDEPMA0GA1UE +CAwGQ2VudGVyMQwwCgYDVQQHDANMb2QxEDAOBgNVBAoMB0dvUHJveHkxEDAOBgNV +BAsMB0dvUHJveHkxGjAYBgNVBAMMEWdvcHJveHkuZ2l0aHViLmlvMSAwHgYJKoZI +hvcNAQkBFhFlbGF6YXJsQGdtYWlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIP +ADCCAgoCggIBAJ4Qy+H6hhoY1s0QRcvIhxrjSHaO/RbaFj3rwqcnpOgFq07gRdI9 +3c0TFKQJHpgv6feLRhEvX/YllFYu4J35lM9ZcYY4qlKFuStcX8Jm8fqpgtmAMBzP +sqtqDi8M9RQGKENzU9IFOnCV7SAeh45scMuI3wz8wrjBcH7zquHkvqUSYZz035t9 +V6WTrHyTEvT4w+lFOVN2bA/6DAIxrjBiF6DhoJqnha0SZtDfv77XpwGG3EhA/qoh +hiYrDruYK7zJdESQL44LwzMPupVigqalfv+YHfQjbhT951IVurW2NJgRyBE62dLr +lHYdtT9tCTCrd+KJNMJ+jp9hAjdIu1Br/kifU4F4+4ZLMR9Ueji0GkkPKsYdyMnq +j0p0PogyvP1l4qmboPImMYtaoFuYmMYlebgC9LN10bL91K4+jLt0I1YntEzrqgJo +WsJztYDw543NzSy5W+/cq4XRYgtq1b0RWwuUiswezmMoeyHZ8BQJe2xMjAOllASD +fqa8OK3WABHJpy4zUrnUBiMuPITzD/FuDx4C5IwwlC68gHAZblNqpBZCX0nFCtKj +YOcI2So5HbQ2OC8QF+zGVuduHUSok4hSy2BBfZ1pfvziqBeetWJwFvapGB44nIHh +WKNKvqOxLNIy7e+TGRiWOomrAWM18VSR9LZbBxpJK7PLSzWqYJYTRCZHAgMBAAGj +UzBRMB0GA1UdDgQWBBR4uDD9Y6x7iUoHO+32ioOcw1ICZTAfBgNVHSMEGDAWgBR4 +uDD9Y6x7iUoHO+32ioOcw1ICZTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEB +CwUAA4ICAQAaCEupzGGqcdh+L7BzhX7zyd7yzAKUoLxFrxaZY34Xyj3lcx1XoK6F +AqsH2JM25GixgadzhNt92JP7vzoWeHZtLfstrPS638Y1zZi6toy4E49viYjFk5J0 +C6ZcFC04VYWWx6z0HwJuAS08tZ37JuFXpJGfXJOjZCQyxse0Lg0tuKLMeXDCk2Y3 +Ba0noeuNyHRoWXXPyiUoeApkVCU5gIsyiJSWOjhJ5hpJG06rQNfNYexgKrrraEin +o0jmEMtJMx5TtD83hSnLCnFGBBq5lkE7jgXME1KsbIE3lJZzRX1mQwUK8CJDYxye +i6M/dzSvy0SsPvz8fTAlprXRtWWtJQmxgWENp3Dv+0Pmux/l+ilk7KA4sMXGhsfr +bvTOeWl1/uoFTPYiWR/ww7QEPLq23yDFY04Q7Un0qjIk8ExvaY8lCkXMgc8i7sGY +VfvOYb0zm67EfAQl3TW8Ky5fl5CcxpVCD360Bzi6hwjYixa3qEeBggOixFQBFWft +8wrkKTHpOQXjn4sDPtet8imm9UYEtzWrFX6T9MFYkBR0/yye0FIh9+YPiTA6WB86 +NCNwK5Yl6HuvF97CIH5CdgO+5C7KifUtqTOL8pQKbNwy0S3sNYvB+njGvRpR7pKV +BUnFpB/Atptqr4CUlTXrc5IPLAqAfmwk5IKcwy3EXUbruf9Dwz69YA== +-----END CERTIFICATE----- diff --git a/craft_application/fetch.py b/craft_application/fetch.py index fefc4fee..04336d2f 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -15,16 +15,19 @@ # with this program. If not, see . """Utilities to interact with the fetch-service.""" import contextlib +import io import pathlib import subprocess from dataclasses import dataclass +from importlib import resources from typing import Any, cast +import craft_providers import requests from pydantic import Field from requests.auth import HTTPBasicAuth -from craft_application import errors +from craft_application import errors, util from craft_application.models import CraftBaseModel from craft_application.util import retry @@ -65,6 +68,29 @@ class SessionData(CraftBaseModel): token: str +class NetInfo: + """Network and proxy info linking a fetch-service session and a build instance.""" + + def __init__( + self, instance: craft_providers.Executor, session_data: SessionData + ) -> None: + self._gateway = _get_gateway(instance) + self._session_data = session_data + + @property + def http_proxy(self) -> str: + """Proxy string in the 'http://:@:/.""" + session = self._session_data + port = _DEFAULT_CONFIG.proxy + gw = self._gateway + return f"http://{session.session_id}:{session.token}@{gw}:{port}/" + + @property + def env(self) -> dict[str, str]: + """Environment variables to use for the proxy.""" + return {"http_proxy": self.http_proxy, "https_proxy": self.http_proxy} + + def is_service_online() -> bool: """Whether the fetch-service is up and listening.""" try: @@ -93,6 +119,25 @@ def start_service() -> subprocess.Popen[str] | None: env = {"FETCH_SERVICE_AUTH": _DEFAULT_CONFIG.auth} + # Add the public key for the Ubuntu archives + archive_keyring = ( + "/snap/fetch-service/current/usr/share/keyrings/ubuntu-archive-keyring.gpg" + ) + archive_key_id = "F6ECB3762474EDA9D21B7022871920D1991BC93C" + archive_key = subprocess.check_output( + [ + "gpg", + "--export", + "--armor", + "--no-default-keyring", + "--keyring", + archive_keyring, + archive_key_id, + ], + text=True, + ) + env["FETCH_APT_RELEASE_PUBLIC_KEY"] = archive_key + # Add the ports cmd.append(f"--control-port={_DEFAULT_CONFIG.control}") cmd.append(f"--proxy-port={_DEFAULT_CONFIG.proxy}") @@ -191,6 +236,20 @@ def teardown_session(session_data: SessionData) -> dict[str, Any]: return cast(dict[str, Any], session_report) +def configure_instance( + instance: craft_providers.Executor, session_data: SessionData +) -> dict[str, str]: + """Configure a build instance to use a given fetch-service session.""" + net_info = NetInfo(instance, session_data) + + _install_certificate(instance) + _configure_pip(instance) + _configure_snapd(instance, net_info) + _configure_apt(instance, net_info) + + return net_info.env + + def _service_request( verb: str, endpoint: str, json: dict[str, Any] | None = None ) -> requests.Response: @@ -222,3 +281,89 @@ def _get_service_base_dir() -> pathlib.Path: ["snap", "run", "--shell", "fetch-service"], text=True, input=input_line ) return pathlib.Path(output.strip()) + + +def _install_certificate(instance: craft_providers.Executor) -> None: + + # Push the local certificate + certs = resources.files("craft_application") / "certs" + with resources.as_file(certs) as certs_dir: + instance.push_file( + source=certs_dir / "local-ca.crt", + destination=pathlib.Path("/usr/local/share/ca-certificates/local-ca.crt"), + ) + # Update the certificates db + instance.execute_run( # pyright: ignore[reportUnknownMemberType] + ["/bin/sh", "-c", "/usr/sbin/update-ca-certificates > /dev/null"], + check=True, + ) + + +def _configure_pip(instance: craft_providers.Executor) -> None: + instance.execute_run( # pyright: ignore[reportUnknownMemberType] + ["mkdir", "-p", "/root/.pip"] + ) + pip_config = b"[global]\ncert=/usr/local/share/ca-certificates/local-ca.crt" + instance.push_file_io( + destination=pathlib.Path("/root/.pip/pip.conf"), + content=io.BytesIO(pip_config), + file_mode="0644", + ) + + +def _configure_snapd(instance: craft_providers.Executor, net_info: NetInfo) -> None: + """Configure snapd to use the proxy and see our certificate. + + Note: This *must* be called *after* _install_certificate(), to ensure that + when the snapd restart happens the new cert is there. + """ + instance.execute_run( # pyright: ignore[reportUnknownMemberType] + ["systemctl", "restart", "snapd"] + ) + for config in ("proxy.http", "proxy.https"): + instance.execute_run( # pyright: ignore[reportUnknownMemberType] + ["snap", "set", "system", f"{config}={net_info.http_proxy}"] + ) + + +def _configure_apt(instance: craft_providers.Executor, net_info: NetInfo) -> None: + apt_config = f'Acquire::http::Proxy "{net_info.http_proxy}";\n' + apt_config += f'Acquire::https::Proxy "{net_info.http_proxy}";\n' + + instance.push_file_io( + destination=pathlib.Path("/etc/apt/apt.conf.d/99proxy"), + content=io.BytesIO(apt_config.encode("utf-8")), + file_mode="0644", + ) + instance.execute_run( # pyright: ignore[reportUnknownMemberType] + ["/bin/rm", "-Rf", "/var/lib/apt/lists"], + check=True, + ) + env = cast(dict[str, str | None], net_info.env) + instance.execute_run( # pyright: ignore[reportUnknownMemberType] + ["apt", "update"], + env=env, + check=True, + ) + + +def _get_gateway(instance: craft_providers.Executor) -> str: + from craft_providers.lxd import LXDInstance + + if not isinstance(instance, LXDInstance): + raise TypeError("Don't know how to handle non-lxd instances") + + instance_name = instance.instance_name + project = instance.project + output = subprocess.check_output( + ["lxc", "--project", project, "config", "show", instance_name, "--expanded"], + text=True, + ) + config = util.safe_yaml_load(io.StringIO(output)) + network = config["devices"]["eth0"]["network"] + + route = subprocess.check_output( + ["ip", "route", "show", "dev", network], + text=True, + ) + return route.strip().split()[-1] diff --git a/craft_application/services/fetch.py b/craft_application/services/fetch.py index 17df57f7..1b4de7c0 100644 --- a/craft_application/services/fetch.py +++ b/craft_application/services/fetch.py @@ -57,32 +57,19 @@ def setup(self) -> None: super().setup() self._fetch_process = fetch.start_service() - def create_session( - self, - instance: craft_providers.Executor, # noqa: ARG002 (unused-method-argument) - ) -> dict[str, str]: + def create_session(self, instance: craft_providers.Executor) -> dict[str, str]: """Create a new session. :return: The environment variables that must be used by any process that will use the new session. """ - # Things to do here (from the prototype): - # To create the session: - # - Create a session (POST), store session data (DONE) - # - Find the gateway for the network used by the instance (need API) - # - Return http_proxy/https_proxy with session data - # - # To configure the instance: - # - Install bundled certificate - # - Configure snap proxy - # - Clean APT's cache if self._session_data is not None: raise ValueError( "create_session() called but there's already a live fetch-service session." ) self._session_data = fetch.create_session() - return {} + return fetch.configure_instance(instance, self._session_data) def teardown_session(self) -> dict[str, typing.Any]: """Teardown and cleanup a previously-created session.""" diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index b7f11343..62721870 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -14,20 +14,23 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . """Tests for FetchService.""" +import contextlib import shutil import socket from unittest import mock import craft_providers import pytest -from craft_application import errors, fetch, services +from craft_application import errors, fetch, services, util +from craft_application.models import BuildInfo +from craft_providers import bases @pytest.fixture(autouse=True) def _set_test_base_dir(mocker): original = fetch._get_service_base_dir() test_dir = original / "test" - test_dir.mkdir(exist_ok=False) + test_dir.mkdir(exist_ok=True) mocker.patch.object(fetch, "_get_service_base_dir", return_value=test_dir) yield shutil.rmtree(test_dir) @@ -92,7 +95,8 @@ def test_shutdown_service(app_service): assert not fetch.is_service_online() -def test_create_teardown_session(app_service): +def test_create_teardown_session(app_service, mocker): + mocker.patch.object(fetch, "_get_gateway", return_value="127.0.0.1") app_service.setup() assert len(fetch.get_service_status()["active-sessions"]) == 0 @@ -106,3 +110,44 @@ def test_create_teardown_session(app_service): assert len(fetch.get_service_status()["active-sessions"]) == 0 assert "artefacts" in report + + +@pytest.fixture() +def lxd_instance(snap_safe_tmp_path, provider_service): + provider_service.get_provider("lxd") + + arch = util.get_host_architecture() + build_info = BuildInfo("foo", arch, arch, bases.BaseName("ubuntu", "22.04")) + instance = provider_service.instance(build_info, work_dir=snap_safe_tmp_path) + + with instance as executor: + yield executor + + if executor is not None: + with contextlib.suppress(craft_providers.ProviderError): + executor.delete() + + +def test_build_instance_integration(app_service, lxd_instance): + + app_service.setup() + + env = app_service.create_session(lxd_instance) + try: + lxd_instance.execute_run( + ["apt", "install", "-y", "hello"], check=True, env=env, capture_output=True + ) + finally: + report = app_service.teardown_session() + + # Check that the installation of the "hello" deb went through the inspector. + debs = set() + deb_type = "application/vnd.debian.binary-package" + + for artefact in report["artefacts"]: + metadata_name = artefact["metadata"]["name"] + metadata_type = artefact["metadata"]["type"] + if metadata_name == "hello" and metadata_type == deb_type: + debs.add(metadata_name) + + assert "hello" in debs diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index 6ef0b7f9..9aab31d1 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -15,11 +15,14 @@ # along with this program. If not, see . """Tests for fetch-service-related functions.""" import subprocess +from pathlib import Path +from unittest import mock from unittest.mock import call import pytest import responses from craft_application import errors, fetch +from craft_providers.lxd import LXDInstance from responses import matchers CONTROL = fetch._DEFAULT_CONFIG.control @@ -81,6 +84,9 @@ def test_start_service(mocker, tmp_path): mock_get_status = mocker.patch.object( fetch, "get_service_status", return_value={"uptime": 10} ) + mock_archive_key = mocker.patch.object( + subprocess, "check_output", return_value="DEADBEEF" + ) mock_popen = mocker.patch.object(subprocess, "Popen") mock_process = mock_popen.return_value @@ -92,6 +98,18 @@ def test_start_service(mocker, tmp_path): assert mock_is_online.called assert mock_base_dir.called assert mock_get_status.called + mock_archive_key.assert_called_once_with( + [ + "gpg", + "--export", + "--armor", + "--no-default-keyring", + "--keyring", + "/snap/fetch-service/current/usr/share/keyrings/ubuntu-archive-keyring.gpg", + "F6ECB3762474EDA9D21B7022871920D1991BC93C", + ], + text=True, + ) popen_call = mock_popen.mock_calls[0] assert popen_call == call( @@ -102,7 +120,7 @@ def test_start_service(mocker, tmp_path): f"--config={tmp_path/'config'}", f"--spool={tmp_path/'spool'}", ], - env={"FETCH_SERVICE_AUTH": AUTH}, + env={"FETCH_SERVICE_AUTH": AUTH, "FETCH_APT_RELEASE_PUBLIC_KEY": "DEADBEEF"}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, @@ -166,3 +184,70 @@ def test_teardown_session(): ) fetch.teardown_session(session_data) + + +def test_configure_build_instance(mocker): + mocker.patch.object(fetch, "_get_gateway", return_value="127.0.0.1") + + session_data = fetch.SessionData(id="my-session-id", token="my-session-token") + instance = mock.MagicMock(spec_set=LXDInstance) + assert isinstance(instance, LXDInstance) + + expected_proxy = f"http://my-session-id:my-session-token@127.0.0.1:{PROXY}/" + expected_env = {"http_proxy": expected_proxy, "https_proxy": expected_proxy} + + env = fetch.configure_instance(instance, session_data) + assert env == expected_env + + # Execution calls on the instance + assert instance.execute_run.mock_calls == [ + call( + ["/bin/sh", "-c", "/usr/sbin/update-ca-certificates > /dev/null"], + check=True, + ), + call(["mkdir", "-p", "/root/.pip"]), + call(["systemctl", "restart", "snapd"]), + call( + [ + "snap", + "set", + "system", + f"proxy.http={expected_proxy}", + ] + ), + call( + [ + "snap", + "set", + "system", + f"proxy.https={expected_proxy}", + ] + ), + call(["/bin/rm", "-Rf", "/var/lib/apt/lists"], check=True), + call( + ["apt", "update"], + env=expected_env, + check=True, + ), + ] + + # Files pushed to the instance + assert instance.push_file.mock_calls == [ + call( + source=mocker.ANY, + destination=Path("/usr/local/share/ca-certificates/local-ca.crt"), + ) + ] + + assert instance.push_file_io.mock_calls == [ + call( + destination=Path("/root/.pip/pip.conf"), + content=mocker.ANY, + file_mode="0644", + ), + call( + destination=Path("/etc/apt/apt.conf.d/99proxy"), + content=mocker.ANY, + file_mode="0644", + ), + ] From 6bcddf9c1fa96b94f2c30c2ba703eba2e6dbbdb3 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Thu, 4 Jul 2024 16:20:31 -0300 Subject: [PATCH 46/82] fix: set REQUESTS_CA_BUNDLE for the fetch-service env. (#16) This variable points to the local-ca.crt self-signed certificate, effectively telling requests to trust that certificate when negotiating https requests. --- craft_application/fetch.py | 14 ++++- tests/integration/services/test_fetch.py | 79 +++++++++++++++++++++--- tests/unit/test_fetch.py | 6 +- 3 files changed, 89 insertions(+), 10 deletions(-) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index 04336d2f..b4feaa9d 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -60,6 +60,11 @@ def auth(self) -> str: password="craft", # noqa: S106 (hardcoded-password-func-arg) ) +# The path to the fetch-service's certificate inside the build instance. +_FETCH_CERT_INSTANCE_PATH = pathlib.Path( + "/usr/local/share/ca-certificates/local-ca.crt" +) + class SessionData(CraftBaseModel): """Fetch service session data.""" @@ -88,7 +93,12 @@ def http_proxy(self) -> str: @property def env(self) -> dict[str, str]: """Environment variables to use for the proxy.""" - return {"http_proxy": self.http_proxy, "https_proxy": self.http_proxy} + return { + "http_proxy": self.http_proxy, + "https_proxy": self.http_proxy, + # This makes the requests lib take our cert into account. + "REQUESTS_CA_BUNDLE": str(_FETCH_CERT_INSTANCE_PATH), + } def is_service_online() -> bool: @@ -290,7 +300,7 @@ def _install_certificate(instance: craft_providers.Executor) -> None: with resources.as_file(certs) as certs_dir: instance.push_file( source=certs_dir / "local-ca.crt", - destination=pathlib.Path("/usr/local/share/ca-certificates/local-ca.crt"), + destination=_FETCH_CERT_INSTANCE_PATH, ) # Update the certificates db instance.execute_run( # pyright: ignore[reportUnknownMemberType] diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index 62721870..f8caad53 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -15,8 +15,11 @@ # along with this program. If not, see . """Tests for FetchService.""" import contextlib +import io +import pathlib import shutil import socket +import textwrap from unittest import mock import craft_providers @@ -112,6 +115,42 @@ def test_create_teardown_session(app_service, mocker): assert "artefacts" in report +# Bash script to setup the build instance before the actual testing. +setup_environment = ( + textwrap.dedent( + """ + #! /bin/bash + set -euo pipefail + + apt install -y python3.10-venv + python3 -m venv venv + venv/bin/pip install requests +""" + ) + .strip() + .encode("ascii") +) + +wheel_url = ( + "https://files.pythonhosted.org/packages/0f/ec/" + "a9b769274512ea65d8484c2beb8c3d2686d1323b450ce9ee6d09452ac430/" + "craft_application-3.0.0-py3-none-any.whl" +) +# Bash script to fetch the craft-application wheel. +check_requests = ( + textwrap.dedent( + f""" + #! /bin/bash + set -euo pipefail + + venv/bin/python -c "import requests; requests.get('{wheel_url}').raise_for_status()" +""" + ) + .strip() + .encode("ascii") +) + + @pytest.fixture() def lxd_instance(snap_safe_tmp_path, provider_service): provider_service.get_provider("lxd") @@ -121,6 +160,16 @@ def lxd_instance(snap_safe_tmp_path, provider_service): instance = provider_service.instance(build_info, work_dir=snap_safe_tmp_path) with instance as executor: + executor.push_file_io( + destination=pathlib.Path("/root/setup-environment.sh"), + content=io.BytesIO(setup_environment), + file_mode="0644", + ) + executor.execute_run( + ["bash", "/root/setup-environment.sh"], + check=True, + capture_output=True, + ) yield executor if executor is not None: @@ -129,25 +178,41 @@ def lxd_instance(snap_safe_tmp_path, provider_service): def test_build_instance_integration(app_service, lxd_instance): - app_service.setup() env = app_service.create_session(lxd_instance) + try: + # Install the hello Ubuntu package. lxd_instance.execute_run( ["apt", "install", "-y", "hello"], check=True, env=env, capture_output=True ) + + # Download the craft-application wheel. + lxd_instance.push_file_io( + destination=pathlib.Path("/root/check-requests.sh"), + content=io.BytesIO(check_requests), + file_mode="0644", + ) + lxd_instance.execute_run( + ["bash", "/root/check-requests.sh"], + check=True, + env=env, + capture_output=True, + ) finally: report = app_service.teardown_session() - # Check that the installation of the "hello" deb went through the inspector. - debs = set() - deb_type = "application/vnd.debian.binary-package" + artefacts_and_types: list[tuple[str, str]] = [] for artefact in report["artefacts"]: metadata_name = artefact["metadata"]["name"] metadata_type = artefact["metadata"]["type"] - if metadata_name == "hello" and metadata_type == deb_type: - debs.add(metadata_name) - assert "hello" in debs + artefacts_and_types.append((metadata_name, metadata_type)) + + # Check that the installation of the "hello" deb went through the inspector. + assert ("hello", "application/vnd.debian.binary-package") in artefacts_and_types + + # Check that the fetching of the "craft-application" wheel went through the inspector. + assert ("craft-application", "application/x.python.wheel") in artefacts_and_types diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index 9aab31d1..cd102b2d 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -194,7 +194,11 @@ def test_configure_build_instance(mocker): assert isinstance(instance, LXDInstance) expected_proxy = f"http://my-session-id:my-session-token@127.0.0.1:{PROXY}/" - expected_env = {"http_proxy": expected_proxy, "https_proxy": expected_proxy} + expected_env = { + "http_proxy": expected_proxy, + "https_proxy": expected_proxy, + "REQUESTS_CA_BUNDLE": "/usr/local/share/ca-certificates/local-ca.crt", + } env = fetch.configure_instance(instance, session_data) assert env == expected_env From 8c1096d9dcaa22c3d249a29a0409ca034b5969cc Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Tue, 23 Jul 2024 17:32:09 -0300 Subject: [PATCH 47/82] feat: generate the fetch-service certificate Recent versions of the fetch-service not longer come with a fixed, "built-in" certificate. Instead, clients are expected to generate their own. To do this, we use a location that is shared among all Craft applications - the key and certificate are generated if not found, and are otherwise shared by all instances of all applications. Fixes #21 --- craft_application/certs/local-ca.crt | 34 -------- craft_application/fetch.py | 102 +++++++++++++++++++++-- tests/integration/services/test_fetch.py | 36 +++++++- tests/unit/test_fetch.py | 23 ++++- 4 files changed, 151 insertions(+), 44 deletions(-) delete mode 100644 craft_application/certs/local-ca.crt diff --git a/craft_application/certs/local-ca.crt b/craft_application/certs/local-ca.crt deleted file mode 100644 index 62653dae..00000000 --- a/craft_application/certs/local-ca.crt +++ /dev/null @@ -1,34 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIF9DCCA9ygAwIBAgIJAODqYUwoVjJkMA0GCSqGSIb3DQEBCwUAMIGOMQswCQYD -VQQGEwJJTDEPMA0GA1UECAwGQ2VudGVyMQwwCgYDVQQHDANMb2QxEDAOBgNVBAoM -B0dvUHJveHkxEDAOBgNVBAsMB0dvUHJveHkxGjAYBgNVBAMMEWdvcHJveHkuZ2l0 -aHViLmlvMSAwHgYJKoZIhvcNAQkBFhFlbGF6YXJsQGdtYWlsLmNvbTAeFw0xNzA0 -MDUyMDAwMTBaFw0zNzAzMzEyMDAwMTBaMIGOMQswCQYDVQQGEwJJTDEPMA0GA1UE -CAwGQ2VudGVyMQwwCgYDVQQHDANMb2QxEDAOBgNVBAoMB0dvUHJveHkxEDAOBgNV -BAsMB0dvUHJveHkxGjAYBgNVBAMMEWdvcHJveHkuZ2l0aHViLmlvMSAwHgYJKoZI -hvcNAQkBFhFlbGF6YXJsQGdtYWlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIP -ADCCAgoCggIBAJ4Qy+H6hhoY1s0QRcvIhxrjSHaO/RbaFj3rwqcnpOgFq07gRdI9 -3c0TFKQJHpgv6feLRhEvX/YllFYu4J35lM9ZcYY4qlKFuStcX8Jm8fqpgtmAMBzP -sqtqDi8M9RQGKENzU9IFOnCV7SAeh45scMuI3wz8wrjBcH7zquHkvqUSYZz035t9 -V6WTrHyTEvT4w+lFOVN2bA/6DAIxrjBiF6DhoJqnha0SZtDfv77XpwGG3EhA/qoh -hiYrDruYK7zJdESQL44LwzMPupVigqalfv+YHfQjbhT951IVurW2NJgRyBE62dLr -lHYdtT9tCTCrd+KJNMJ+jp9hAjdIu1Br/kifU4F4+4ZLMR9Ueji0GkkPKsYdyMnq -j0p0PogyvP1l4qmboPImMYtaoFuYmMYlebgC9LN10bL91K4+jLt0I1YntEzrqgJo -WsJztYDw543NzSy5W+/cq4XRYgtq1b0RWwuUiswezmMoeyHZ8BQJe2xMjAOllASD -fqa8OK3WABHJpy4zUrnUBiMuPITzD/FuDx4C5IwwlC68gHAZblNqpBZCX0nFCtKj -YOcI2So5HbQ2OC8QF+zGVuduHUSok4hSy2BBfZ1pfvziqBeetWJwFvapGB44nIHh -WKNKvqOxLNIy7e+TGRiWOomrAWM18VSR9LZbBxpJK7PLSzWqYJYTRCZHAgMBAAGj -UzBRMB0GA1UdDgQWBBR4uDD9Y6x7iUoHO+32ioOcw1ICZTAfBgNVHSMEGDAWgBR4 -uDD9Y6x7iUoHO+32ioOcw1ICZTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEB -CwUAA4ICAQAaCEupzGGqcdh+L7BzhX7zyd7yzAKUoLxFrxaZY34Xyj3lcx1XoK6F -AqsH2JM25GixgadzhNt92JP7vzoWeHZtLfstrPS638Y1zZi6toy4E49viYjFk5J0 -C6ZcFC04VYWWx6z0HwJuAS08tZ37JuFXpJGfXJOjZCQyxse0Lg0tuKLMeXDCk2Y3 -Ba0noeuNyHRoWXXPyiUoeApkVCU5gIsyiJSWOjhJ5hpJG06rQNfNYexgKrrraEin -o0jmEMtJMx5TtD83hSnLCnFGBBq5lkE7jgXME1KsbIE3lJZzRX1mQwUK8CJDYxye -i6M/dzSvy0SsPvz8fTAlprXRtWWtJQmxgWENp3Dv+0Pmux/l+ilk7KA4sMXGhsfr -bvTOeWl1/uoFTPYiWR/ww7QEPLq23yDFY04Q7Un0qjIk8ExvaY8lCkXMgc8i7sGY -VfvOYb0zm67EfAQl3TW8Ky5fl5CcxpVCD360Bzi6hwjYixa3qEeBggOixFQBFWft -8wrkKTHpOQXjn4sDPtet8imm9UYEtzWrFX6T9MFYkBR0/yye0FIh9+YPiTA6WB86 -NCNwK5Yl6HuvF97CIH5CdgO+5C7KifUtqTOL8pQKbNwy0S3sNYvB+njGvRpR7pKV -BUnFpB/Atptqr4CUlTXrc5IPLAqAfmwk5IKcwy3EXUbruf9Dwz69YA== ------END CERTIFICATE----- diff --git a/craft_application/fetch.py b/craft_application/fetch.py index b4feaa9d..70fc9948 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -19,10 +19,10 @@ import pathlib import subprocess from dataclasses import dataclass -from importlib import resources from typing import Any, cast import craft_providers +import platformdirs import requests from pydantic import Field from requests.auth import HTTPBasicAuth @@ -160,6 +160,11 @@ def start_service() -> subprocess.Popen[str] | None: dir_path.mkdir(exist_ok=True) cmd.append(f"--{dir_name}={dir_path}") + cert, cert_key = _obtain_certificate() + + cmd.append(f"--cert={cert}") + cmd.append(f"--key={cert_key}") + fetch_process = subprocess.Popen( cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) @@ -296,12 +301,11 @@ def _get_service_base_dir() -> pathlib.Path: def _install_certificate(instance: craft_providers.Executor) -> None: # Push the local certificate - certs = resources.files("craft_application") / "certs" - with resources.as_file(certs) as certs_dir: - instance.push_file( - source=certs_dir / "local-ca.crt", - destination=_FETCH_CERT_INSTANCE_PATH, - ) + cert, _key = _obtain_certificate() + instance.push_file( + source=cert, + destination=_FETCH_CERT_INSTANCE_PATH, + ) # Update the certificates db instance.execute_run( # pyright: ignore[reportUnknownMemberType] ["/bin/sh", "-c", "/usr/sbin/update-ca-certificates > /dev/null"], @@ -377,3 +381,87 @@ def _get_gateway(instance: craft_providers.Executor) -> str: text=True, ) return route.strip().split()[-1] + + +def _obtain_certificate() -> tuple[pathlib.Path, pathlib.Path]: + """Retrieve, possibly creating, the certificate and key for the fetch service. + + :return: The full paths to the self-signed certificate and its private key. + """ + cert_dir = _get_certificate_dir() + + cert_dir.mkdir(parents=True, exist_ok=True) + + cert = cert_dir / "local-ca.pem" + key = cert_dir / "local-ca.key.pem" + + if cert.is_file() and key.is_file(): + # Certificate and key already generated + # TODO check that the certificate hasn't expired + return cert, key + + # At least one is missing, regenerate both + key_tmp = cert_dir / "key-tmp.pem" + cert_tmp = cert_dir / "cert-tmp.pem" + + # Create the key + subprocess.run( + [ + "openssl", + "genrsa", + "-aes256", + "-passout", + "pass:1", + "-out", + key_tmp, + "4096", + ], + check=True, + ) + + subprocess.run( + [ + "openssl", + "rsa", + "-passin", + "pass:1", + "-in", + key_tmp, + "-out", + key_tmp, + ], + check=True, + ) + + # Create a certificate with the key + subprocess.run( + [ + "openssl", + "req", + "-subj", + "/CN=root@localhost", + "-key", + key_tmp, + "-new", + "-x509", + "-days", + "7300", + "-sha256", + "-extensions", + "v3_ca", + "-out", + cert_tmp, + ], + check=True, + ) + + cert_tmp.rename(cert) + key_tmp.rename(key) + + return cert, key + + +def _get_certificate_dir() -> pathlib.Path: + """Get the location that should contain the fetch-service certificate and key.""" + data_dir = pathlib.Path(platformdirs.user_data_dir(appname="craft-application")) + return data_dir / "fetch-certificate" diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index f8caad53..ace7259a 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -20,6 +20,7 @@ import shutil import socket import textwrap +from functools import cache from unittest import mock import craft_providers @@ -29,13 +30,40 @@ from craft_providers import bases +@cache +def _get_fake_certificate_dir(): + base_dir = fetch._get_service_base_dir() + + return base_dir / "test-craft-app/fetch-certificate" + + +@pytest.fixture(autouse=True, scope="session") +def _set_test_certificate_dir(): + """A session-scoped fixture so that we generate the certificate only once""" + cert_dir = _get_fake_certificate_dir() + if cert_dir.is_dir(): + shutil.rmtree(cert_dir) + + with mock.patch.object(fetch, "_get_certificate_dir", return_value=cert_dir): + fetch._obtain_certificate() + + yield + + shutil.rmtree(cert_dir) + + @pytest.fixture(autouse=True) -def _set_test_base_dir(mocker): +def _set_test_base_dirs(mocker): original = fetch._get_service_base_dir() test_dir = original / "test" test_dir.mkdir(exist_ok=True) mocker.patch.object(fetch, "_get_service_base_dir", return_value=test_dir) + + cert_dir = _get_fake_certificate_dir() + mocker.patch.object(fetch, "_get_certificate_dir", return_value=cert_dir) + yield + shutil.rmtree(test_dir) @@ -212,7 +240,11 @@ def test_build_instance_integration(app_service, lxd_instance): artefacts_and_types.append((metadata_name, metadata_type)) # Check that the installation of the "hello" deb went through the inspector. - assert ("hello", "application/vnd.debian.binary-package") in artefacts_and_types + + # NOTE: the right type is missing on deb artefacts currently - the "type" + # field is empty. If this fails, set "application/vnd.debian.binary-package" + # instead of "". + assert ("hello", "") in artefacts_and_types # Check that the fetching of the "craft-application" wheel went through the inspector. assert ("craft-application", "application/x.python.wheel") in artefacts_and_types diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index cd102b2d..471281ba 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -19,6 +19,7 @@ from unittest import mock from unittest.mock import call +import platformdirs import pytest import responses from craft_application import errors, fetch @@ -88,6 +89,11 @@ def test_start_service(mocker, tmp_path): subprocess, "check_output", return_value="DEADBEEF" ) + fake_cert, fake_key = tmp_path / "cert.crt", tmp_path / "key.pem" + mock_obtain_certificate = mocker.patch.object( + fetch, "_obtain_certificate", return_value=(fake_cert, fake_key) + ) + mock_popen = mocker.patch.object(subprocess, "Popen") mock_process = mock_popen.return_value mock_process.poll.return_value = None @@ -111,6 +117,8 @@ def test_start_service(mocker, tmp_path): text=True, ) + assert mock_obtain_certificate.called + popen_call = mock_popen.mock_calls[0] assert popen_call == call( [ @@ -119,6 +127,8 @@ def test_start_service(mocker, tmp_path): f"--proxy-port={PROXY}", f"--config={tmp_path/'config'}", f"--spool={tmp_path/'spool'}", + f"--cert={fake_cert}", + f"--key={fake_key}", ], env={"FETCH_SERVICE_AUTH": AUTH, "FETCH_APT_RELEASE_PUBLIC_KEY": "DEADBEEF"}, stdout=subprocess.PIPE, @@ -188,6 +198,9 @@ def test_teardown_session(): def test_configure_build_instance(mocker): mocker.patch.object(fetch, "_get_gateway", return_value="127.0.0.1") + mocker.patch.object( + fetch, "_obtain_certificate", return_value=("fake-cert.crt", "key.pem") + ) session_data = fetch.SessionData(id="my-session-id", token="my-session-token") instance = mock.MagicMock(spec_set=LXDInstance) @@ -238,7 +251,7 @@ def test_configure_build_instance(mocker): # Files pushed to the instance assert instance.push_file.mock_calls == [ call( - source=mocker.ANY, + source="fake-cert.crt", destination=Path("/usr/local/share/ca-certificates/local-ca.crt"), ) ] @@ -255,3 +268,11 @@ def test_configure_build_instance(mocker): file_mode="0644", ), ] + + +def test_get_certificate_dir(): + cert_dir = fetch._get_certificate_dir() + + data_dir = Path(platformdirs.user_data_path("craft-application")) + expected = data_dir / "fetch-certificate" + assert cert_dir == expected From 5a369df3b81b5de6bf38d1f1c1e8853bab4f7907 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Thu, 25 Jul 2024 09:19:59 -0300 Subject: [PATCH 48/82] fix: set CARGO_HTTP_CAINFO for the fetch-service env This lets cargo allow our self-signed certificate when making https requests to its registry. Fixes #20 --- craft_application/fetch.py | 2 ++ tests/unit/test_fetch.py | 1 + 2 files changed, 3 insertions(+) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index 70fc9948..a2a83650 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -98,6 +98,8 @@ def env(self) -> dict[str, str]: "https_proxy": self.http_proxy, # This makes the requests lib take our cert into account. "REQUESTS_CA_BUNDLE": str(_FETCH_CERT_INSTANCE_PATH), + # Same, but for cargo. + "CARGO_HTTP_CAINFO": str(_FETCH_CERT_INSTANCE_PATH), } diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index 471281ba..f95f6580 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -211,6 +211,7 @@ def test_configure_build_instance(mocker): "http_proxy": expected_proxy, "https_proxy": expected_proxy, "REQUESTS_CA_BUNDLE": "/usr/local/share/ca-certificates/local-ca.crt", + "CARGO_HTTP_CAINFO": "/usr/local/share/ca-certificates/local-ca.crt", } env = fetch.configure_instance(instance, session_data) From ae025ad57c6c53aba5b1b99abcab21ba6a4539e6 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 26 Jul 2024 13:42:45 -0300 Subject: [PATCH 49/82] chore(test): set correct deb mimetype back Now that the fetch-service's bug is fixed we can revert our temporary change --- tests/integration/services/test_fetch.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index ace7259a..b6978b89 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -240,11 +240,7 @@ def test_build_instance_integration(app_service, lxd_instance): artefacts_and_types.append((metadata_name, metadata_type)) # Check that the installation of the "hello" deb went through the inspector. - - # NOTE: the right type is missing on deb artefacts currently - the "type" - # field is empty. If this fails, set "application/vnd.debian.binary-package" - # instead of "". - assert ("hello", "") in artefacts_and_types + assert ("hello", "application/vnd.debian.binary-package") in artefacts_and_types # Check that the fetching of the "craft-application" wheel went through the inspector. assert ("craft-application", "application/x.python.wheel") in artefacts_and_types From d521195275759bd78d9f91788cd04e6f3d40fa16 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Thu, 27 Jun 2024 09:16:11 -0300 Subject: [PATCH 50/82] feat: dump the fetch-service report This is temporary - in the near future we'll use the in-memory report to generate a 'proper' manifest, but for now this lets us at least have something to inspect/validate. --- craft_application/services/fetch.py | 11 +++++++++++ tests/integration/services/test_fetch.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/craft_application/services/fetch.py b/craft_application/services/fetch.py index 1b4de7c0..5549b4d2 100644 --- a/craft_application/services/fetch.py +++ b/craft_application/services/fetch.py @@ -16,10 +16,13 @@ """Service class to communicate with the fetch-service.""" from __future__ import annotations +import json +import pathlib import subprocess import typing import craft_providers +from craft_cli import emit from typing_extensions import override from craft_application import fetch, services @@ -78,6 +81,14 @@ def teardown_session(self) -> dict[str, typing.Any]: "teardown_session() called with no live fetch-service session." ) report = fetch.teardown_session(self._session_data) + + # TODO for now we just dump the session report locally + # (will use it in the future to create a manifest) + report_path = pathlib.Path("session-report.json") + with report_path.open("w") as f: + json.dump(report, f, ensure_ascii=False, indent=4) + emit.debug(f"Session report dumped to {report_path}") + self._session_data = None return report diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index b6978b89..ce773f0c 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -126,7 +126,8 @@ def test_shutdown_service(app_service): assert not fetch.is_service_online() -def test_create_teardown_session(app_service, mocker): +def test_create_teardown_session(app_service, mocker, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) mocker.patch.object(fetch, "_get_gateway", return_value="127.0.0.1") app_service.setup() @@ -137,10 +138,15 @@ def test_create_teardown_session(app_service, mocker): ) assert len(fetch.get_service_status()["active-sessions"]) == 1 + # Check that when closing the session its report is dumped to cwd + expected_report = tmp_path / "session-report.json" + assert not expected_report.is_file() + report = app_service.teardown_session() assert len(fetch.get_service_status()["active-sessions"]) == 0 assert "artefacts" in report + assert expected_report.is_file() # Bash script to setup the build instance before the actual testing. @@ -205,7 +211,9 @@ def lxd_instance(snap_safe_tmp_path, provider_service): executor.delete() -def test_build_instance_integration(app_service, lxd_instance): +def test_build_instance_integration(app_service, lxd_instance, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + app_service.setup() env = app_service.create_session(lxd_instance) From 33595c0feba96cfdf7eee0d2418aed415fcee17d Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 26 Jul 2024 13:37:54 -0300 Subject: [PATCH 51/82] feat(fetch): default to creating permissive sessions The strict mode is not yet implemented fully, so until further notice we should create permissive sessions. --- craft_application/fetch.py | 7 ++++++- tests/unit/test_fetch.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index a2a83650..be7c3c98 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -167,6 +167,9 @@ def start_service() -> subprocess.Popen[str] | None: cmd.append(f"--cert={cert}") cmd.append(f"--key={cert_key}") + # Accept permissive sessions + cmd.append("--permissive-mode") + fetch_process = subprocess.Popen( cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) @@ -221,7 +224,9 @@ def create_session() -> SessionData: :return: a SessionData object containing the session's id and token. """ - data = _service_request("post", "session", json={}).json() + # For now we'll always create permissive (as opposed to strict) sessions. + json = {"policy": "permissive"} + data = _service_request("post", "session", json=json).json() return SessionData.unmarshal(data=data) diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index f95f6580..f1a5bbe1 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -129,6 +129,7 @@ def test_start_service(mocker, tmp_path): f"--spool={tmp_path/'spool'}", f"--cert={fake_cert}", f"--key={fake_key}", + "--permissive-mode", ], env={"FETCH_SERVICE_AUTH": AUTH, "FETCH_APT_RELEASE_PUBLIC_KEY": "DEADBEEF"}, stdout=subprocess.PIPE, @@ -155,6 +156,7 @@ def test_create_session(): f"http://localhost:{CONTROL}/session", json={"id": "my-session-id", "token": "my-session-token"}, status=200, + match=[matchers.json_params_matcher({"policy": "permissive"})], ) session_data = fetch.create_session() From 5827eeb5ce14f4ef06b43e1cffea0e87234c51f1 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 29 Jul 2024 10:09:28 -0300 Subject: [PATCH 52/82] fix: use fetch-service's common dir for certificate The snap's current confinement does not let it access the user's home files, so /.local/... is inaccessible. --- craft_application/fetch.py | 6 +++--- tests/unit/test_fetch.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index be7c3c98..5ad83d8f 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -22,7 +22,6 @@ from typing import Any, cast import craft_providers -import platformdirs import requests from pydantic import Field from requests.auth import HTTPBasicAuth @@ -470,5 +469,6 @@ def _obtain_certificate() -> tuple[pathlib.Path, pathlib.Path]: def _get_certificate_dir() -> pathlib.Path: """Get the location that should contain the fetch-service certificate and key.""" - data_dir = pathlib.Path(platformdirs.user_data_dir(appname="craft-application")) - return data_dir / "fetch-certificate" + base_dir = _get_service_base_dir() + + return base_dir / "craft/fetch-certificate" diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index f1a5bbe1..34fe8f25 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -19,7 +19,6 @@ from unittest import mock from unittest.mock import call -import platformdirs import pytest import responses from craft_application import errors, fetch @@ -273,9 +272,13 @@ def test_configure_build_instance(mocker): ] -def test_get_certificate_dir(): +def test_get_certificate_dir(mocker): + mocker.patch.object( + fetch, + "_get_service_base_dir", + return_value=Path("/home/user/snap/fetch-service/common"), + ) cert_dir = fetch._get_certificate_dir() - data_dir = Path(platformdirs.user_data_path("craft-application")) - expected = data_dir / "fetch-certificate" + expected = Path("/home/user/snap/fetch-service/common/craft/fetch-certificate") assert cert_dir == expected From b6caa9c8634ee1621b73311a2062c779609a14dc Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 29 Jul 2024 10:17:43 -0300 Subject: [PATCH 53/82] chore: log the fetch-service command line --- craft_application/fetch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index 5ad83d8f..1740dd2d 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -17,12 +17,14 @@ import contextlib import io import pathlib +import shlex import subprocess from dataclasses import dataclass from typing import Any, cast import craft_providers import requests +from craft_cli import emit from pydantic import Field from requests.auth import HTTPBasicAuth @@ -169,6 +171,8 @@ def start_service() -> subprocess.Popen[str] | None: # Accept permissive sessions cmd.append("--permissive-mode") + emit.debug(f"Launching fetch-service with '{shlex.join(cmd)}'") + fetch_process = subprocess.Popen( cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) From b7355b56bb59ca7e59377d23567bc045663ee0c4 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 29 Jul 2024 10:28:49 -0300 Subject: [PATCH 54/82] feat: always shutdown the fetch-service There seems to be a bug currently with creating consecutive sessions, so for now let's always shut down the fetch-service after the managed run is done. --- craft_application/application.py | 2 +- tests/unit/test_application_fetch.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index aef40f9b..07b50c69 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -399,7 +399,7 @@ def run_managed(self, platform: str | None, build_for: str | None) -> None: self.services.fetch.teardown_session() if self.app.features.fetch_service: - self.services.fetch.shutdown() + self.services.fetch.shutdown(force=True) def configure(self, global_args: dict[str, Any]) -> None: """Configure the application using any global arguments.""" diff --git a/tests/unit/test_application_fetch.py b/tests/unit/test_application_fetch.py index 6d35ff2e..4b527e6e 100644 --- a/tests/unit/test_application_fetch.py +++ b/tests/unit/test_application_fetch.py @@ -80,6 +80,6 @@ def test_run_managed_fetch_service(app, fake_project, fake_build_plan): "teardown_session", "create_session", "teardown_session", - # One call to shut down (without `force`) - "shutdown(False)", + # One call to shut down (with `force`) + "shutdown(True)", ] From 15b3c1dac35f281c881148408cbb1d0809eb9a7c Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Mon, 29 Jul 2024 15:28:13 -0300 Subject: [PATCH 55/82] chore: redirect output of 'apt update' With an open_stream() the output shows up in a single, updating line in brief mode. This output is useful because the call takes a while. --- craft_application/fetch.py | 9 ++++----- tests/unit/test_fetch.py | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index 1740dd2d..d61896a3 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -364,11 +364,10 @@ def _configure_apt(instance: craft_providers.Executor, net_info: NetInfo) -> Non check=True, ) env = cast(dict[str, str | None], net_info.env) - instance.execute_run( # pyright: ignore[reportUnknownMemberType] - ["apt", "update"], - env=env, - check=True, - ) + with emit.open_stream() as fd: + instance.execute_run( # pyright: ignore[reportUnknownMemberType] + ["apt", "update"], env=env, check=True, stdout=fd, stderr=fd + ) def _get_gateway(instance: craft_providers.Executor) -> str: diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index 34fe8f25..25e71f2e 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -247,6 +247,8 @@ def test_configure_build_instance(mocker): ["apt", "update"], env=expected_env, check=True, + stdout=mocker.ANY, + stderr=mocker.ANY, ), ] From 9a6d4503ef8ea15613b7ead63979b3f357093c1c Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Wed, 28 Aug 2024 16:39:28 -0300 Subject: [PATCH 56/82] style(lint): fix ruff 0.6.0 fetch-service errors --- tests/conftest.py | 2 +- tests/integration/services/test_fetch.py | 4 ++-- tests/unit/services/test_fetch.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4e1d0adc..db7ed725 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -331,6 +331,6 @@ def _extra_yaml_transform( return yaml_data -@pytest.fixture() +@pytest.fixture def app(app_metadata, fake_services): return FakeApplication(app_metadata, fake_services) diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index ce773f0c..d2a175bd 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -67,7 +67,7 @@ def _set_test_base_dirs(mocker): shutil.rmtree(test_dir) -@pytest.fixture() +@pytest.fixture def app_service(app_metadata, fake_services): fetch_service = services.FetchService(app_metadata, fake_services) yield fetch_service @@ -185,7 +185,7 @@ def test_create_teardown_session(app_service, mocker, tmp_path, monkeypatch): ) -@pytest.fixture() +@pytest.fixture def lxd_instance(snap_safe_tmp_path, provider_service): provider_service.get_provider("lxd") diff --git a/tests/unit/services/test_fetch.py b/tests/unit/services/test_fetch.py index a615ee4d..e5cf0ddb 100644 --- a/tests/unit/services/test_fetch.py +++ b/tests/unit/services/test_fetch.py @@ -29,7 +29,7 @@ from craft_application import fetch, services -@pytest.fixture() +@pytest.fixture def fetch_service(app, fake_services): return services.FetchService(app, fake_services) From a70bf86b758aca94b0c07b33ccfa075674fd264d Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Thu, 29 Aug 2024 17:57:46 -0300 Subject: [PATCH 57/82] fix: better error if the fetch-service is missing (#55) This implementation checks the presence of the fetch-service snap through the hardcoded path (/snap/bin/fetch-service). If, in the future, we need a more sophisticated approach we can copy craft-provider's logic for checking the lxd snap through snapd's api. Fixes #36 --- craft_application/fetch.py | 15 +++++++++++++++ tests/unit/test_fetch.py | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index d61896a3..7ec262e1 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -128,6 +128,16 @@ def start_service() -> subprocess.Popen[str] | None: # Nothing to do, service is already up. return None + # Check that the fetch service is actually installed + if not _check_installed(): + raise errors.FetchServiceError( + "The 'fetch-service' snap is not installed.", + resolution=( + "Install the fetch-service snap via " + "'snap install --channel=beta fetch-service'." + ), + ) + cmd = [_FETCH_BINARY] env = {"FETCH_SERVICE_AUTH": _DEFAULT_CONFIG.auth} @@ -475,3 +485,8 @@ def _get_certificate_dir() -> pathlib.Path: base_dir = _get_service_base_dir() return base_dir / "craft/fetch-certificate" + + +def _check_installed() -> bool: + """Check whether the fetch-service is installed.""" + return pathlib.Path(_FETCH_BINARY).is_file() diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index 25e71f2e..30765de8 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . """Tests for fetch-service-related functions.""" +import re import subprocess from pathlib import Path from unittest import mock @@ -78,6 +79,7 @@ def test_is_service_online(status, json, expected): def test_start_service(mocker, tmp_path): mock_is_online = mocker.patch.object(fetch, "is_service_online", return_value=False) + mocker.patch.object(fetch, "_check_installed", return_value=True) mock_base_dir = mocker.patch.object( fetch, "_get_service_base_dir", return_value=tmp_path ) @@ -148,6 +150,15 @@ def test_start_service_already_up(mocker): assert not mock_popen.called +def test_start_service_not_installed(mocker): + mocker.patch.object(fetch, "is_service_online", return_value=False) + mocker.patch.object(fetch, "_check_installed", return_value=False) + + expected = re.escape("The 'fetch-service' snap is not installed.") + with pytest.raises(errors.FetchServiceError, match=expected): + fetch.start_service() + + @assert_requests def test_create_session(): responses.add( From ef85eeeb04a087a570900c23e3aa3caa0d18b4e3 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 30 Aug 2024 13:36:00 -0300 Subject: [PATCH 58/82] feat: add an argument to enable the fetch service (#53) With this commit the fetch-service service is disabled by default; the way to enable it is through a new "--use-fetch-service" command line parameter. This new parameter is initiallly only available to the "pack" command, as the cache-clearing that the service needs precludes the use of other lifecycle commands, and the manifest that the application will create based on the fetch-service's session report is only produced during packing. Fixes #15 --- craft_application/application.py | 18 +++++--- craft_application/commands/lifecycle.py | 6 +++ tests/unit/commands/test_lifecycle.py | 1 + tests/unit/test_application_fetch.py | 57 ++++++++++++++++--------- 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index 07b50c69..7fd109aa 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -65,9 +65,6 @@ class AppFeatures: build_secrets: bool = False """Support for build-time secrets.""" - fetch_service: bool = False - """Support for the fetch service.""" - @final @dataclass(frozen=True) @@ -159,6 +156,9 @@ def __init__( else: self._work_dir = pathlib.Path.cwd() + # Whether the command execution should use the fetch-service + self._use_fetch_service = False + @property def app_config(self) -> dict[str, Any]: """Get the configuration passed to dispatcher.load_command(). @@ -373,7 +373,7 @@ def run_managed(self, platform: str | None, build_for: str | None) -> None: with self.services.provider.instance( build_info, work_dir=self._work_dir ) as instance: - if self.app.features.fetch_service: + if self._use_fetch_service: session_env = self.services.fetch.create_session(instance) env.update(session_env) @@ -395,10 +395,10 @@ def run_managed(self, platform: str | None, build_for: str | None) -> None: f"Failed to execute {self.app.name} in instance." ) from exc finally: - if self.app.features.fetch_service: + if self._use_fetch_service: self.services.fetch.teardown_session() - if self.app.features.fetch_service: + if self._use_fetch_service: self.services.fetch.shutdown(force=True) def configure(self, global_args: dict[str, Any]) -> None: @@ -494,12 +494,14 @@ def _pre_run(self, dispatcher: craft_cli.Dispatcher) -> None: At the time this is run, the command is loaded in the dispatcher, but the project has not yet been loaded. """ + args = dispatcher.parsed_args() + # Some commands might have a project_dir parameter. Those commands and # only those commands should get a project directory, but only when # not managed. if self.is_managed(): self.project_dir = pathlib.Path("/root/project") - elif project_dir := getattr(dispatcher.parsed_args(), "project_dir", None): + elif project_dir := getattr(args, "project_dir", None): self.project_dir = pathlib.Path(project_dir).expanduser().resolve() if self.project_dir.exists() and not self.project_dir.is_dir(): raise errors.ProjectFileMissingError( @@ -508,6 +510,8 @@ def _pre_run(self, dispatcher: craft_cli.Dispatcher) -> None: resolution="Ensure the path entered is correct.", ) + self._use_fetch_service = getattr(args, "use_fetch_service", False) + def get_arg_or_config( self, parsed_args: argparse.Namespace, item: str ) -> Any: # noqa: ANN401 diff --git a/craft_application/commands/lifecycle.py b/craft_application/commands/lifecycle.py index 2d77285c..fb11c6cf 100644 --- a/craft_application/commands/lifecycle.py +++ b/craft_application/commands/lifecycle.py @@ -355,6 +355,12 @@ def _fill_parser(self, parser: argparse.ArgumentParser) -> None: help="Output directory for created packages.", ) + parser.add_argument( + "--use-fetch-service", + action="store_true", + help="Use the Fetch Service to inspect downloaded assets.", + ) + @override def _run( self, diff --git a/tests/unit/commands/test_lifecycle.py b/tests/unit/commands/test_lifecycle.py index 7a643e54..0c4a1e07 100644 --- a/tests/unit/commands/test_lifecycle.py +++ b/tests/unit/commands/test_lifecycle.py @@ -433,6 +433,7 @@ def test_pack_fill_parser( "platform": None, "build_for": None, "output": pathlib.Path(output_arg), + "use_fetch_service": False, **shell_dict, **debug_dict, **build_env_dict, diff --git a/tests/unit/test_application_fetch.py b/tests/unit/test_application_fetch.py index 4b527e6e..1def7870 100644 --- a/tests/unit/test_application_fetch.py +++ b/tests/unit/test_application_fetch.py @@ -20,18 +20,15 @@ import craft_providers import pytest from craft_application import services -from craft_application.util import get_host_architecture from typing_extensions import override class FakeFetchService(services.FetchService): """Fake FetchService that tracks calls""" - calls: list[str] - - def __init__(self, *args, **kwargs): + def __init__(self, *args, fetch_calls: list[str], **kwargs): super().__init__(*args, **kwargs) - self.calls = [] + self.calls = fetch_calls @override def setup(self) -> None: @@ -55,31 +52,49 @@ def shutdown(self, *, force: bool = False) -> None: self.calls.append(f"shutdown({force})") -@pytest.mark.enable_features("fetch_service") @pytest.mark.parametrize("fake_build_plan", [2], indirect=True) -def test_run_managed_fetch_service(app, fake_project, fake_build_plan): +@pytest.mark.parametrize( + ("pack_args", "expected_calls"), + [ + # No --use-fetch-service: no calls to the FetchService + ( + [], + [], + ), + # --use-fetch-service: full expected calls to the FetchService + ( + ["--use-fetch-service"], + [ + # One call to setup + "setup", + # Two pairs of create/teardown sessions, for two builds + "create_session", + "teardown_session", + "create_session", + "teardown_session", + # One call to shut down (with `force`) + "shutdown(True)", + ], + ), + ], +) +def test_run_managed_fetch_service( + app, fake_project, fake_build_plan, monkeypatch, pack_args, expected_calls +): """Test that the application calls the correct FetchService methods.""" mock_provider = mock.MagicMock(spec_set=services.ProviderService) app.services.provider = mock_provider - app.project = fake_project + app.set_project(fake_project) expected_build_infos = 2 assert len(fake_build_plan) == expected_build_infos app._build_plan = fake_build_plan + fetch_calls: list[str] = [] app.services.FetchClass = FakeFetchService + app.services.set_kwargs("fetch", fetch_calls=fetch_calls) - app.run_managed(None, get_host_architecture()) + monkeypatch.setattr("sys.argv", ["testcraft", "pack", *pack_args]) + app.run() - fetch_service = app.services.fetch - assert fetch_service.calls == [ - # One call to setup - "setup", - # Two pairs of create/teardown sessions, for two builds - "create_session", - "teardown_session", - "create_session", - "teardown_session", - # One call to shut down (with `force`) - "shutdown(True)", - ] + assert fetch_calls == expected_calls From c9ccccfe0aef21ad90dc5f69c6ca62351825017e Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 6 Sep 2024 13:01:28 -0300 Subject: [PATCH 59/82] feat: proper logging for the fetch-service (#56) This commit redirects the output of the fetch-service to a file. Since we plan to have the fetch-service outlive the application that spawned it, we need to use bash to redirect the fetch-service's output to a file in a way that persists after the application ends. Fixes #51 --- craft_application/fetch.py | 48 ++++++++++++++++++------ tests/integration/services/test_fetch.py | 42 ++++++++++++++++----- tests/unit/test_fetch.py | 25 ++++++++---- 3 files changed, 87 insertions(+), 28 deletions(-) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index 7ec262e1..b75924d6 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -16,9 +16,12 @@ """Utilities to interact with the fetch-service.""" import contextlib import io +import os import pathlib import shlex +import signal import subprocess +import time from dataclasses import dataclass from typing import Any, cast @@ -181,10 +184,21 @@ def start_service() -> subprocess.Popen[str] | None: # Accept permissive sessions cmd.append("--permissive-mode") - emit.debug(f"Launching fetch-service with '{shlex.join(cmd)}'") + log_filepath = _get_log_filepath() + log_filepath.parent.mkdir(parents=True, exist_ok=True) + + str_cmd = f"{shlex.join(cmd)} > {log_filepath.absolute()}" + emit.debug(f"Launching fetch-service with '{str_cmd}'") fetch_process = subprocess.Popen( - cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ["bash", "-c", str_cmd], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + # Start a new session because when killing the service we need to kill + # both 'bash' and the 'fetch' it spawns. + start_new_session=True, ) # Wait a bit for the service to come online @@ -193,17 +207,18 @@ def start_service() -> subprocess.Popen[str] | None: if fetch_process.poll() is not None: # fetch-service already exited, something is wrong - stdout = "" - if fetch_process.stdout is not None: - stdout = fetch_process.stdout.read() + log = log_filepath.read_text() + lines = log.splitlines() + error_lines = [line for line in lines if "ERROR:" in line] + error_text = "\n".join(error_lines) - if "bind: address already in use" in stdout: + if "bind: address already in use" in error_text: proxy, control = _DEFAULT_CONFIG.proxy, _DEFAULT_CONFIG.control message = f"fetch-service ports {proxy} and {control} are already in use." details = None else: message = "Error spawning the fetch-service." - details = stdout + details = error_text raise errors.FetchServiceError(message, details=details) status = retry( @@ -225,11 +240,16 @@ def stop_service(fetch_process: subprocess.Popen[str]) -> None: This function first calls terminate(), and then kill() after a short time. """ - fetch_process.terminate() try: - fetch_process.wait(timeout=1.0) - except subprocess.TimeoutExpired: - fetch_process.kill() + os.killpg(os.getpgid(fetch_process.pid), signal.SIGTERM) + except ProcessLookupError: + return + + # Give the shell and fetch-service a chance to terminate + time.sleep(0.2) + + with contextlib.suppress(ProcessLookupError): + os.killpg(os.getpgid(fetch_process.pid), signal.SIGKILL) def create_session() -> SessionData: @@ -490,3 +510,9 @@ def _get_certificate_dir() -> pathlib.Path: def _check_installed() -> bool: """Check whether the fetch-service is installed.""" return pathlib.Path(_FETCH_BINARY).is_file() + + +def _get_log_filepath() -> pathlib.Path: + base_dir = _get_service_base_dir() + + return base_dir / "craft/fetch-log.txt" diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index d2a175bd..0f2bb32d 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -47,25 +47,19 @@ def _set_test_certificate_dir(): with mock.patch.object(fetch, "_get_certificate_dir", return_value=cert_dir): fetch._obtain_certificate() - yield - - shutil.rmtree(cert_dir) - @pytest.fixture(autouse=True) def _set_test_base_dirs(mocker): original = fetch._get_service_base_dir() test_dir = original / "test" - test_dir.mkdir(exist_ok=True) + if test_dir.exists(): + shutil.rmtree(test_dir) + test_dir.mkdir() mocker.patch.object(fetch, "_get_service_base_dir", return_value=test_dir) cert_dir = _get_fake_certificate_dir() mocker.patch.object(fetch, "_get_certificate_dir", return_value=cert_dir) - yield - - shutil.rmtree(test_dir) - @pytest.fixture def app_service(app_metadata, fake_services): @@ -149,6 +143,36 @@ def test_create_teardown_session(app_service, mocker, tmp_path, monkeypatch): assert expected_report.is_file() +def test_service_logging(app_service, mocker, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + mocker.patch.object(fetch, "_get_gateway", return_value="127.0.0.1") + + logfile = fetch._get_log_filepath() + assert not logfile.is_file() + + app_service.setup() + + mock_instance = mock.MagicMock(spec_set=craft_providers.Executor) + + # Create and teardown two sessions + app_service.create_session(mock_instance) + app_service.teardown_session() + app_service.create_session(mock_instance) + app_service.teardown_session() + + # Check the logfile for the creation/deletion of the two sessions + expected = 2 + assert logfile.is_file() + lines = logfile.read_text().splitlines() + create = discard = 0 + for line in lines: + if "creating session" in line: + create += 1 + if "discarding session" in line: + discard += 1 + assert create == discard == expected + + # Bash script to setup the build instance before the actual testing. setup_environment = ( textwrap.dedent( diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index 30765de8..fbf5d6c4 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -15,6 +15,7 @@ # along with this program. If not, see . """Tests for fetch-service-related functions.""" import re +import shlex import subprocess from pathlib import Path from unittest import mock @@ -123,19 +124,27 @@ def test_start_service(mocker, tmp_path): popen_call = mock_popen.mock_calls[0] assert popen_call == call( [ - fetch._FETCH_BINARY, - f"--control-port={CONTROL}", - f"--proxy-port={PROXY}", - f"--config={tmp_path/'config'}", - f"--spool={tmp_path/'spool'}", - f"--cert={fake_cert}", - f"--key={fake_key}", - "--permissive-mode", + "bash", + "-c", + shlex.join( + [ + fetch._FETCH_BINARY, + f"--control-port={CONTROL}", + f"--proxy-port={PROXY}", + f"--config={tmp_path/'config'}", + f"--spool={tmp_path/'spool'}", + f"--cert={fake_cert}", + f"--key={fake_key}", + "--permissive-mode", + ] + ) + + f" > {fetch._get_log_filepath()}", ], env={"FETCH_SERVICE_AUTH": AUTH, "FETCH_APT_RELEASE_PUBLIC_KEY": "DEADBEEF"}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + start_new_session=True, ) From c8e923a45d0277f5042d588743e5d010bb8b0509 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Wed, 11 Sep 2024 14:11:10 -0300 Subject: [PATCH 60/82] chore: disable port conflict test (#64) Test is failing due to https://github.com/canonical/fetch-service/issues/208; once that's fixed we can revert this commit. Note that only one of the ports is failing; the proxy port is still giving the expected error output. --- tests/integration/services/test_fetch.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index 0f2bb32d..babeee06 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -87,7 +87,17 @@ def test_start_service_already_up(app_service, request): @pytest.mark.parametrize( - "port", [fetch._DEFAULT_CONFIG.control, fetch._DEFAULT_CONFIG.proxy] + "port", + [ + pytest.param( + fetch._DEFAULT_CONFIG.control, + marks=pytest.mark.xfail( + reason="Needs https://github.com/canonical/fetch-service/issues/208 fixed", + strict=True, + ), + ), + fetch._DEFAULT_CONFIG.proxy, + ], ) def test_start_service_port_taken(app_service, request, port): # "Occupy" one of the necessary ports manually. From 1ae919cb1362f5257a21ddbbe30fbca94c7145e6 Mon Sep 17 00:00:00 2001 From: Callahan Date: Thu, 12 Sep 2024 14:59:09 -0500 Subject: [PATCH 61/82] docs(changelog): add release notes for 4.1.3 (#459) Signed-off-by: Callahan Kovacs --- docs/reference/changelog.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 2b5be99d..dcf0490f 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,17 @@ Changelog ********* +4.1.3 (2024-Sep-12) +------------------- + +Models +====== + +- Fix a regression where numeric part properties could not be parsed. + +For a complete list of commits, check out the `4.1.3`_ release on GitHub. + + 4.1.2 (2024-Sep-05) ------------------- @@ -225,3 +236,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _4.1.0: https://github.com/canonical/craft-application/releases/tag/4.1.0 .. _4.1.1: https://github.com/canonical/craft-application/releases/tag/4.1.1 .. _4.1.2: https://github.com/canonical/craft-application/releases/tag/4.1.2 +.. _4.1.3: https://github.com/canonical/craft-application/releases/tag/4.1.3 From f175c968cf4222f3993f077821abad1cded78ede Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 6 Sep 2024 16:41:12 -0400 Subject: [PATCH 62/82] fix(models): coerce numbers to strings by default (#450) --- craft_application/models/base.py | 1 + tests/unit/models/test_base.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/craft_application/models/base.py b/craft_application/models/base.py index 08e30438..cd261f77 100644 --- a/craft_application/models/base.py +++ b/craft_application/models/base.py @@ -38,6 +38,7 @@ class CraftBaseModel(pydantic.BaseModel): extra="forbid", populate_by_name=True, alias_generator=alias_generator, + coerce_numbers_to_str=True, ) def marshal(self) -> dict[str, str | list[str] | dict[str, Any]]: diff --git a/tests/unit/models/test_base.py b/tests/unit/models/test_base.py index 2d9390a8..da7067c6 100644 --- a/tests/unit/models/test_base.py +++ b/tests/unit/models/test_base.py @@ -19,6 +19,7 @@ import pydantic import pytest from craft_application import errors, models +from hypothesis import given, strategies from overrides import override @@ -58,3 +59,22 @@ def test_model_reference_slug_errors(): ) assert str(err.value) == expected assert err.value.doc_slug == "/mymodel.html" + + +class CoerceModel(models.CraftBaseModel): + + stringy: str + + +@given( + strategies.one_of( + strategies.integers(), + strategies.floats(), + strategies.decimals(), + strategies.text(), + ) +) +def test_model_coerces_to_strings(value): + result = CoerceModel.model_validate({"stringy": value}) + + assert result.stringy == str(value) From 4dcba1cbe57feb533110592b8df0355d0eb0eefa Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 13 Sep 2024 08:28:55 -0400 Subject: [PATCH 63/82] docs(changelog): add 4.2.1 to changelog --- docs/reference/changelog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index a7267d9e..bdcb1519 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,16 @@ Changelog ********* +4.2.1 (2024-Sep-13) +------------------- + +Models +====== + +- Fix a regression where numeric part properties could not be parsed. + +For a complete list of commits, check out the `4.2.1`_ release on GitHub. + 4.2.0 (2024-Sep-12) ------------------- @@ -244,3 +254,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _4.1.1: https://github.com/canonical/craft-application/releases/tag/4.1.1 .. _4.1.2: https://github.com/canonical/craft-application/releases/tag/4.1.2 .. _4.2.0: https://github.com/canonical/craft-application/releases/tag/4.2.0 +.. _4.2.1: https://github.com/canonical/craft-application/releases/tag/4.2.1 From ac73285202b4ae003f1791955085606a602476b6 Mon Sep 17 00:00:00 2001 From: Sergio Schvezov Date: Fri, 13 Sep 2024 08:46:03 -0300 Subject: [PATCH 64/82] chore: factor run logic outside of try block This allows applications to override the inner run logic, with the end goal of capturing application specific exceptions and raise appropriate ones for Craft Application to handle. Without the logic split, overriding run to capture exceptions is virtually impossible as the run method holds a generic Exception handler. Signed-off-by: Sergio Schvezov --- craft_application/application.py | 96 +++++++++++++++++--------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index 7fd109aa..e92a38c3 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -405,8 +405,9 @@ def configure(self, global_args: dict[str, Any]) -> None: """Configure the application using any global arguments.""" def _get_dispatcher(self) -> craft_cli.Dispatcher: - """Configure this application. Should be called by the run method. + """Configure this application. + Should be called by the _run_inner method. Side-effect: This method may exit the process. :returns: A ready-to-run Dispatcher object @@ -523,59 +524,62 @@ def get_arg_or_config( """ return getattr(parsed_args, item, self.services.config.get(item)) - def run( # noqa: PLR0912,PLR0915 (too many branches, too many statements) - self, - ) -> int: - """Bootstrap and run the application.""" - self._setup_logging() - self._initialize_craft_parts() + def _run_inner(self) -> int: + """Actual run implementation.""" dispatcher = self._get_dispatcher() - craft_cli.emit.debug("Preparing application...") - - return_code = 1 # General error - try: - command = cast( - commands.AppCommand, - dispatcher.load_command(self.app_config), + command = cast( + commands.AppCommand, + dispatcher.load_command(self.app_config), + ) + parsed_args = dispatcher.parsed_args() + platform = self.get_arg_or_config(parsed_args, "platform") + build_for = self.get_arg_or_config(parsed_args, "build_for") + + # Some commands (e.g. remote build) can allow multiple platforms + # or build-fors, comma-separated. In these cases, we create the + # project using the first defined platform. + if platform and "," in platform: + platform = platform.split(",", maxsplit=1)[0] + if build_for and "," in build_for: + build_for = build_for.split(",", maxsplit=1)[0] + + provider_name = command.provider_name(dispatcher.parsed_args()) + + craft_cli.emit.debug(f"Build plan: platform={platform}, build_for={build_for}") + self._pre_run(dispatcher) + + managed_mode = command.run_managed(dispatcher.parsed_args()) + if managed_mode or command.needs_project(dispatcher.parsed_args()): + self.services.project = self.get_project( + platform=platform, build_for=build_for ) - parsed_args = dispatcher.parsed_args() - platform = self.get_arg_or_config(parsed_args, "platform") - build_for = self.get_arg_or_config(parsed_args, "build_for") - # Some commands (e.g. remote build) can allow multiple platforms - # or build-fors, comma-separated. In these cases, we create the - # project using the first defined platform. - if platform and "," in platform: - platform = platform.split(",", maxsplit=1)[0] - if build_for and "," in build_for: - build_for = build_for.split(",", maxsplit=1)[0] + self._configure_services(provider_name) - provider_name = command.provider_name(dispatcher.parsed_args()) + return_code = 1 # General error + if not managed_mode: + # command runs in the outer instance + craft_cli.emit.debug(f"Running {self.app.name} {command.name} on host") + return_code = dispatcher.run() or os.EX_OK + elif not self.is_managed(): + # command runs in inner instance, but this is the outer instance + self.run_managed(platform, build_for) + return_code = os.EX_OK + else: + # command runs in inner instance + return_code = dispatcher.run() or 0 - craft_cli.emit.debug( - f"Build plan: platform={platform}, build_for={build_for}" - ) - self._pre_run(dispatcher) + return return_code - managed_mode = command.run_managed(dispatcher.parsed_args()) - if managed_mode or command.needs_project(dispatcher.parsed_args()): - self.services.project = self.get_project( - platform=platform, build_for=build_for - ) + def run(self) -> int: + """Bootstrap and run the application.""" + self._setup_logging() + self._initialize_craft_parts() - self._configure_services(provider_name) + craft_cli.emit.debug("Preparing application...") - if not managed_mode: - # command runs in the outer instance - craft_cli.emit.debug(f"Running {self.app.name} {command.name} on host") - return_code = dispatcher.run() or os.EX_OK - elif not self.is_managed(): - # command runs in inner instance, but this is the outer instance - self.run_managed(platform, build_for) - return_code = os.EX_OK - else: - # command runs in inner instance - return_code = dispatcher.run() or 0 + try: + return_code = self._run_inner() except craft_cli.ArgumentParsingError as err: print(err, file=sys.stderr) # to stderr, as argparse normally does craft_cli.emit.ended_ok() From a5349c511fdde3b702861e9e06ed94306720fab6 Mon Sep 17 00:00:00 2001 From: Sergio Schvezov Date: Fri, 13 Sep 2024 08:46:03 -0300 Subject: [PATCH 65/82] chore: factor run logic outside of try block This allows applications to override the inner run logic, with the end goal of capturing application specific exceptions and raise appropriate ones for Craft Application to handle. Without the logic split, overriding run to capture exceptions is virtually impossible as the run method holds a generic Exception handler. Signed-off-by: Sergio Schvezov --- craft_application/application.py | 96 +++++++++++++++++--------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index 3a65cfb6..076d252c 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -392,8 +392,9 @@ def configure(self, global_args: dict[str, Any]) -> None: """Configure the application using any global arguments.""" def _get_dispatcher(self) -> craft_cli.Dispatcher: - """Configure this application. Should be called by the run method. + """Configure this application. + Should be called by the _run_inner method. Side-effect: This method may exit the process. :returns: A ready-to-run Dispatcher object @@ -506,59 +507,62 @@ def get_arg_or_config( """ return getattr(parsed_args, item, self.services.config.get(item)) - def run( # noqa: PLR0912,PLR0915 (too many branches, too many statements) - self, - ) -> int: - """Bootstrap and run the application.""" - self._setup_logging() - self._initialize_craft_parts() + def _run_inner(self) -> int: + """Actual run implementation.""" dispatcher = self._get_dispatcher() - craft_cli.emit.debug("Preparing application...") - - return_code = 1 # General error - try: - command = cast( - commands.AppCommand, - dispatcher.load_command(self.app_config), + command = cast( + commands.AppCommand, + dispatcher.load_command(self.app_config), + ) + parsed_args = dispatcher.parsed_args() + platform = self.get_arg_or_config(parsed_args, "platform") + build_for = self.get_arg_or_config(parsed_args, "build_for") + + # Some commands (e.g. remote build) can allow multiple platforms + # or build-fors, comma-separated. In these cases, we create the + # project using the first defined platform. + if platform and "," in platform: + platform = platform.split(",", maxsplit=1)[0] + if build_for and "," in build_for: + build_for = build_for.split(",", maxsplit=1)[0] + + provider_name = command.provider_name(dispatcher.parsed_args()) + + craft_cli.emit.debug(f"Build plan: platform={platform}, build_for={build_for}") + self._pre_run(dispatcher) + + managed_mode = command.run_managed(dispatcher.parsed_args()) + if managed_mode or command.needs_project(dispatcher.parsed_args()): + self.services.project = self.get_project( + platform=platform, build_for=build_for ) - parsed_args = dispatcher.parsed_args() - platform = self.get_arg_or_config(parsed_args, "platform") - build_for = self.get_arg_or_config(parsed_args, "build_for") - # Some commands (e.g. remote build) can allow multiple platforms - # or build-fors, comma-separated. In these cases, we create the - # project using the first defined platform. - if platform and "," in platform: - platform = platform.split(",", maxsplit=1)[0] - if build_for and "," in build_for: - build_for = build_for.split(",", maxsplit=1)[0] + self._configure_services(provider_name) - provider_name = command.provider_name(dispatcher.parsed_args()) + return_code = 1 # General error + if not managed_mode: + # command runs in the outer instance + craft_cli.emit.debug(f"Running {self.app.name} {command.name} on host") + return_code = dispatcher.run() or os.EX_OK + elif not self.is_managed(): + # command runs in inner instance, but this is the outer instance + self.run_managed(platform, build_for) + return_code = os.EX_OK + else: + # command runs in inner instance + return_code = dispatcher.run() or 0 - craft_cli.emit.debug( - f"Build plan: platform={platform}, build_for={build_for}" - ) - self._pre_run(dispatcher) + return return_code - managed_mode = command.run_managed(dispatcher.parsed_args()) - if managed_mode or command.needs_project(dispatcher.parsed_args()): - self.services.project = self.get_project( - platform=platform, build_for=build_for - ) + def run(self) -> int: + """Bootstrap and run the application.""" + self._setup_logging() + self._initialize_craft_parts() - self._configure_services(provider_name) + craft_cli.emit.debug("Preparing application...") - if not managed_mode: - # command runs in the outer instance - craft_cli.emit.debug(f"Running {self.app.name} {command.name} on host") - return_code = dispatcher.run() or os.EX_OK - elif not self.is_managed(): - # command runs in inner instance, but this is the outer instance - self.run_managed(platform, build_for) - return_code = os.EX_OK - else: - # command runs in inner instance - return_code = dispatcher.run() or 0 + try: + return_code = self._run_inner() except craft_cli.ArgumentParsingError as err: print(err, file=sys.stderr) # to stderr, as argparse normally does craft_cli.emit.ended_ok() From ca42796758464584d8bba394c55bee6fe737a97c Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 13 Sep 2024 18:05:03 -0400 Subject: [PATCH 66/82] docs: add 4.2.2 to changelog --- docs/reference/changelog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index bdcb1519..2619b5e3 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,16 @@ Changelog ********* +4.2.2 (2024-Sep-13) +------------------- + +Application +=========== + +- Add a ``_run_inner`` method to override or wrap the core run logic. + +For a complete list of commits, check out the `4.2.2`_ release on GitHub. + 4.2.1 (2024-Sep-13) ------------------- @@ -255,3 +265,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _4.1.2: https://github.com/canonical/craft-application/releases/tag/4.1.2 .. _4.2.0: https://github.com/canonical/craft-application/releases/tag/4.2.0 .. _4.2.1: https://github.com/canonical/craft-application/releases/tag/4.2.1 +.. _4.2.2: https://github.com/canonical/craft-application/releases/tag/4.2.2 From d872bfd963d67232c085da384238df1a81bbd40e Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Fri, 13 Sep 2024 19:45:06 -0400 Subject: [PATCH 67/82] ci: use the default version of lxd for integration tests (#461) --- .github/workflows/tests.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8801ac2a..ad9d5328 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -60,10 +60,6 @@ jobs: 3.10 3.12 cache: 'pip' - - name: Setup LXD - uses: canonical/setup-lxd@v0.1.1 - with: - channel: latest/stable - name: Configure environment run: | echo "::group::apt-get" @@ -111,8 +107,6 @@ jobs: cache: 'pip' - name: Setup LXD uses: canonical/setup-lxd@v0.1.1 - with: - channel: latest/stable - name: Configure environment run: | echo "::group::Begin snap install" From ba3be78c2427b06ff781dfa8819d585df6b4f0b2 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 13 Sep 2024 14:51:43 -0300 Subject: [PATCH 68/82] feat(provider): add a way to clean existing instances This new parameter to ProviderService.instance() defaults to False and, if True, will cause the service to clean a pre-existing instance and create a new one. --- craft_application/services/provider.py | 21 ++++++++++-- tests/unit/services/test_provider.py | 47 ++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/craft_application/services/provider.py b/craft_application/services/provider.py index a42c508e..c61972fc 100644 --- a/craft_application/services/provider.py +++ b/craft_application/services/provider.py @@ -112,6 +112,7 @@ def instance( *, work_dir: pathlib.Path, allow_unstable: bool = True, + clean_existing: bool = False, **kwargs: bool | str | None, ) -> Generator[craft_providers.Executor, None, None]: """Context manager for getting a provider instance. @@ -119,6 +120,8 @@ def instance( :param build_info: Build information for the instance. :param work_dir: Local path to mount inside the provider instance. :param allow_unstable: Whether to allow the use of unstable images. + :param clean_existing: Whether pre-existing instances should be wiped + and re-created. :returns: a context manager of the provider instance. """ instance_name = self._get_instance_name(work_dir, build_info) @@ -129,6 +132,9 @@ def instance( provider.ensure_provider_is_available() + if clean_existing: + self._clean_instance(provider, work_dir, build_info) + emit.progress(f"Launching managed {base_name[0]} {base_name[1]} instance...") with provider.launched_environment( project_name=self._project.name, @@ -263,9 +269,7 @@ def clean_instances(self) -> None: emit.progress(f"Cleaning build {target}") for info in build_plan: - instance_name = self._get_instance_name(self._work_dir, info) - emit.debug(f"Cleaning instance {instance_name}") - provider.clean_project_environments(instance_name=instance_name) + self._clean_instance(provider, self._work_dir, info) def _get_instance_name( self, work_dir: pathlib.Path, build_info: models.BuildInfo @@ -328,3 +332,14 @@ def _setup_instance_bashrc(self, instance: craft_providers.Executor) -> None: content=io.BytesIO(bashrc), file_mode="644", ) + + def _clean_instance( + self, + provider: craft_providers.Provider, + work_dir: pathlib.Path, + info: models.BuildInfo, + ) -> None: + """Clean an instance, if it exists.""" + instance_name = self._get_instance_name(work_dir, info) + emit.debug(f"Cleaning instance {instance_name}") + provider.clean_project_environments(instance_name=instance_name) diff --git a/tests/unit/services/test_provider.py b/tests/unit/services/test_provider.py index 8c3abd8f..c36e1c47 100644 --- a/tests/unit/services/test_provider.py +++ b/tests/unit/services/test_provider.py @@ -30,6 +30,18 @@ from craft_providers.actions.snap_installer import Snap +@pytest.fixture +def mock_provider(monkeypatch, provider_service): + mocked_provider = mock.MagicMock(spec=craft_providers.Provider) + monkeypatch.setattr( + provider_service, + "get_provider", + lambda name: mocked_provider, # noqa: ARG005 (unused argument) + ) + + return mocked_provider + + @pytest.mark.parametrize( ("given_environment", "expected_environment"), [ @@ -384,7 +396,6 @@ def test_get_base_packages(provider_service): ], ) def test_instance( - monkeypatch, check, emitter, tmp_path, @@ -393,13 +404,8 @@ def test_instance( provider_service, base_name, allow_unstable, + mock_provider, ): - mock_provider = mock.MagicMock(spec=craft_providers.Provider) - monkeypatch.setattr( - provider_service, - "get_provider", - lambda name: mock_provider, # noqa: ARG005 (unused argument) - ) arch = util.get_host_architecture() build_info = models.BuildInfo("foo", arch, arch, base_name) @@ -429,6 +435,33 @@ def test_instance( emitter.assert_progress("Launching managed .+ instance...", regex=True) +@pytest.mark.parametrize("clean_existing", [True, False]) +def test_instance_clean_existing( + tmp_path, + provider_service, + mock_provider, + clean_existing, +): + arch = util.get_host_architecture() + base_name = bases.BaseName("ubuntu", "24.04") + build_info = models.BuildInfo("foo", arch, arch, base_name) + + with provider_service.instance( + build_info, work_dir=tmp_path, clean_existing=clean_existing + ) as _instance: + pass + + clean_called = mock_provider.clean_project_environments.called + assert clean_called == clean_existing + + if clean_existing: + work_dir_inode = tmp_path.stat().st_ino + expected_name = f"testcraft-full-project-on-{arch}-for-{arch}-{work_dir_inode}" + mock_provider.clean_project_environments.assert_called_once_with( + instance_name=expected_name + ) + + def test_load_bashrc(emitter): """Test that we are able to load the bashrc file from the craft-application package.""" bashrc = pkgutil.get_data("craft_application", "misc/instance_bashrc") From e931a597773dc98092e6fb15c11a1715993f930e Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 13 Sep 2024 14:53:02 -0300 Subject: [PATCH 69/82] feat: wipe existing instances when using the fetch-service This is necessary because the fetch-service needs visibility on *all* assets downloaded during a build, so pre-existing instances that might already have existing items cannot be re-used. Fixes #40 --- craft_application/application.py | 4 +++- tests/unit/test_application.py | 25 +++++++++++++++++++------ tests/unit/test_application_fetch.py | 27 +++++++++++++++++++++++++-- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/craft_application/application.py b/craft_application/application.py index e92a38c3..f4472230 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -371,7 +371,9 @@ def run_managed(self, platform: str | None, build_for: str | None) -> None: instance_path = pathlib.PosixPath("/root/project") with self.services.provider.instance( - build_info, work_dir=self._work_dir + build_info, + work_dir=self._work_dir, + clean_existing=self._use_fetch_service, ) as instance: if self._use_fetch_service: session_env = self.services.fetch.create_session(instance) diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 869a693b..671d264f 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -485,6 +485,7 @@ def test_run_managed_success(mocker, app, fake_project, fake_build_plan): mock.call( fake_build_plan[0], work_dir=mock.ANY, + clean_existing=False, ) in mock_provider.instance.mock_calls ) @@ -543,8 +544,12 @@ def test_run_managed_multiple(app, fake_project): app.run_managed(None, None) - assert mock.call(info2, work_dir=mock.ANY) in mock_provider.instance.mock_calls - assert mock.call(info1, work_dir=mock.ANY) in mock_provider.instance.mock_calls + extra_args = { + "work_dir": mock.ANY, + "clean_existing": False, + } + assert mock.call(info2, **extra_args) in mock_provider.instance.mock_calls + assert mock.call(info1, **extra_args) in mock_provider.instance.mock_calls def test_run_managed_specified_arch(app, fake_project): @@ -559,8 +564,12 @@ def test_run_managed_specified_arch(app, fake_project): app.run_managed(None, "arch2") - assert mock.call(info2, work_dir=mock.ANY) in mock_provider.instance.mock_calls - assert mock.call(info1, work_dir=mock.ANY) not in mock_provider.instance.mock_calls + extra_args = { + "work_dir": mock.ANY, + "clean_existing": False, + } + assert mock.call(info2, **extra_args) in mock_provider.instance.mock_calls + assert mock.call(info1, **extra_args) not in mock_provider.instance.mock_calls def test_run_managed_specified_platform(app, fake_project): @@ -575,8 +584,12 @@ def test_run_managed_specified_platform(app, fake_project): app.run_managed("a2", None) - assert mock.call(info2, work_dir=mock.ANY) in mock_provider.instance.mock_calls - assert mock.call(info1, work_dir=mock.ANY) not in mock_provider.instance.mock_calls + extra_args = { + "work_dir": mock.ANY, + "clean_existing": False, + } + assert mock.call(info2, **extra_args) in mock_provider.instance.mock_calls + assert mock.call(info1, **extra_args) not in mock_provider.instance.mock_calls def test_run_managed_empty_plan(app, fake_project): diff --git a/tests/unit/test_application_fetch.py b/tests/unit/test_application_fetch.py index 1def7870..98e4e032 100644 --- a/tests/unit/test_application_fetch.py +++ b/tests/unit/test_application_fetch.py @@ -54,12 +54,13 @@ def shutdown(self, *, force: bool = False) -> None: @pytest.mark.parametrize("fake_build_plan", [2], indirect=True) @pytest.mark.parametrize( - ("pack_args", "expected_calls"), + ("pack_args", "expected_calls", "expected_clean_existing"), [ # No --use-fetch-service: no calls to the FetchService ( [], [], + False, ), # --use-fetch-service: full expected calls to the FetchService ( @@ -75,11 +76,18 @@ def shutdown(self, *, force: bool = False) -> None: # One call to shut down (with `force`) "shutdown(True)", ], + True, ), ], ) def test_run_managed_fetch_service( - app, fake_project, fake_build_plan, monkeypatch, pack_args, expected_calls + app, + fake_project, + fake_build_plan, + monkeypatch, + pack_args, + expected_calls, + expected_clean_existing, ): """Test that the application calls the correct FetchService methods.""" mock_provider = mock.MagicMock(spec_set=services.ProviderService) @@ -98,3 +106,18 @@ def test_run_managed_fetch_service( app.run() assert fetch_calls == expected_calls + + # Check that the provider service was correctly instructed to clean, or not + # clean, the existing instance. + + # Filter out the various calls to entering and exiting the instance() + # context manager. + instance_calls = [ + call + for call in mock_provider.instance.mock_calls + if "work_dir" in call.kwargs and "clean_existing" in call.kwargs + ] + + assert len(instance_calls) == len(fake_build_plan) + for call in instance_calls: + assert call.kwargs["clean_existing"] == expected_clean_existing From 77a3582b81bfdc703d415a8dcb1a747ab340492c Mon Sep 17 00:00:00 2001 From: Callahan Date: Mon, 16 Sep 2024 10:45:29 -0500 Subject: [PATCH 70/82] fix: inject aliased snaps into build environment (#467) Signed-off-by: Callahan Kovacs --- craft_application/services/provider.py | 16 ++++--- tests/unit/services/test_provider.py | 64 +++++++++++++++++++------- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/craft_application/services/provider.py b/craft_application/services/provider.py index a42c508e..c746500c 100644 --- a/craft_application/services/provider.py +++ b/craft_application/services/provider.py @@ -98,12 +98,16 @@ def setup(self) -> None: self.environment[f"{scheme.upper()}_PROXY"] = value if self._install_snap: - channel = ( - None - if util.is_running_from_snap(self._app.name) - else os.getenv("CRAFT_SNAP_CHANNEL", "latest/stable") - ) - self.snaps.append(Snap(name=self._app.name, channel=channel, classic=True)) + if util.is_running_from_snap(self._app.name): + # use the aliased name of the snap when injecting + name = os.getenv("SNAP_INSTANCE_NAME", self._app.name) + channel = None + else: + # use the snap name when installing from the store + name = self._app.name + channel = os.getenv("CRAFT_SNAP_CHANNEL", "latest/stable") + + self.snaps.append(Snap(name=name, channel=channel, classic=True)) @contextlib.contextmanager def instance( diff --git a/tests/unit/services/test_provider.py b/tests/unit/services/test_provider.py index 8c3abd8f..c63b7c5d 100644 --- a/tests/unit/services/test_provider.py +++ b/tests/unit/services/test_provider.py @@ -84,41 +84,70 @@ def test_setup_proxy_environment( @pytest.mark.parametrize( - ("install_snap", "environment", "snaps"), + ("environment", "snaps"), [ - (True, {}, [Snap(name="testcraft", channel="latest/stable", classic=True)]), - ( - True, + pytest.param( + {}, + [Snap(name="testcraft", channel="latest/stable", classic=True)], + id="install-from-store-default-channel", + ), + pytest.param( {"CRAFT_SNAP_CHANNEL": "something"}, [Snap(name="testcraft", channel="something", classic=True)], + id="install-from-store-with-channel", ), - ( - True, - {"SNAP_NAME": "testcraft", "SNAP": "/snap/testcraft/x1"}, - [Snap(name="testcraft", channel=None, classic=True)], + pytest.param( + { + "SNAP_NAME": "testcraft", + "SNAP_INSTANCE_NAME": "testcraft_1", + "SNAP": "/snap/testcraft/x1", + }, + [Snap(name="testcraft_1", channel=None, classic=True)], + id="inject-from-host", ), - ( - True, + pytest.param( { "SNAP_NAME": "testcraft", + "SNAP_INSTANCE_NAME": "testcraft_1", "SNAP": "/snap/testcraft/x1", "CRAFT_SNAP_CHANNEL": "something", }, + [Snap(name="testcraft_1", channel=None, classic=True)], + id="inject-from-host-ignore-channel", + ), + pytest.param( + # SNAP_INSTANCE_NAME may not exist if snapd < 2.43 or feature is disabled + { + "SNAP_NAME": "testcraft", + "SNAP": "/snap/testcraft/x1", + }, [Snap(name="testcraft", channel=None, classic=True)], + id="missing-snap-instance-name", ), - (False, {}, []), - (False, {"CRAFT_SNAP_CHANNEL": "something"}, []), - ( - False, + pytest.param( + # SNAP_INSTANCE_NAME may not exist if snapd < 2.43 or feature is disabled { "SNAP_NAME": "testcraft", "SNAP": "/snap/testcraft/x1", + # CRAFT_SNAP_CHANNEL should be ignored "CRAFT_SNAP_CHANNEL": "something", }, - [], + [Snap(name="testcraft", channel=None, classic=True)], + id="missing-snap-instance-name-ignore-snap-channel", + ), + pytest.param( + # this can happen when running testcraft from a venv in a snapped terminal + { + "SNAP_NAME": "kitty", + "SNAP_INSTANCE_NAME": "kitty", + "SNAP": "/snap/kitty/x1", + }, + [Snap(name="testcraft", channel="latest/stable", classic=True)], + id="running-inside-another-snap", ), ], ) +@pytest.mark.parametrize("install_snap", [True, False]) def test_install_snap( monkeypatch, app_metadata, @@ -143,7 +172,10 @@ def test_install_snap( ) service.setup() - assert service.snaps == snaps + if install_snap: + assert service.snaps == snaps + else: + assert service.snaps == [] @pytest.mark.parametrize( From e546faeb3ac9117ce05a53a3999238c58416d60d Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Fri, 6 Sep 2024 16:13:57 -0300 Subject: [PATCH 71/82] feat: set the fetch-service to idle shutdown Set it to shutdown after 5 minutes of no live sessions. --- craft_application/fetch.py | 3 +++ tests/unit/test_fetch.py | 1 + 2 files changed, 4 insertions(+) diff --git a/craft_application/fetch.py b/craft_application/fetch.py index b75924d6..97bde7ce 100644 --- a/craft_application/fetch.py +++ b/craft_application/fetch.py @@ -184,6 +184,9 @@ def start_service() -> subprocess.Popen[str] | None: # Accept permissive sessions cmd.append("--permissive-mode") + # Shutdown after 5 minutes with no live sessions + cmd.append("--idle-shutdown=300") + log_filepath = _get_log_filepath() log_filepath.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/unit/test_fetch.py b/tests/unit/test_fetch.py index fbf5d6c4..edb7132c 100644 --- a/tests/unit/test_fetch.py +++ b/tests/unit/test_fetch.py @@ -136,6 +136,7 @@ def test_start_service(mocker, tmp_path): f"--cert={fake_cert}", f"--key={fake_key}", "--permissive-mode", + "--idle-shutdown=300", ] ) + f" > {fetch._get_log_filepath()}", From f6627fb244846d102e1c45b7675f67d48c944567 Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 18 Sep 2024 11:55:59 -0400 Subject: [PATCH 72/82] fix(app): get_arg_or_config gets environment correctly Previously, if a value was in the passed Namespace, it would return that. The correct behaviour however is to check the environment for the value and return that instead. --- craft_application/application.py | 5 ++++- tests/unit/test_application.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/craft_application/application.py b/craft_application/application.py index 076d252c..90900a01 100644 --- a/craft_application/application.py +++ b/craft_application/application.py @@ -505,7 +505,10 @@ def get_arg_or_config( :param item: the name of the namespace or config item. :returns: the requested value. """ - return getattr(parsed_args, item, self.services.config.get(item)) + arg_value = getattr(parsed_args, item, None) + if arg_value is not None: + return arg_value + return self.services.config.get(item) def _run_inner(self) -> int: """Actual run implementation.""" diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 6059636c..49cd843a 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -614,6 +614,38 @@ def test_run_managed_empty_plan(app, fake_project): app.run_managed(None, None) +@pytest.mark.parametrize( + ("parsed_args", "environ", "item", "expected"), + [ + (argparse.Namespace(), {}, "build_for", None), + (argparse.Namespace(build_for=None), {}, "build_for", None), + ( + argparse.Namespace(build_for=None), + {"CRAFT_BUILD_FOR": "arm64"}, + "build_for", + "arm64", + ), + ( + argparse.Namespace(build_for=None), + {"TESTCRAFT_BUILD_FOR": "arm64"}, + "build_for", + "arm64", + ), + ( + argparse.Namespace(build_for="riscv64"), + {"TESTCRAFT_BUILD_FOR": "arm64"}, + "build_for", + "riscv64", + ), + ], +) +def test_get_arg_or_config(monkeypatch, app, parsed_args, environ, item, expected): + for var, content in environ.items(): + monkeypatch.setenv(var, content) + + assert app.get_arg_or_config(parsed_args, item) == expected + + @pytest.mark.parametrize( ("managed", "error", "exit_code", "message"), [ From 8afd5069e3bd6b6027a8bb6eaa8b1013bf37578a Mon Sep 17 00:00:00 2001 From: Alex Lowe Date: Wed, 18 Sep 2024 11:58:39 -0400 Subject: [PATCH 73/82] docs(changelog): add 4.2.3 to changelog --- docs/reference/changelog.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index 2619b5e3..ee74bafa 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -2,6 +2,17 @@ Changelog ********* +4.2.3 (2024-Sep-18) +------------------- + +Application +=========== + +- ``get_arg_or_config`` now correctly checks the config service if the passed + namespace has ``None`` as the value of the requested item. + +For a complete list of commits, check out the `4.2.3`_ release on GitHub. + 4.2.2 (2024-Sep-13) ------------------- @@ -266,3 +277,4 @@ For a complete list of commits, check out the `2.7.0`_ release on GitHub. .. _4.2.0: https://github.com/canonical/craft-application/releases/tag/4.2.0 .. _4.2.1: https://github.com/canonical/craft-application/releases/tag/4.2.1 .. _4.2.2: https://github.com/canonical/craft-application/releases/tag/4.2.2 +.. _4.2.3: https://github.com/canonical/craft-application/releases/tag/4.2.3 From d448c4494d3834679da95db479f952da9fe4cf37 Mon Sep 17 00:00:00 2001 From: Matt Culler Date: Wed, 18 Sep 2024 16:09:04 -0400 Subject: [PATCH 74/82] feat(docs): update partitions docs (#475) * feat: update partitions docs & fix autobuild path * Apply suggestions from code review * Apply suggestions from code review Co-authored-by: Michael DuBelko * fix(docs): use Michael's suggested text Co-authored-by: Michael DuBelko * Update docs/howto/partitions.rst Co-authored-by: Sergio Schvezov * docs: update other reference w/ Sergio's suggestion Co-authored-by: Sergio Schvezov --------- Co-authored-by: Michael DuBelko Co-authored-by: Sergio Schvezov --- docs/howto/partitions.rst | 60 +++++++++++++++++++++++---------------- tox.ini | 2 +- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/docs/howto/partitions.rst b/docs/howto/partitions.rst index f3dd4d02..72d4a9d6 100644 --- a/docs/howto/partitions.rst +++ b/docs/howto/partitions.rst @@ -45,45 +45,57 @@ Required application changes To add partition support to an application, two basic changes are needed: -#. Enable the feature +#. Enable the feature. - Use the :class:`Features ` class to specify that the - application will use partitions: + In your Application subclass, override the following method and invoke the + :class:`Features ` class: .. code-block:: python from craft_parts import Features - Features.reset() - Features(enable_partitions=True) + class ExampleApplication(Application): - .. NOTE:: - The ``craft-application`` class :class:`AppFeatures - ` has a similar name and serves a similar - purpose to ``craft-parts``'s :class:`Features `, - but partitions cannot be enabled via :class:`AppFeatures - `! + ... -#. Define the list of partitions + @override + def _enable_craft_parts_features(self) -> None: + Features(enable_partitions=True) - We need to tell the :class:`LifecycleManager ` - class about our partitions, but applications do not usually directly - instantiate the LifecycleManager. + You can only be enable partitions with the :class:`Features + ` class from craft-parts. In craft-application + there's a similarly-named :class:`AppFeatures + ` class which serves a similar purpose, + but it can't enable partitions. - Instead, override your :class:`Application - `'s ``_setup_partitions`` method, and return - a list of the partitions, which will eventually be passed to the - :class:`LifecycleManager `: + .. Tip:: + In unit tests, the :class:`Features ` global + singleton may raise exceptions when successive tests repeatedly try to + enable partitions. + + To prevent these errors, reset the features at the start of each test: + + .. code-block:: python + + Features.reset() + + + +#. Define the list of partitions. + + Override the ``_setup_partitions`` method of your :class:`Application + ` class and return the list of the + partitions. .. code-block:: python - class SnackcraftApplication(Application): + class ExampleApplication(Application): - ... + ... - @override - def _setup_partitions(self, yaml_data: dict[str, Any]) -> list[str] | None: - return ["default", "kernel", "component/bar-baz"] + @override + def _setup_partitions(self, yaml_data: dict[str, Any]) -> list[str] | None: + return ["default", "kernel", "component/bar-baz"] Using the partitions ==================== diff --git a/tox.ini b/tox.ini index 1299849e..4cffea77 100644 --- a/tox.ini +++ b/tox.ini @@ -119,7 +119,7 @@ commands = sphinx-build {posargs:-b html} -W {tox_root}/docs {tox_root}/docs/_bu [testenv:autobuild-docs] description = Build documentation with an autoupdating server base = docs -commands = sphinx-autobuild {posargs:-b html --open-browser --port 8080} -W --watch {tox_root}/craft_application {tox_root}/docs {tox_root}/docs/_build/html +commands = sphinx-autobuild {posargs:-b html --open-browser --port 8080} -W --watch {tox_root}/craft_application {tox_root}/docs {tox_root}/docs/_build [testenv:lint-docs] description = Lint the documentation with sphinx-lint From 092d34aa2693ab19053d4b4cf134755eb6db744a Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Thu, 19 Sep 2024 13:21:04 -0400 Subject: [PATCH 75/82] test: fix broken uts after merge Signed-off-by: Dariusz Duda --- tests/conftest.py | 15 ++++--- tests/unit/services/test_config.py | 5 ++- tests/unit/services/test_lifecycle.py | 20 +++++----- tests/unit/services/test_provider.py | 16 ++++---- tests/unit/test_application.py | 56 ++++++++++++--------------- 5 files changed, 53 insertions(+), 59 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6a771f99..b651da7e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,10 +27,10 @@ import craft_parts import craft_platforms import distro +import pydantic import pytest -from craft_application import application, launchpad, models, services, util +from craft_application import application, launchpad, models, services from craft_cli import EmitterMode, emit -from craft_providers import bases from typing_extensions import override if TYPE_CHECKING: # pragma: no cover @@ -39,12 +39,11 @@ def _create_fake_build_plan(num_infos: int = 1) -> list[craft_platforms.BuildInfo]: """Create a build plan that is able to execute on the running system.""" - arch = util.get_host_architecture() return [ craft_platforms.BuildInfo( "foo", - craft_platforms.DebianArchitecture(arch), - craft_platforms.DebianArchitecture(arch), + craft_platforms.DebianArchitecture.from_host(), + craft_platforms.DebianArchitecture.from_host(), craft_platforms.DistroBase.from_linux_distribution( distro.LinuxDistribution() ), @@ -129,7 +128,7 @@ def app_metadata_docs(features) -> craft_application.AppMetadata: @pytest.fixture def fake_project() -> models.Project: - arch = util.get_host_architecture() + arch = craft_platforms.DebianArchitecture.from_host() return models.Project( name="full-project", # pyright: ignore[reportArgumentType] title="A fully-defined project", # pyright: ignore[reportArgumentType] @@ -149,13 +148,13 @@ def fake_project() -> models.Project: @pytest.fixture -def fake_build_plan(request) -> list[models.BuildInfo]: +def fake_build_plan(request) -> list[craft_platforms.BuildInfo]: num_infos = getattr(request, "param", 1) return _create_fake_build_plan(num_infos) @pytest.fixture -def full_build_plan(mocker) -> list[models.BuildInfo]: +def full_build_plan(mocker) -> list[craft_platforms.BuildInfo]: """A big build plan with multiple bases and build-for targets.""" host_arch = craft_platforms.DebianArchitecture.from_host() build_plan = [] diff --git a/tests/unit/services/test_config.py b/tests/unit/services/test_config.py index 85d0c41f..cbbeacb6 100644 --- a/tests/unit/services/test_config.py +++ b/tests/unit/services/test_config.py @@ -144,7 +144,10 @@ def test_craft_environment_handler_error( ) def test_snap_config_handler(snap_config_handler, item: str, content: str): snap_item = item.replace("_", "-") - with pytest_subprocess.FakeProcess.context() as fp, pytest.MonkeyPatch.context() as mp: + with ( + pytest_subprocess.FakeProcess.context() as fp, + pytest.MonkeyPatch.context() as mp, + ): mp.setattr("snaphelpers._ctl.Popen", subprocess.Popen) fp.register( ["/usr/bin/snapctl", "get", "-d", snap_item], diff --git a/tests/unit/services/test_lifecycle.py b/tests/unit/services/test_lifecycle.py index 54051c70..d7fe4b1d 100644 --- a/tests/unit/services/test_lifecycle.py +++ b/tests/unit/services/test_lifecycle.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License along # with this program. If not, see . """Unit tests for parts lifecycle.""" + from __future__ import annotations import dataclasses @@ -29,7 +30,6 @@ import pytest_check from craft_application import errors, models, util from craft_application.errors import InvalidParameterError, PartsLifecycleError -from craft_application.models.project import BuildInfo from craft_application.services import lifecycle from craft_application.util import repositories from craft_parts import ( @@ -362,44 +362,44 @@ def test_get_primed_stage_packages(lifecycle_service): ([], None), ( [ - BuildInfo( + craft_platforms.BuildInfo( "my-platform", build_on="any", build_for="all", - base=bases.BaseName("ubuntu", "24.04"), + build_base=craft_platforms.DistroBase("ubuntu", "24.04"), ) ], None, ), ( [ - BuildInfo( + craft_platforms.BuildInfo( "my-platform", build_on="any", build_for="amd64", - base=bases.BaseName("ubuntu", "24.04"), + build_base=craft_platforms.DistroBase("ubuntu", "24.04"), ) ], "amd64", ), ( [ - BuildInfo( + craft_platforms.BuildInfo( "my-platform", build_on="any", build_for="arm64", - base=bases.BaseName("ubuntu", "24.04"), + build_base=craft_platforms.DistroBase("ubuntu", "24.04"), ) ], "arm64", ), ( [ - BuildInfo( + craft_platforms.BuildInfo( "my-platform", build_on="any", build_for="riscv64", - base=bases.BaseName("ubuntu", "24.04"), + build_base=craft_platforms.DistroBase("ubuntu", "24.04"), ) ], "riscv64", @@ -409,7 +409,7 @@ def test_get_primed_stage_packages(lifecycle_service): def test_get_build_for( fake_host_architecture, fake_parts_lifecycle: lifecycle.LifecycleService, - build_plan: list[BuildInfo], + build_plan: list[craft_platforms.BuildInfo], expected: str | None, ): if expected is None: diff --git a/tests/unit/services/test_provider.py b/tests/unit/services/test_provider.py index 71432687..bec7b697 100644 --- a/tests/unit/services/test_provider.py +++ b/tests/unit/services/test_provider.py @@ -421,13 +421,11 @@ def test_get_base_packages(provider_service): "base_name", [ craft_platforms.DistroBase("ubuntu", "devel"), + craft_platforms.DistroBase("ubuntu", "24.10"), + craft_platforms.DistroBase("ubuntu", "24.04"), craft_platforms.DistroBase("ubuntu", "22.04"), - ("ubuntu", "devel"), - ("ubuntu", "24.10"), - ("ubuntu", "24.04"), - ("ubuntu", "22.04"), - ("ubuntu", "20.04"), - ("almalinux", "9"), + craft_platforms.DistroBase("ubuntu", "20.04"), + craft_platforms.DistroBase("almalinux", "9"), ], ) def test_instance( @@ -484,9 +482,9 @@ def test_instance_clean_existing( mock_provider, clean_existing, ): - arch = util.get_host_architecture() - base_name = bases.BaseName("ubuntu", "24.04") - build_info = models.BuildInfo("foo", arch, arch, base_name) + arch = craft_platforms.DebianArchitecture.from_host() + base_name = craft_platforms.DistroBase(distribution="ubuntu", series="24.04") + build_info = craft_platforms.BuildInfo("foo", arch, arch, base_name) with provider_service.instance( build_info, work_dir=tmp_path, clean_existing=clean_existing diff --git a/tests/unit/test_application.py b/tests/unit/test_application.py index 829a781f..d801f3e6 100644 --- a/tests/unit/test_application.py +++ b/tests/unit/test_application.py @@ -47,10 +47,6 @@ models, secrets, services, - util, -) -from craft_application.util import ( - get_host_architecture, # pyright: ignore[reportGeneralTypeIssues] ) from craft_cli import emit from craft_parts.plugins.plugins import PluginType @@ -477,7 +473,7 @@ def test_run_managed_success(mocker, app, fake_project, fake_build_plan): app.project = fake_project app._build_plan = fake_build_plan mock_pause = mocker.spy(craft_cli.emit, "pause") - arch = get_host_architecture() + arch = craft_platforms.DebianArchitecture.from_host() app.run_managed(None, arch) @@ -501,7 +497,7 @@ def test_run_managed_failure(app, fake_project, fake_build_plan): app._build_plan = fake_build_plan with pytest.raises(craft_providers.ProviderError) as exc_info: - app.run_managed(None, get_host_architecture()) + app.run_managed(None, craft_platforms.DebianArchitecture.from_host()) assert exc_info.value.brief == "Failed to execute testcraft in instance." @@ -523,7 +519,7 @@ def test_run_managed_secrets(app, fake_project, fake_build_plan): secret_strings=set(), ) - app.run_managed(None, get_host_architecture()) + app.run_managed(None, craft_platforms.DebianArchitecture.from_host()) # Check that the encoded secrets were propagated to the managed instance. assert len(mock_execute.mock_calls) == 1 @@ -869,7 +865,7 @@ def test_run_success_managed(monkeypatch, app, fake_project, mocker): def test_run_success_managed_with_arch(monkeypatch, app, fake_project, mocker): mocker.patch.object(app, "get_project", return_value=fake_project) app.run_managed = mock.Mock() - arch = get_host_architecture() + arch = craft_platforms.DebianArchitecture.from_host() monkeypatch.setattr(sys, "argv", ["testcraft", "pull", f"--build-for={arch}"]) pytest_check.equal(app.run(), 0) @@ -893,12 +889,12 @@ def test_run_success_managed_with_platform(monkeypatch, app, fake_project, mocke ([], mock.call(None, None)), (["--platform=s390x"], mock.call("s390x", None)), ( - ["--platform", get_host_architecture()], - mock.call(get_host_architecture(), None), + ["--platform", craft_platforms.DebianArchitecture.from_host()], + mock.call(craft_platforms.DebianArchitecture.from_host(), None), ), ( - ["--build-for", get_host_architecture()], - mock.call(None, get_host_architecture()), + ["--build-for", craft_platforms.DebianArchitecture.from_host()], + mock.call(None, craft_platforms.DebianArchitecture.from_host()), ), (["--build-for", "s390x"], mock.call(None, "s390x")), (["--platform", "s390x,riscv64"], mock.call("s390x", None)), @@ -1298,7 +1294,7 @@ def test_work_dir_project_non_managed(monkeypatch, app_metadata, fake_services): app = application.Application(app_metadata, fake_services) assert app._work_dir == pathlib.Path.cwd() - project = app.get_project(build_for=get_host_architecture()) + project = app.get_project(build_for=craft_platforms.DebianArchitecture.from_host()) # Make sure the project is loaded correctly (from the cwd) assert project is not None @@ -1313,7 +1309,7 @@ def test_work_dir_project_managed(monkeypatch, app_metadata, fake_services): app = application.Application(app_metadata, fake_services) assert app._work_dir == pathlib.PosixPath("/root") - project = app.get_project(build_for=get_host_architecture()) + project = app.get_project(build_for=craft_platforms.DebianArchitecture.from_host()) # Make sure the project is loaded correctly (from the cwd) assert project is not None @@ -1350,7 +1346,6 @@ def environment_project(monkeypatch, tmp_path): return project_path -@pytest.mark.xfail(reason="craft-platforms does not support 'all'.", strict=True) def test_expand_environment_build_for_all( monkeypatch, app_metadata, tmp_path, fake_services, emitter ): @@ -1366,7 +1361,7 @@ def test_expand_environment_build_for_all( base: ubuntu@24.04 platforms: platform1: - build-on: [{util.get_host_architecture()}] + build-on: [{craft_platforms.DebianArchitecture.from_host()}] build-for: [all] parts: mypart: @@ -1385,12 +1380,12 @@ def test_expand_environment_build_for_all( # Make sure the project is loaded correctly (from the cwd) assert project is not None assert project.parts["mypart"]["build-environment"] == [ - {"BUILD_ON": util.get_host_architecture()}, - {"BUILD_FOR": util.get_host_architecture()}, + {"BUILD_ON": craft_platforms.DebianArchitecture.from_host()}, + {"BUILD_FOR": craft_platforms.DebianArchitecture.from_host()}, ] emitter.assert_debug( "Expanding environment variables with the host architecture " - f"{util.get_host_architecture()!r} as the build-for architecture " + f"'{craft_platforms.DebianArchitecture.from_host()}' as the build-for architecture " "because 'all' was specified." ) @@ -1398,14 +1393,14 @@ def test_expand_environment_build_for_all( @pytest.mark.usefixtures("environment_project") def test_application_expand_environment(app_metadata, fake_services): app = application.Application(app_metadata, fake_services) - project = app.get_project(build_for=get_host_architecture()) + project = app.get_project(build_for=craft_platforms.DebianArchitecture.from_host()) # Make sure the project is loaded correctly (from the cwd) assert project is not None assert project.parts["mypart"]["source-tag"] == "v1.2.3" assert project.parts["mypart"]["build-environment"] == [ - {"BUILD_ON": util.get_host_architecture()}, - {"BUILD_FOR": util.get_host_architecture()}, + {"BUILD_ON": craft_platforms.DebianArchitecture.from_host()}, + {"BUILD_FOR": craft_platforms.DebianArchitecture.from_host()}, ] @@ -1444,7 +1439,7 @@ def test_application_build_secrets(app_metadata, fake_services, monkeypatch, moc spied_set_secrets = mocker.spy(craft_cli.emit, "set_secrets") app = application.Application(app_metadata, fake_services) - project = app.get_project(build_for=get_host_architecture()) + project = app.get_project(build_for=craft_platforms.DebianArchitecture.from_host()) # Make sure the project is loaded correctly (from the cwd) assert project is not None @@ -1559,7 +1554,7 @@ def test_extra_yaml_transform(tmp_path, app_metadata, fake_services): app.project_dir = tmp_path _ = app.get_project(build_for="s390x") - assert app.build_on == util.get_host_architecture() + assert app.build_on == craft_platforms.DebianArchitecture.from_host() assert app.build_for == "s390x" @@ -1575,7 +1570,7 @@ def test_mandatory_adoptable_fields(tmp_path, app_metadata, fake_services): app.project_dir = tmp_path with pytest.raises(errors.CraftValidationError) as exc_info: - _ = app.get_project(build_for=get_host_architecture()) + _ = app.get_project(build_for=craft_platforms.DebianArchitecture.from_host()) assert ( str(exc_info.value) @@ -1727,7 +1722,6 @@ def test_process_grammar_build_for(grammar_app_mini): ] -@pytest.mark.xfail(reason="craft-platforms does not support 'all'.", strict=True) def test_process_grammar_to_all(tmp_path, app_metadata, fake_services): """Test that 'to all' is a valid grammar statement.""" contents = dedent( @@ -1737,18 +1731,18 @@ def test_process_grammar_to_all(tmp_path, app_metadata, fake_services): base: ubuntu@24.04 platforms: myplatform: - build-on: [{util.get_host_architecture()}] + build-on: [{craft_platforms.DebianArchitecture.from_host()}] build-for: [all] parts: mypart: plugin: nil build-packages: - test-package - - on {util.get_host_architecture()} to all: + - on {craft_platforms.DebianArchitecture.from_host()} to all: - on-host-to-all - to all: - to-all - - on {util.get_host_architecture()} to s390x: + - on {craft_platforms.DebianArchitecture.from_host()} to s390x: - on-host-to-s390x - to s390x: - on-amd64-to-s390x @@ -1858,7 +1852,7 @@ def test_process_yaml_from_extra_transform( ] assert project.parts["mypart"]["build-environment"] == [ # evaluate project variables - {"hello": get_host_architecture()}, + {"hello": craft_platforms.DebianArchitecture.from_host()}, # render secrets {"MY_VAR": "secret-value"}, ] @@ -1908,7 +1902,7 @@ def environment_partitions_project(monkeypatch, tmp_path): @pytest.mark.usefixtures("environment_partitions_project") def test_partition_application_expand_environment(app_metadata, fake_services): app = FakePartitionsApplication(app_metadata, fake_services) - project = app.get_project(build_for=get_host_architecture()) + project = app.get_project(build_for=craft_platforms.DebianArchitecture.from_host()) assert craft_parts.Features().enable_partitions is True # Make sure the project is loaded correctly (from the cwd) From d5ef878012b3f3ad02a91b6f255c7d05603e5bb1 Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Fri, 20 Sep 2024 12:15:54 -0400 Subject: [PATCH 76/82] test: fix fixtures instantation order Signed-off-by: Dariusz Duda --- tests/unit/services/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/services/test_config.py b/tests/unit/services/test_config.py index cbbeacb6..0f81dca3 100644 --- a/tests/unit/services/test_config.py +++ b/tests/unit/services/test_config.py @@ -217,8 +217,8 @@ def test_default_config_handler_success(default_config_handler, item, expected): ) def test_config_service_converts_type( monkeypatch: pytest.MonkeyPatch, + fake_services, # Has to be intantiated before `fake_process` as it calls lsb_release fake_process: pytest_subprocess.FakeProcess, - fake_services, item: str, environment_variables: dict[str, str], expected, From e3c2b488e4cc98b4f00252148c63bcea29963c1b Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Fri, 20 Sep 2024 12:17:53 -0400 Subject: [PATCH 77/82] test: fix fetch-service tests Signed-off-by: Dariusz Duda --- tests/integration/services/test_fetch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index babeee06..2f9fe9f0 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . """Tests for FetchService.""" + import contextlib import io import pathlib @@ -26,8 +27,7 @@ import craft_providers import pytest from craft_application import errors, fetch, services, util -from craft_application.models import BuildInfo -from craft_providers import bases +from craft_platforms import BuildInfo, DistroBase @cache @@ -224,7 +224,7 @@ def lxd_instance(snap_safe_tmp_path, provider_service): provider_service.get_provider("lxd") arch = util.get_host_architecture() - build_info = BuildInfo("foo", arch, arch, bases.BaseName("ubuntu", "22.04")) + build_info = BuildInfo("foo", arch, arch, DistroBase("ubuntu", "22.04")) instance = provider_service.instance(build_info, work_dir=snap_safe_tmp_path) with instance as executor: From 93de59b58d6871936c19af899f3166a88e0e6888 Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Fri, 20 Sep 2024 17:20:34 -0400 Subject: [PATCH 78/82] test: uncomment test for allbuilf-for Signed-off-by: Dariusz Duda --- tests/unit/models/test_project.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index 0b642275..6823b701 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . """Tests for BaseProject""" + import copy import pathlib import re @@ -591,18 +592,18 @@ def test_platform_invalid_build_arch(model, field_name, basic_project_dict): ], id="complex", ), - # pytest.param( - # {"arm64": {"build-on": ["arm64"], "build-for": ["all"]}}, - # [ - # craft_platforms.BuildInfo( - # platform="arm64", - # build_on=craft_platforms.DebianArchitecture("arm64"), - # build_for="all", - # build_base=craft_platforms.DistroBase("ubuntu", "24.04"), - # ), - # ], - # id="all", - # ), + pytest.param( + {"arm64": {"build-on": ["arm64"], "build-for": ["all"]}}, + [ + craft_platforms.BuildInfo( + platform="arm64", + build_on=craft_platforms.DebianArchitecture("arm64"), + build_for="all", + build_base=craft_platforms.DistroBase("ubuntu", "24.04"), + ), + ], + id="all", + ), ], ) def test_get_build_plan(platforms, expected_build_info): From 760ecad85090874ba6100ceef9c5cae8433cd64e Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Fri, 20 Sep 2024 17:48:24 -0400 Subject: [PATCH 79/82] chore: use craft-platforms in MultipleBuildsError Signed-off-by: Dariusz Duda --- craft_application/errors.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/craft_application/errors.py b/craft_application/errors.py index c5ee0e33..b5632fcf 100644 --- a/craft_application/errors.py +++ b/craft_application/errors.py @@ -17,6 +17,7 @@ All errors inherit from craft_cli.CraftError. """ + from __future__ import annotations import os @@ -25,9 +26,9 @@ import yaml from craft_cli import CraftError +from craft_platforms import BuildInfo from craft_providers import bases -from craft_application import models from craft_application.util.error_formatting import format_pydantic_errors from craft_application.util.string import humanize_list @@ -164,7 +165,7 @@ def __init__(self) -> None: class MultipleBuildsError(CraftError): """The build plan contains multiple possible builds.""" - def __init__(self, matching_builds: list[models.BuildInfo] | None = None) -> None: + def __init__(self, matching_builds: list[BuildInfo] | None = None) -> None: message = "Multiple builds match the current platform" if matching_builds: message += ": " + humanize_list( From 062ebd023f70181e63a7f251fe0f980fb59697a3 Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Fri, 20 Sep 2024 17:48:56 -0400 Subject: [PATCH 80/82] chore: use craft-platforms in remaing tests Signed-off-by: Dariusz Duda --- tests/integration/services/test_fetch.py | 4 ++-- tests/unit/services/test_lifecycle.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/integration/services/test_fetch.py b/tests/integration/services/test_fetch.py index 2f9fe9f0..85100c72 100644 --- a/tests/integration/services/test_fetch.py +++ b/tests/integration/services/test_fetch.py @@ -27,7 +27,7 @@ import craft_providers import pytest from craft_application import errors, fetch, services, util -from craft_platforms import BuildInfo, DistroBase +from craft_platforms import BuildInfo, DistroBase, DebianArchitecture @cache @@ -223,7 +223,7 @@ def test_service_logging(app_service, mocker, tmp_path, monkeypatch): def lxd_instance(snap_safe_tmp_path, provider_service): provider_service.get_provider("lxd") - arch = util.get_host_architecture() + arch = DebianArchitecture.from_host() build_info = BuildInfo("foo", arch, arch, DistroBase("ubuntu", "22.04")) instance = provider_service.instance(build_info, work_dir=snap_safe_tmp_path) diff --git a/tests/unit/services/test_lifecycle.py b/tests/unit/services/test_lifecycle.py index d7fe4b1d..b489cbea 100644 --- a/tests/unit/services/test_lifecycle.py +++ b/tests/unit/services/test_lifecycle.py @@ -376,33 +376,33 @@ def test_get_primed_stage_packages(lifecycle_service): craft_platforms.BuildInfo( "my-platform", build_on="any", - build_for="amd64", + build_for=craft_platforms.DebianArchitecture.AMD64, build_base=craft_platforms.DistroBase("ubuntu", "24.04"), ) ], - "amd64", + craft_platforms.DebianArchitecture.AMD64, ), ( [ craft_platforms.BuildInfo( "my-platform", build_on="any", - build_for="arm64", + build_for=craft_platforms.DebianArchitecture.ARM64, build_base=craft_platforms.DistroBase("ubuntu", "24.04"), ) ], - "arm64", + craft_platforms.DebianArchitecture.ARM64, ), ( [ craft_platforms.BuildInfo( "my-platform", build_on="any", - build_for="riscv64", + build_for=craft_platforms.DebianArchitecture.RISCV64, build_base=craft_platforms.DistroBase("ubuntu", "24.04"), ) ], - "riscv64", + craft_platforms.DebianArchitecture.RISCV64, ), ], ) From 65bcf1a7208a417af94595579ebe1f1a1f2501e2 Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Fri, 20 Sep 2024 17:49:49 -0400 Subject: [PATCH 81/82] chore: cast items before passing to build_plan Signed-off-by: Dariusz Duda --- craft_application/models/project.py | 9 +++++---- craft_application/services/provider.py | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/craft_application/models/project.py b/craft_application/models/project.py index 27392109..1feef50e 100644 --- a/craft_application/models/project.py +++ b/craft_application/models/project.py @@ -17,10 +17,11 @@ This defines the structure of the input file (e.g. snapcraft.yaml) """ + import abc import dataclasses from collections.abc import Mapping -from typing import Annotated, Any +from typing import Annotated, Any, cast import craft_parts import craft_platforms @@ -200,9 +201,9 @@ def get_build_plan(self) -> list[craft_platforms.BuildInfo]: """Obtain the list of architectures and bases from the Project.""" data = self.marshal() build_infos = craft_platforms.get_platforms_build_plan( - base=data["base"], - platforms=data["platforms"], - build_base=data.get("build-base"), + base=cast(str, data["base"]), + platforms=cast(craft_platforms.Platforms, data["platforms"]), + build_base=cast(str | None, data.get("build-base")), ) return list(build_infos) diff --git a/craft_application/services/provider.py b/craft_application/services/provider.py index c86f0b32..f809dab9 100644 --- a/craft_application/services/provider.py +++ b/craft_application/services/provider.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . """Service class for craft-providers.""" + from __future__ import annotations import contextlib @@ -185,14 +186,14 @@ def get_base( ( base_name.distribution, base_name.series, - ) + ) # type: ignore[arg-type] ) else: alias = bases.get_base_alias( ( base_name[0], base_name[1], - ) + ) # type: ignore[arg-type] ) base_class = bases.get_base_from_alias(alias) if base_class is bases.BuilddBase: @@ -358,7 +359,7 @@ def _clean_instance( self, provider: craft_providers.Provider, work_dir: pathlib.Path, - info: models.BuildInfo, + info: craft_platforms.BuildInfo, ) -> None: """Clean an instance, if it exists.""" instance_name = self._get_instance_name(work_dir, info) From 6a744365165402864bb39ec6c6befef3ef647c3d Mon Sep 17 00:00:00 2001 From: Dariusz Duda Date: Fri, 20 Sep 2024 17:56:53 -0400 Subject: [PATCH 82/82] chore(lint): fix codespell complaints Signed-off-by: Dariusz Duda --- tests/unit/services/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/services/test_config.py b/tests/unit/services/test_config.py index 0f81dca3..d9f40a26 100644 --- a/tests/unit/services/test_config.py +++ b/tests/unit/services/test_config.py @@ -217,7 +217,7 @@ def test_default_config_handler_success(default_config_handler, item, expected): ) def test_config_service_converts_type( monkeypatch: pytest.MonkeyPatch, - fake_services, # Has to be intantiated before `fake_process` as it calls lsb_release + fake_services, # Has to be instantiated before `fake_process` as it calls lsb_release fake_process: pytest_subprocess.FakeProcess, item: str, environment_variables: dict[str, str],