From b26285f6a1267d2638a1337e1191a6b9ab7b6a3a Mon Sep 17 00:00:00 2001 From: liushuyu Date: Sun, 30 Jul 2023 12:36:24 +0800 Subject: [PATCH] Rust plugin: rewrite the Rust plugin ... ... using the new plugin system Signed-off-by: Zixing Liu --- snapcraft/parts/plugins/__init__.py | 2 + snapcraft/parts/plugins/register.py | 2 + snapcraft/parts/plugins/rust_plugin.py | 185 +++++++++++++++++++ snapcraft/plugins/v2/__init__.py | 4 - snapcraft_legacy/plugins/_plugin_finder.py | 1 - snapcraft_legacy/plugins/v2/__init__.py | 1 - snapcraft_legacy/plugins/v2/rust.py | 109 ----------- tests/legacy/unit/plugins/v2/test_rust.py | 108 ----------- tests/unit/parts/plugins/test_rust_plugin.py | 154 +++++++++++++++ 9 files changed, 343 insertions(+), 223 deletions(-) create mode 100644 snapcraft/parts/plugins/rust_plugin.py delete mode 100644 snapcraft_legacy/plugins/v2/rust.py delete mode 100644 tests/legacy/unit/plugins/v2/test_rust.py create mode 100644 tests/unit/parts/plugins/test_rust_plugin.py diff --git a/snapcraft/parts/plugins/__init__.py b/snapcraft/parts/plugins/__init__.py index 385a1c0c739..1d7bcb5fa6e 100644 --- a/snapcraft/parts/plugins/__init__.py +++ b/snapcraft/parts/plugins/__init__.py @@ -23,6 +23,7 @@ from .kernel_plugin import KernelPlugin from .python_plugin import PythonPlugin from .register import register +from .rust_plugin import RustPlugin __all__ = [ "ColconPlugin", @@ -30,5 +31,6 @@ "FlutterPlugin", "KernelPlugin", "PythonPlugin", + "RustPlugin", "register", ] diff --git a/snapcraft/parts/plugins/register.py b/snapcraft/parts/plugins/register.py index 87b0a731eb9..eacbcd51e96 100644 --- a/snapcraft/parts/plugins/register.py +++ b/snapcraft/parts/plugins/register.py @@ -23,6 +23,7 @@ from .flutter_plugin import FlutterPlugin from .kernel_plugin import KernelPlugin from .python_plugin import PythonPlugin +from .rust_plugin import RustPlugin def register() -> None: @@ -32,3 +33,4 @@ def register() -> None: craft_parts.plugins.register({"flutter": FlutterPlugin}) craft_parts.plugins.register({"python": PythonPlugin}) craft_parts.plugins.register({"kernel": KernelPlugin}) + craft_parts.plugins.register({"rust": RustPlugin}) diff --git a/snapcraft/parts/plugins/rust_plugin.py b/snapcraft/parts/plugins/rust_plugin.py new file mode 100644 index 00000000000..23ceafaeeb6 --- /dev/null +++ b/snapcraft/parts/plugins/rust_plugin.py @@ -0,0 +1,185 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright (C) 2023 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The Snapcraft Rust plugin.""" + +import logging +import subprocess +from typing import Any, Dict, List, Set, cast + +from craft_parts import plugins +from overrides import overrides + +logger = logging.getLogger(__name__) + + +class RustPluginProperties(plugins.PluginProperties, plugins.PluginModel): + """The part properties used by the Rust plugin.""" + + source: str + rust_channel: str = "stable" + rust_features: List[str] = [] + rust_path: List[str] = ["."] + rust_use_global_lto: bool = False + rust_no_default_features: bool = False + + @classmethod + @overrides + def unmarshal(cls, data: Dict[str, Any]) -> "RustPluginProperties": + """Populate class attributes from the part specification. + + :param data: A dictionary containing part properties. + + :return: The populated plugin properties data object. + + :raise pydantic.ValidationError: If validation fails. + """ + plugin_data = plugins.extract_plugin_properties( + data, + plugin_name="rust", + required=["source"], + ) + return cls(**plugin_data) + + +class RustPlugin(plugins.Plugin): + """A Snapcraft plugin for Rust applications. + + This Rust plugin is useful for building Rust based parts. + + Rust uses cargo to drive the build. + + This plugin uses the common plugin keywords as well as those for "sources". + For more information check the 'plugins' topic for the former and the + 'sources' topic for the latter. + + Additionally, this plugin uses the following plugin-specific keywords: + - rust-channel + (string, default "stable") + Used to select which Rust channel or version to use. + It can be one of "stable", "beta", "nightly" or a version number. + If you don't want this plugin to install Rust toolchain for you, + you can put "none" for this option. + + - rust-features + (list of strings) + Features used to build optional dependencies + + - rust-path + (list of strings, default [.]) + Build specific crates inside the workspace + + - rust-no-default-features + (boolean, default False) + Whether to disable the default features in this crate. + Equivalent to setting `--no-default-features` on the commandline. + + - rust-use-global-lto + (boolean, default False) + Whether to use global LTO. + This option may significantly impact the build performance but + reducing the final binary size. + This will forcibly enable LTO for all the crates you specified, + regardless of whether you have LTO enabled in the Cargo.toml file + """ + + properties_class = RustPluginProperties + + @overrides + def get_build_snaps(self) -> Set[str]: + return set() + + @overrides + def get_build_packages(self) -> Set[str]: + return {"curl", "gcc", "git", "pkg-config", "findutils"} + + def _check_rustup(self) -> bool: + try: + rustup_version = subprocess.check_output(["rustup", "--version"]) + return "rustup" in rustup_version.decode("utf-8") + except (subprocess.CalledProcessError, FileNotFoundError): + return False + + def _get_setup_rustup(self, channel: str) -> List[str]: + return [ + f"""\ +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ +sh -s -- -y --no-modify-path --profile=minimal --default-toolchain {channel} +""" + ] + + @overrides + def get_build_environment(self) -> Dict[str, str]: + return { + "PATH": "${HOME}/.cargo/bin:${PATH}", + } + + @overrides + def get_build_commands(self) -> List[str]: + options = cast(RustPluginProperties, self._options) + + rust_install_cmd: List[str] = [] + rust_build_cmd: List[str] = [] + config_cmd: List[str] = [] + + if options.rust_channel == "none": + rust_install_cmd = [] + elif not self._check_rustup(): + logger.info("Rustup not found, installing it") + rust_install_cmd = self._get_setup_rustup(options.rust_channel) + else: + logger.info("Switch rustup channel to %s", options.rust_channel) + rust_install_cmd = [f"rustup default {options.rust_channel}"] + + if options.rust_features: + features_string = " ".join(options.rust_features) + config_cmd.extend(["--features", f"'{features_string}'"]) + + if options.rust_use_global_lto: + logger.info("Adding overrides for LTO support") + config_cmd.extend( + [ + "--config 'profile.release.lto = true'", + "--config 'profile.release.codegen-units = 1'", + ] + ) + + if options.rust_no_default_features: + config_cmd.append("--no-default-features") + + for crate in options.rust_path: + logger.info("Generating build commands for %s", crate) + config_cmd_string = " ".join(config_cmd) + # TODO(liushuyu): the current fallback installation method will also install + # compiler plugins into the final Snap, which is likely not what we want. + rust_build_cmd_single = f"""\ +if cargo read-manifest --manifest-path "{crate}"/Cargo.toml > /dev/null; then + cargo install -f --locked --path "{crate}" --root "${{SNAPCRAFT_PART_INSTALL}}" {config_cmd_string} + # remove the installation metadata + rm -f "${{SNAPCRAFT_PART_INSTALL}}"/.crates{{.toml,2.json}} +else + # virtual workspace is a bit tricky, + # we need to build the whole workspace and then copy the binaries ourselves + pushd "{crate}" + cargo build --workspace --release {config_cmd_string} + # install the final binaries + # TODO(liushuyu): this will also install proc macros in the workspace, + # which the user may not want to keep in the final installation + find ./target/release -maxdepth 1 -executable -exec install -Dvm755 {{}} "${{SNAPCRAFT_PART_INSTALL}}" ';' + popd +fi""" + rust_build_cmd.append(rust_build_cmd_single) + return rust_install_cmd + rust_build_cmd diff --git a/snapcraft/plugins/v2/__init__.py b/snapcraft/plugins/v2/__init__.py index b10899a0601..0e61bf05ce0 100644 --- a/snapcraft/plugins/v2/__init__.py +++ b/snapcraft/plugins/v2/__init__.py @@ -35,7 +35,6 @@ PluginV2, PythonPlugin, QMakePlugin, - RustPlugin, autotools, catkin, catkin_tools, @@ -52,7 +51,6 @@ npm, python, qmake, - rust, ) __all__ = [ @@ -72,7 +70,6 @@ "npm", "python", "qmake", - "rust", "PluginV2", "AutotoolsPlugin", "CatkinPlugin", @@ -90,5 +87,4 @@ "NpmPlugin", "PythonPlugin", "QMakePlugin", - "RustPlugin", ] diff --git a/snapcraft_legacy/plugins/_plugin_finder.py b/snapcraft_legacy/plugins/_plugin_finder.py index b6f809d1fa2..e556ddc03db 100644 --- a/snapcraft_legacy/plugins/_plugin_finder.py +++ b/snapcraft_legacy/plugins/_plugin_finder.py @@ -41,7 +41,6 @@ "npm": v2.NpmPlugin, "python": v2.PythonPlugin, "qmake": v2.QMakePlugin, - "rust": v2.RustPlugin, } else: # We cannot import the plugins on anything but linux. diff --git a/snapcraft_legacy/plugins/v2/__init__.py b/snapcraft_legacy/plugins/v2/__init__.py index 272f94b6a02..8fcb892684a 100644 --- a/snapcraft_legacy/plugins/v2/__init__.py +++ b/snapcraft_legacy/plugins/v2/__init__.py @@ -36,4 +36,3 @@ from .npm import NpmPlugin # noqa: F401 from .python import PythonPlugin # noqa: F401 from .qmake import QMakePlugin # noqa: F401 - from .rust import RustPlugin # noqa: F401 diff --git a/snapcraft_legacy/plugins/v2/rust.py b/snapcraft_legacy/plugins/v2/rust.py deleted file mode 100644 index afb1c17e0ef..00000000000 --- a/snapcraft_legacy/plugins/v2/rust.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- -# -# Copyright (C) 2020 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""This rust plugin is useful for building rust based parts. - -Rust uses cargo to drive the build. - -This plugin uses the common plugin keywords as well as those for "sources". -For more information check the 'plugins' topic for the former and the -'sources' topic for the latter. - -Additionally, this plugin uses the following plugin-specific keywords: - - - rust-features - (list of strings) - Features used to build optional dependencies - - - rust-path - (list of strings, default [.]) - Build specific workspace crates - Only one item is currently supported. -""" - -from textwrap import dedent -from typing import Any, Dict, List, Set - -from snapcraft_legacy.plugins.v2 import PluginV2 - - -class RustPlugin(PluginV2): - @classmethod - def get_schema(cls) -> Dict[str, Any]: - return { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": False, - "properties": { - "rust-path": { - "type": "array", - "minItems": 1, - # TODO support more than one item. - "maxItems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": ["."], - }, - "rust-features": { - "type": "array", - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - }, - }, - "required": ["source"], - } - - def get_build_snaps(self) -> Set[str]: - return set() - - def get_build_packages(self) -> Set[str]: - return {"curl", "gcc", "git"} - - def get_build_environment(self) -> Dict[str, str]: - return {"PATH": "${HOME}/.cargo/bin:${PATH}"} - - def _get_rustup_command(self) -> str: - return dedent( - """\ - if ! command -v rustup 2>/dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile=minimal - export PATH="${HOME}/.cargo/bin:${PATH}" - fi - """ - ) - - def _get_install_command(self) -> str: - cmd = [ - "cargo", - "install", - "--locked", - "--path", - self.options.rust_path[0], - "--root", - '"${SNAPCRAFT_PART_INSTALL}"', - "--force", - ] - - if self.options.rust_features: - cmd.extend( - ["--features", "'{}'".format(" ".join(self.options.rust_features))] - ) - - return " ".join(cmd) - - def get_build_commands(self) -> List[str]: - return [self._get_rustup_command(), self._get_install_command()] diff --git a/tests/legacy/unit/plugins/v2/test_rust.py b/tests/legacy/unit/plugins/v2/test_rust.py deleted file mode 100644 index d3ab8832b8e..00000000000 --- a/tests/legacy/unit/plugins/v2/test_rust.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- -# -# Copyright (C) 2020 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from textwrap import dedent - -from testtools import TestCase -from testtools.matchers import Equals - -from snapcraft_legacy.plugins.v2.rust import RustPlugin - - -class RustPluginTest(TestCase): - def test_schema(self): - schema = RustPlugin.get_schema() - - self.assertThat( - schema, - Equals( - { - "$schema": "http://json-schema.org/draft-04/schema#", - "additionalProperties": False, - "properties": { - "rust-features": { - "default": [], - "items": {"type": "string"}, - "type": "array", - "uniqueItems": True, - }, - "rust-path": { - "default": ["."], - "items": {"type": "string"}, - "maxItems": 1, - "minItems": 1, - "type": "array", - "uniqueItems": True, - }, - }, - "required": ["source"], - "type": "object", - } - ), - ) - - def test_get_build_packages(self): - plugin = RustPlugin(part_name="my-part", options=lambda: None) - - self.assertThat(plugin.get_build_packages(), Equals({"curl", "gcc", "git"})) - - def test_get_build_environment(self): - plugin = RustPlugin(part_name="my-part", options=lambda: None) - - self.assertThat( - plugin.get_build_environment(), - Equals({"PATH": "${HOME}/.cargo/bin:${PATH}"}), - ) - - def test_get_build_commands(self): - class Options: - rust_channel = "" - rust_path = ["."] - rust_features = [] - - plugin = RustPlugin(part_name="my-part", options=Options()) - - self.assertThat( - plugin.get_build_commands(), - Equals( - [ - dedent( - """\ - if ! command -v rustup 2>/dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile=minimal - export PATH="${HOME}/.cargo/bin:${PATH}" - fi - """ - ), - 'cargo install --locked --path . --root "${SNAPCRAFT_PART_INSTALL}" --force', - ] - ), - ) - - def test_get_install_command_with_features(self): - class Options: - rust_channel = "" - rust_path = ["path"] - rust_features = ["my-feature", "your-feature"] - - plugin = RustPlugin(part_name="my-part", options=Options()) - - self.assertThat( - plugin._get_install_command(), - Equals( - "cargo install --locked --path path --root \"${SNAPCRAFT_PART_INSTALL}\" --force --features 'my-feature your-feature'" - ), - ) diff --git a/tests/unit/parts/plugins/test_rust_plugin.py b/tests/unit/parts/plugins/test_rust_plugin.py new file mode 100644 index 00000000000..bb86e94aa42 --- /dev/null +++ b/tests/unit/parts/plugins/test_rust_plugin.py @@ -0,0 +1,154 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright 2023 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + + +import pytest +from craft_parts import Part, PartInfo, ProjectInfo + +from snapcraft.parts.plugins import RustPlugin + + +@pytest.fixture(autouse=True) +def part_info(new_dir): + yield PartInfo( + project_info=ProjectInfo( + application_name="test", project_name="test-snap", cache_dir=new_dir + ), + part=Part("my-part", {}), + ) + + +def test_get_build_snaps(part_info): + properties = RustPlugin.properties_class.unmarshal({"source": "."}) + plugin = RustPlugin(properties=properties, part_info=part_info) + assert plugin.get_build_snaps() == set() + + +def test_get_build_packages(part_info): + properties = RustPlugin.properties_class.unmarshal({"source": "."}) + plugin = RustPlugin(properties=properties, part_info=part_info) + assert plugin.get_build_packages() == { + "curl", + "gcc", + "git", + "pkg-config", + "findutils", + } + + +def test_get_build_environment(part_info): + properties = RustPlugin.properties_class.unmarshal({"source": "."}) + plugin = RustPlugin(properties=properties, part_info=part_info) + + assert plugin.get_build_environment() == {"PATH": "${HOME}/.cargo/bin:${PATH}"} + + +def test_get_build_commands_default(part_info): + properties = RustPlugin.properties_class.unmarshal({"source": "."}) + plugin = RustPlugin(properties=properties, part_info=part_info) + plugin._check_rustup = lambda: False + + commands = plugin.get_build_commands() + assert ( + commands[0] + == """curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ +sh -s -- -y --no-modify-path --profile=minimal --default-toolchain stable +""" + ) + assert 'cargo install -f --locked --path "."' in commands[1] + + +def test_get_build_commands_no_install(part_info): + def _check_rustup(): + raise RuntimeError("should not be called") + + properties = RustPlugin.properties_class.unmarshal( + {"source": ".", "rust-channel": "none"} + ) + plugin = RustPlugin(properties=properties, part_info=part_info) + plugin._check_rustup = _check_rustup + + commands = plugin.get_build_commands() + assert len(commands) == 1 + assert "curl" not in commands[0] + assert 'cargo install -f --locked --path "."' in commands[0] + + +def test_get_build_commands_use_lto(part_info): + properties = RustPlugin.properties_class.unmarshal( + {"source": ".", "rust-use-global-lto": True} + ) + plugin = RustPlugin(properties=properties, part_info=part_info) + plugin._check_rustup = lambda: True + + commands = plugin.get_build_commands() + assert len(commands) == 2 + assert "curl" not in commands[0] + assert 'cargo install -f --locked --path "."' in commands[1] + assert "--config 'profile.release.lto = true'" in commands[1] + + +def test_get_build_commands_multiple_crates(part_info): + properties = RustPlugin.properties_class.unmarshal( + {"source": ".", "rust-path": ["a", "b", "c"]} + ) + plugin = RustPlugin(properties=properties, part_info=part_info) + plugin._check_rustup = lambda: True + + commands = plugin.get_build_commands() + assert len(commands) == 4 + assert "curl" not in commands[0] + assert 'cargo install -f --locked --path "a"' in commands[1] + assert 'cargo install -f --locked --path "b"' in commands[2] + assert 'cargo install -f --locked --path "c"' in commands[3] + + +def test_get_build_commands_multiple_features(part_info): + properties = RustPlugin.properties_class.unmarshal( + {"source": ".", "rust-features": ["ft-a", "ft-b"]} + ) + plugin = RustPlugin(properties=properties, part_info=part_info) + plugin._check_rustup = lambda: True + + commands = plugin.get_build_commands() + assert len(commands) == 2 + assert "curl" not in commands[0] + assert 'cargo install -f --locked --path "."' in commands[1] + assert "--features 'ft-a ft-b'" in commands[1] + + +@pytest.mark.parametrize( + "value", + [ + "stable", + "nightly", + "beta", + "1.65", + "1.71.1", + "nightly-2022-12-01", + "stable-x86_64-fortanix-unknown-sgx", + "nightly-2023-06-14-aarch64-nintendo-switch-freestanding", + ], +) +def test_get_build_commands_different_channels(part_info, value): + properties = RustPlugin.properties_class.unmarshal( + {"source": ".", "rust-channel": value} + ) + plugin = RustPlugin(properties=properties, part_info=part_info) + plugin._check_rustup = lambda: False + commands = plugin.get_build_commands() + assert len(commands) == 2 + assert f"--default-toolchain {value}" in commands[0]