From fe8b13ea1ed432a084db69ffaa3d17da7aa28296 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Tue, 27 Aug 2024 11:56:50 +0000 Subject: [PATCH 01/10] Fix hassfest to use hassfest docker image to verify core --- script/hassfest/model.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/script/hassfest/model.py b/script/hassfest/model.py index 736fb6874be8c5..a342bb7024abfc 100644 --- a/script/hassfest/model.py +++ b/script/hassfest/model.py @@ -150,7 +150,12 @@ def domain(self) -> str: @property def core(self) -> bool: """Core integration.""" - return self.path.as_posix().startswith("homeassistant/components") + return self.path.as_posix().startswith( + ( + "homeassistant/components", # used by the CI and local development + "/homeassistant/components", # used by the hassfest docker image + ) + ) @property def disabled(self) -> str | None: From 2dac4e8c334d650b6ad2256d5dd4d6affeef60ff Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 07:01:45 +0000 Subject: [PATCH 02/10] Add hassfest argument to specify core integration path --- script/hassfest/__main__.py | 11 +++++++++-- script/hassfest/model.py | 12 ++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/script/hassfest/__main__.py b/script/hassfest/__main__.py index ea3c56200a25f6..60e4b4339af820 100644 --- a/script/hassfest/__main__.py +++ b/script/hassfest/__main__.py @@ -107,6 +107,12 @@ def get_config() -> Config: default=ALL_PLUGIN_NAMES, help="Comma-separate list of plugins to run. Valid plugin names: %(default)s", ) + parser.add_argument( + "--core-integrations-path", + type=pathlib.Path, + default=pathlib.Path("homeassistant/components"), + help="Path to core integrations", + ) parsed = parser.parse_args() if parsed.action is None: @@ -129,6 +135,7 @@ def get_config() -> Config: action=parsed.action, requirements=parsed.requirements, plugins=set(parsed.plugins), + core_integrations_path=parsed.core_integrations_path, ) @@ -146,12 +153,12 @@ def main() -> int: integrations = {} for int_path in config.specific_integrations: - integration = Integration(int_path) + integration = Integration(int_path, config) integration.load_manifest() integrations[integration.domain] = integration else: - integrations = Integration.load_dir(pathlib.Path("homeassistant/components")) + integrations = Integration.load_dir(config) plugins += HASS_PLUGINS for plugin in plugins: diff --git a/script/hassfest/model.py b/script/hassfest/model.py index a342bb7024abfc..0aab27cf1ef9f5 100644 --- a/script/hassfest/model.py +++ b/script/hassfest/model.py @@ -29,6 +29,7 @@ class Config: root: pathlib.Path action: Literal["validate", "generate"] requirements: bool + core_integrations_path: pathlib.Path errors: list[Error] = field(default_factory=list) cache: dict[str, Any] = field(default_factory=dict) plugins: set[str] = field(default_factory=set) @@ -105,8 +106,9 @@ class Integration: """Represent an integration in our validator.""" @classmethod - def load_dir(cls, path: pathlib.Path) -> dict[str, Integration]: + def load_dir(cls, config: Config) -> dict[str, Integration]: """Load all integrations in a directory.""" + path = config.core_integrations_path assert path.is_dir() integrations: dict[str, Integration] = {} for fil in path.iterdir(): @@ -123,13 +125,14 @@ def load_dir(cls, path: pathlib.Path) -> dict[str, Integration]: ) continue - integration = cls(fil) + integration = cls(fil, config) integration.load_manifest() integrations[integration.domain] = integration return integrations path: pathlib.Path + _config: Config _manifest: dict[str, Any] | None = None manifest_path: pathlib.Path | None = None errors: list[Error] = field(default_factory=list) @@ -151,10 +154,7 @@ def domain(self) -> str: def core(self) -> bool: """Core integration.""" return self.path.as_posix().startswith( - ( - "homeassistant/components", # used by the CI and local development - "/homeassistant/components", # used by the hassfest docker image - ) + self._config.core_integrations_path.as_posix() ) @property From 733ea7807eadaab120030eed8b0bf960ebb61616 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 07:14:40 +0000 Subject: [PATCH 03/10] fix --- script/gen_requirements_all.py | 20 +++++++++++++++++--- script/hassfest/__main__.py | 2 +- script/hassfest/model.py | 3 +-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/script/gen_requirements_all.py b/script/gen_requirements_all.py index 6ce97468699f0c..770507241402d6 100755 --- a/script/gen_requirements_all.py +++ b/script/gen_requirements_all.py @@ -15,7 +15,7 @@ from typing import Any from homeassistant.util.yaml.loader import load_yaml -from script.hassfest.model import Integration +from script.hassfest.model import Config, Integration # Requirements which can't be installed on all systems because they rely on additional # system packages. Requirements listed in EXCLUDED_REQUIREMENTS_ALL will be commented-out @@ -276,7 +276,9 @@ def gather_recursive_requirements( seen = set() seen.add(domain) - integration = Integration(Path(f"homeassistant/components/{domain}")) + integration = Integration( + Path(f"homeassistant/components/{domain}"), _get_hassfest_config() + ) integration.load_manifest() reqs = {x for x in integration.requirements if x not in CONSTRAINT_BASE} for dep_domain in integration.dependencies: @@ -342,7 +344,8 @@ def gather_requirements_from_manifests( errors: list[str], reqs: dict[str, list[str]] ) -> None: """Gather all of the requirements from manifests.""" - integrations = Integration.load_dir(Path("homeassistant/components")) + config = _get_hassfest_config() + integrations = Integration.load_dir(config.core_integrations_path, config) for domain in sorted(integrations): integration = integrations[domain] @@ -590,6 +593,17 @@ def main(validate: bool, ci: bool) -> int: return 0 +def _get_hassfest_config() -> Config: + """Get hassfest config.""" + return Config( + root=Path(".").absolute(), + specific_integrations=None, + action="validate", + requirements=True, + core_integrations_path=Path("homeassistant/components"), + ) + + if __name__ == "__main__": _VAL = sys.argv[-1] == "validate" _CI = sys.argv[-1] == "ci" diff --git a/script/hassfest/__main__.py b/script/hassfest/__main__.py index 60e4b4339af820..b48871b465163d 100644 --- a/script/hassfest/__main__.py +++ b/script/hassfest/__main__.py @@ -158,7 +158,7 @@ def main() -> int: integrations[integration.domain] = integration else: - integrations = Integration.load_dir(config) + integrations = Integration.load_dir(config.core_integrations_path, config) plugins += HASS_PLUGINS for plugin in plugins: diff --git a/script/hassfest/model.py b/script/hassfest/model.py index 0aab27cf1ef9f5..63e9b025ed459e 100644 --- a/script/hassfest/model.py +++ b/script/hassfest/model.py @@ -106,9 +106,8 @@ class Integration: """Represent an integration in our validator.""" @classmethod - def load_dir(cls, config: Config) -> dict[str, Integration]: + def load_dir(cls, path: pathlib.Path, config: Config) -> dict[str, Integration]: """Load all integrations in a directory.""" - path = config.core_integrations_path assert path.is_dir() integrations: dict[str, Integration] = {} for fil in path.iterdir(): From 37bfe95891913fc435795f8eea8919238662aa31 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 08:31:24 +0000 Subject: [PATCH 04/10] Fix tests --- tests/hassfest/test_requirements.py | 9 ++++++++- tests/hassfest/test_version.py | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/hassfest/test_requirements.py b/tests/hassfest/test_requirements.py index f3b008a6113ed5..433e63d904c5f1 100644 --- a/tests/hassfest/test_requirements.py +++ b/tests/hassfest/test_requirements.py @@ -4,7 +4,7 @@ import pytest -from script.hassfest.model import Integration +from script.hassfest.model import Config, Integration from script.hassfest.requirements import validate_requirements_format @@ -13,6 +13,13 @@ def integration(): """Fixture for hassfest integration model.""" return Integration( path=Path("homeassistant/components/test"), + _config=Config( + root=Path(".").absolute(), + specific_integrations=None, + action="validate", + requirements=True, + core_integrations_path=Path("homeassistant/components"), + ), _manifest={ "domain": "test", "documentation": "https://example.com", diff --git a/tests/hassfest/test_version.py b/tests/hassfest/test_version.py index bfe15018fe22de..30677356101e5b 100644 --- a/tests/hassfest/test_version.py +++ b/tests/hassfest/test_version.py @@ -1,5 +1,7 @@ """Tests for hassfest version.""" +from pathlib import Path + import pytest import voluptuous as vol @@ -7,13 +9,22 @@ CUSTOM_INTEGRATION_MANIFEST_SCHEMA, validate_version, ) -from script.hassfest.model import Integration +from script.hassfest.model import Config, Integration @pytest.fixture def integration(): """Fixture for hassfest integration model.""" - integration = Integration("") + integration = Integration( + "", + _config=Config( + root=Path(".").absolute(), + specific_integrations=None, + action="validate", + requirements=True, + core_integrations_path=Path("homeassistant/components"), + ), + ) integration._manifest = { "domain": "test", "documentation": "https://example.com", From abd29c65d3ced738299ffef9d4198a50745e3f33 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 10:21:25 +0000 Subject: [PATCH 05/10] Move hassfest docker image to core --- .github/workflows/builder.yml | 50 ++++++++++++++++++++++++++++ script/hassfest/docker/Dockerfile | 22 ++++++++++++ script/hassfest/docker/entrypoint.sh | 16 +++++++++ 3 files changed, 88 insertions(+) create mode 100644 script/hassfest/docker/Dockerfile create mode 100644 script/hassfest/docker/entrypoint.sh diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index e1f95c5c0a9bc0..79af5f61c223d6 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -482,3 +482,53 @@ jobs: export TWINE_PASSWORD="${{ secrets.TWINE_TOKEN }}" twine upload dist/* --skip-existing + + hassfest-image: + name: Build and test hassfest image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + needs: ["init", "build_base"] + env: + HASSFEST_IMAGE_NAME: ghcr.io/homeassistant/hassfest + HASSFEST_IMAGE_TAG: ghcr.io/homeassistant/hassfest:latest + steps: + - name: Checkout repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Login to GitHub Container Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Docker image + uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 + with: + context: ./script/hassfest/docker + build-args: BASE_IMAGE="ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }}" + load: true + tags: ${{ env.HASSFEST_IMAGE_TAG }} + + - name: Run hassfest against core + run: docker run --rm -v ${{ github.workspace }}/homeassistant:/github/workspace/homeassistant ${{ env.HASSFEST_IMAGE_TAG }} --core-integrations-path=/github/workspace/homeassistant/components + + - name: Push Docker image + id: push + uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 + with: + context: ./hassfest + build-args: FROM_VERSION=${{ github.event.inputs.version }} + push: true + tags: ${{ env.HASSFEST_IMAGE_TAG }} + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@6149ea5740be74af77f260b9db67e633f6b0a9a1 # v1.4.2 + with: + subject-name: ${{ env.HASSFEST_IMAGE_NAME }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/script/hassfest/docker/Dockerfile b/script/hassfest/docker/Dockerfile new file mode 100644 index 00000000000000..e07b4efea9f143 --- /dev/null +++ b/script/hassfest/docker/Dockerfile @@ -0,0 +1,22 @@ +ARG BASE_IMAGE +FROM BASE_IMAGE + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +COPY entrypoint.sh /entrypoint.sh + +RUN \ + uv pip install stdlib-list==0.10.0 \ + $(grep -e "^pipdeptree" -e "^tqdm" /usr/src/homeassistant/requirements_test.txt) \ + $(grep -e "^ruff" /usr/src/homeassistant/requirements_test_pre_commit.txt) + +WORKDIR "/github/workspace" +ENTRYPOINT ["/entrypoint.sh"] + +LABEL "name"="hassfest" +LABEL "maintainer"="Home Assistant " + +LABEL "com.github.actions.name"="hassfest" +LABEL "com.github.actions.description"="Run hassfest to validate standalone integration repositories" +LABEL "com.github.actions.icon"="terminal" +LABEL "com.github.actions.color"="gray-dark" diff --git a/script/hassfest/docker/entrypoint.sh b/script/hassfest/docker/entrypoint.sh new file mode 100644 index 00000000000000..33330f63161dff --- /dev/null +++ b/script/hassfest/docker/entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bashio +declare -a integrations +declare integration_path + +shopt -s globstar nullglob +for manifest in **/manifest.json; do + manifest_path=$(realpath "${manifest}") + integrations+=(--integration-path "${manifest_path%/*}") +done + +if [[ ${#integrations[@]} -eq 0 ]]; then + bashio::exit.nok "No integrations found!" +fi + +cd /usr/src/homeassistant +exec python3 -m script.hassfest --action validate "${integrations[@]}" "$@" \ No newline at end of file From 00da659f6890cbd53749798bd9e905ba069aa1e3 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 12:16:16 +0000 Subject: [PATCH 06/10] fix --- .github/workflows/builder.yml | 13 +++++++++---- script/hassfest/docker/Dockerfile | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index 79af5f61c223d6..4ac1f06beaee0d 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -492,9 +492,10 @@ jobs: attestations: write id-token: write needs: ["init", "build_base"] + if: github.repository_owner == 'home-assistant' env: HASSFEST_IMAGE_NAME: ghcr.io/homeassistant/hassfest - HASSFEST_IMAGE_TAG: ghcr.io/homeassistant/hassfest:latest + HASSFEST_IMAGE_TAG: ghcr.io/homeassistant/hassfest:${{ needs.init.outputs.version }} steps: - name: Checkout repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -518,15 +519,19 @@ jobs: run: docker run --rm -v ${{ github.workspace }}/homeassistant:/github/workspace/homeassistant ${{ env.HASSFEST_IMAGE_TAG }} --core-integrations-path=/github/workspace/homeassistant/components - name: Push Docker image + if: needs.init.outputs.channel != 'dev' && needs.init.outputs.publish == 'true' id: push uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 with: - context: ./hassfest - build-args: FROM_VERSION=${{ github.event.inputs.version }} + context: ./script/hassfest/docker + build-args: BASE_IMAGE="ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }}" push: true - tags: ${{ env.HASSFEST_IMAGE_TAG }} + tags: + - ${{ env.HASSFEST_IMAGE_TAG }} + - ${{ env.HASSFEST_IMAGE_NAME }}:latest - name: Generate artifact attestation + if: needs.init.outputs.channel != 'dev' && needs.init.outputs.publish == 'true' uses: actions/attest-build-provenance@6149ea5740be74af77f260b9db67e633f6b0a9a1 # v1.4.2 with: subject-name: ${{ env.HASSFEST_IMAGE_NAME }} diff --git a/script/hassfest/docker/Dockerfile b/script/hassfest/docker/Dockerfile index e07b4efea9f143..c459d68a65f5ba 100644 --- a/script/hassfest/docker/Dockerfile +++ b/script/hassfest/docker/Dockerfile @@ -1,5 +1,5 @@ -ARG BASE_IMAGE -FROM BASE_IMAGE +ARG BASE_IMAGE="ghcr.io/home-assistant/home-assistant:beta" +FROM $BASE_IMAGE SHELL ["/bin/bash", "-o", "pipefail", "-c"] From b94cd20ab8ee37c0e9f93e8a05e89fcc648692ac Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 12:19:30 +0000 Subject: [PATCH 07/10] Fix workflow --- .github/workflows/builder.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index 4ac1f06beaee0d..cd3b725ede0c89 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -526,9 +526,7 @@ jobs: context: ./script/hassfest/docker build-args: BASE_IMAGE="ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }}" push: true - tags: - - ${{ env.HASSFEST_IMAGE_TAG }} - - ${{ env.HASSFEST_IMAGE_NAME }}:latest + tags: ${{ env.HASSFEST_IMAGE_TAG }},${{ env.HASSFEST_IMAGE_NAME }}:latest - name: Generate artifact attestation if: needs.init.outputs.channel != 'dev' && needs.init.outputs.publish == 'true' From 04bb36aa1341243f131a30e2c17355641ef1374a Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 12:32:19 +0000 Subject: [PATCH 08/10] Remove " --- .github/workflows/builder.yml | 4 ++-- script/hassfest/docker/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index cd3b725ede0c89..d47e45e14e227f 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -511,7 +511,7 @@ jobs: uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 with: context: ./script/hassfest/docker - build-args: BASE_IMAGE="ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }}" + build-args: BASE_IMAGE=ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }} load: true tags: ${{ env.HASSFEST_IMAGE_TAG }} @@ -524,7 +524,7 @@ jobs: uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0 with: context: ./script/hassfest/docker - build-args: BASE_IMAGE="ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }}" + build-args: BASE_IMAGE=ghcr.io/home-assistant/amd64-homeassistant:${{ needs.init.outputs.version }} push: true tags: ${{ env.HASSFEST_IMAGE_TAG }},${{ env.HASSFEST_IMAGE_NAME }}:latest diff --git a/script/hassfest/docker/Dockerfile b/script/hassfest/docker/Dockerfile index c459d68a65f5ba..8921d92307e269 100644 --- a/script/hassfest/docker/Dockerfile +++ b/script/hassfest/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE="ghcr.io/home-assistant/home-assistant:beta" +ARG BASE_IMAGE=ghcr.io/home-assistant/home-assistant:beta FROM $BASE_IMAGE SHELL ["/bin/bash", "-o", "pipefail", "-c"] From c7991fe4c19fb4661661cfba2f2887a0611a0ba3 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 12:49:27 +0000 Subject: [PATCH 09/10] chmod --- script/hassfest/docker/entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 script/hassfest/docker/entrypoint.sh diff --git a/script/hassfest/docker/entrypoint.sh b/script/hassfest/docker/entrypoint.sh old mode 100644 new mode 100755 From 84c989b6a2cd922a793f26c1aa2d6d7eafdb65d4 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 28 Aug 2024 13:26:33 +0000 Subject: [PATCH 10/10] Typo --- .github/workflows/builder.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index d47e45e14e227f..d206f8fe8c803a 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -494,8 +494,8 @@ jobs: needs: ["init", "build_base"] if: github.repository_owner == 'home-assistant' env: - HASSFEST_IMAGE_NAME: ghcr.io/homeassistant/hassfest - HASSFEST_IMAGE_TAG: ghcr.io/homeassistant/hassfest:${{ needs.init.outputs.version }} + HASSFEST_IMAGE_NAME: ghcr.io/home-assistant/hassfest + HASSFEST_IMAGE_TAG: ghcr.io/home-assistant/hassfest:${{ needs.init.outputs.version }} steps: - name: Checkout repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7