From a918560f77f4ca47aa48da3522283355db24c4b4 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sat, 28 Dec 2024 10:50:36 +0100 Subject: [PATCH] generate build matrix from config --- .../workflows/{build.yml => build_all.yml} | 4 +- .github/workflows/build_inner.yml | 170 ------------- .github/workflows/build_one.yml | 98 +++++++ defaults.yaml | 8 +- scripts/list_build_matrix.py | 240 ++++++++++++++++++ scripts/list_package_references.py | 11 +- 6 files changed, 353 insertions(+), 178 deletions(-) rename .github/workflows/{build.yml => build_all.yml} (97%) delete mode 100644 .github/workflows/build_inner.yml create mode 100644 .github/workflows/build_one.yml create mode 100644 scripts/list_build_matrix.py diff --git a/.github/workflows/build.yml b/.github/workflows/build_all.yml similarity index 97% rename from .github/workflows/build.yml rename to .github/workflows/build_all.yml index 34b068d..9d8ef58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build_all.yml @@ -54,7 +54,7 @@ jobs: run: python -m pip install --upgrade pip conan semver - name: conan config - run: conan config install .github/config/ubuntu-22.04/conan + run: conan config install .github/config/ubuntu-24.04/conan - name: Conan export all packages run: python scripts/conan_export_all_packages.py @@ -93,7 +93,7 @@ jobs: fail-fast: false matrix: package: ${{ fromJson(needs.generate-matrix.outputs.packages) }} - uses: ./.github/workflows/build_inner.yml + uses: ./.github/workflows/build_one.yml with: conanfile: ${{ matrix.package.conanfile }} package_version: ${{ matrix.package.version }} diff --git a/.github/workflows/build_inner.yml b/.github/workflows/build_inner.yml deleted file mode 100644 index d243d8c..0000000 --- a/.github/workflows/build_inner.yml +++ /dev/null @@ -1,170 +0,0 @@ -on: - workflow_call: - inputs: - conanfile: - required: true - type: string - package_version: - required: true - type: string - build_dependencies_from_source: - required: true - type: boolean - artifactory_upload: - required: true - type: boolean - secrets: - ARTIFACTORY: - required: true - -jobs: - build: - runs-on: ${{ matrix.config.build_machine }} - name : ${{ matrix.config.host_profile }} - strategy: - fail-fast: false - matrix: - conanfile: ${{ fromJSON(format('["{0}"]', inputs.conanfile)) }} - config: - - { build_machine: ubuntu-22.04, host_profile: android-23-armv8, ndk_version: 26.3.11579264 } - - { build_machine: ubuntu-22.04, host_profile: android-23-armv7, ndk_version: 26.3.11579264 } - - { build_machine: ubuntu-22.04, host_profile: android-23-x86, ndk_version: 26.3.11579264 } - - { build_machine: ubuntu-22.04, host_profile: android-23-x86_64, ndk_version: 26.3.11579264 } - - { build_machine: ubuntu-22.04, host_profile: android-21-armv8, ndk_version: 26.3.11579264 } - - { build_machine: ubuntu-22.04, host_profile: android-21-armv7, ndk_version: 26.3.11579264 } - - { build_machine: ubuntu-22.04, host_profile: android-21-x86, ndk_version: 26.3.11579264 } - - { build_machine: ubuntu-22.04, host_profile: android-21-x86_64, ndk_version: 26.3.11579264 } - - { build_machine: macos-13, host_profile: apple-clang-14 } - - { build_machine: macos-14, host_profile: apple-armv8-clang-14 } - - { build_machine: ubuntu-24.04, host_profile: gcc-13 } - - { build_machine: ubuntu-24.04, host_profile: gcc-14 } - - { build_machine: ubuntu-24.04, host_profile: clang-18 } - - { build_machine: windows-2022, host_profile: msvc-1940 } - exclude: - # OpenLibm does not support Windows - - conanfile: 'recipes/openlibm/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - # Cairo fails to compile on Windows - # https://github.com/conan-io/conan-center-index/issues/23786 - # https://gitlab.freedesktop.org/cairo/cairo/-/issues/808 - - conanfile: 'recipes/cairo/meson/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - # No Cairo means no Poppler and no pdf2htmlEX - - conanfile: 'recipes/poppler/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - conanfile: 'recipes/pdf2htmlex/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - # Fontforge build issue on msvc-1940 - - conanfile: 'recipes/fontforge/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - # pdf2htmlEX and wvWare not yet supported on Windows, GLib isn't needed then either - - conanfile: 'recipes/glib/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - # Autotools are problematic on Windows - - conanfile: 'recipes/libgsf/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - conanfile: 'recipes/libwmf/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - conanfile: 'recipes/wvware/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - # tmpfile is needed only for Android - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: macos-13, host_profile: apple-clang-14 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: macos-14, host_profile: apple-armv8-clang-14 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: gcc-12 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: gcc-13 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: gcc-14 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: clang-16 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: clang-17 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: clang-18 } - - conanfile: 'recipes/tmpfile/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - - # poppler-data is the same package for all configurations - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-23-armv8, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-23-armv7, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-23-x86, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-23-x86_64, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-21-armv8, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-21-armv7, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-21-x86, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-22.04, host_profile: android-21-x86_64, ndk_version: 26.3.11579264 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: macos-13, host_profile: apple-clang-14 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: macos-14, host_profile: apple-armv8-clang-14 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: gcc-13 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: ubuntu-24.04, host_profile: clang-18 } - - conanfile: 'recipes/poppler-data/all/conanfile.py' - config: { build_machine: windows-2022, host_profile: msvc-1940 } - steps: - - name: checkout - uses: actions/checkout@v4 - - - name: setup python 3.12 - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: install python dependencies and setuptools (required to build GLib) - run: python -m pip install --upgrade pip conan setuptools - - - name: install NDK - if: startsWith(matrix.config.host_profile, 'android') - run: echo "y" | sudo ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install "ndk;${{ matrix.config.ndk_version }}" - - - name: conan remote - run: conan remote add odr https://artifactory.opendocument.app/artifactory/api/conan/conan - - name: conan config - run: conan config install .github/config/${{ matrix.config.build_machine }}/conan - - - name: Parse build from source option - id: build_from_source - shell: bash - run: | - if [ ${{ inputs.build_dependencies_from_source }} == true ]; then - echo argument=--build=* | tee $GITHUB_OUTPUT - else - echo argument=--build=missing | tee $GITHUB_OUTPUT - fi - - # No need to export local packages, if the previous build jobs in this chain already published to artifactory - - name: Conan export all packages - if: ${{ inputs.artifactory_upload != true }} - run: python scripts/conan_export_all_packages.py - - - name: conan install - run: conan install ${{ inputs.conanfile }} --version ${{ inputs.package_version }} --profile:host ${{ matrix.config.host_profile }} --profile:build default ${{ steps.build_from_source.outputs.argument }} - - - name: conan create - run: conan create ${{ inputs.conanfile }} --version ${{ inputs.package_version }} --profile:host ${{ matrix.config.host_profile }} --profile:build default - - - name: conan login - if: ${{ inputs.artifactory_upload }} - run: conan remote login odr admin --password '${{ secrets.ARTIFACTORY }}' - - - name: conan upload - if: ${{ inputs.artifactory_upload }} - run: conan upload "*" --check --confirm --remote odr diff --git a/.github/workflows/build_one.yml b/.github/workflows/build_one.yml new file mode 100644 index 0000000..39ddf3e --- /dev/null +++ b/.github/workflows/build_one.yml @@ -0,0 +1,98 @@ +on: + workflow_call: + inputs: + conanfile: + required: true + type: string + package_version: + required: true + type: string + build_dependencies_from_source: + required: true + type: boolean + artifactory_upload: + required: true + type: boolean + secrets: + ARTIFACTORY: + required: true + +jobs: + generate-matrix: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.list-build-matrix.outputs.matrix }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: setup python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: install python dependencies + run: python -m pip install --upgrade pip conan semver + + - name: conan config + run: conan config install .github/config/ubuntu-24.04/conan + + - name: list build matrix + id: list-build-matrix + run: python scripts/list_build_matrix.py + + build: + name : ${{ matrix.config.host_profile }} + needs: + - generate-matrix + runs-on: ${{ matrix.config.build_machine }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: setup python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: install python dependencies and setuptools (required to build GLib) + run: python -m pip install --upgrade pip conan setuptools + + - name: install NDK + if: startsWith(matrix.config.host_profile, 'android') + run: echo "y" | sudo ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install "ndk;${{ matrix.config.ndk_version }}" + + - name: conan remote + run: conan remote add odr https://artifactory.opendocument.app/artifactory/api/conan/conan + - name: conan config + run: conan config install .github/config/${{ matrix.config.build_machine }}/conan + + - name: Parse build from source option + id: build_from_source + shell: bash + run: | + if [ ${{ inputs.build_dependencies_from_source }} == true ]; then + echo argument=--build=* | tee $GITHUB_OUTPUT + else + echo argument=--build=missing | tee $GITHUB_OUTPUT + fi + + # No need to export local packages, if the previous build jobs in this chain already published to artifactory + - name: Conan export all packages + if: ${{ inputs.artifactory_upload != true }} + run: python scripts/conan_export_all_packages.py + + - name: conan install + run: conan install ${{ inputs.conanfile }} --version ${{ inputs.package_version }} --profile:host ${{ matrix.config.host_profile }} --profile:build default ${{ steps.build_from_source.outputs.argument }} + + - name: conan create + run: conan create ${{ inputs.conanfile }} --version ${{ inputs.package_version }} --profile:host ${{ matrix.config.host_profile }} --profile:build default + + - name: conan login + if: ${{ inputs.artifactory_upload }} + run: conan remote login odr admin --password '${{ secrets.ARTIFACTORY }}' + + - name: conan upload + if: ${{ inputs.artifactory_upload }} + run: conan upload "*" --check --confirm --remote odr diff --git a/defaults.yaml b/defaults.yaml index ea87713..c5b2cfd 100644 --- a/defaults.yaml +++ b/defaults.yaml @@ -23,7 +23,7 @@ profiles: include: "*" rules: - - exclude: + - type: "exclude" packages: # OpenLibm does not support Windows - "openlibm/*" @@ -44,16 +44,16 @@ rules: - "wvware/*" platforms: "windows*" # tmpfile is needed only for Android - - exclude: + - type: "exclude" packages: "tmpfile/*" platforms: - "macos*" - "ubuntu*" - "windows*" # poppler-data is the same package for all configurations - - exclude: + - type: "exclude" packages: "poppler-data/*" - - include: + - type: "include" packages: "poppler-data/*" platforms: "ubuntu-24.04" profiles: "clang-18" diff --git a/scripts/list_build_matrix.py b/scripts/list_build_matrix.py new file mode 100644 index 0000000..fc27a55 --- /dev/null +++ b/scripts/list_build_matrix.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import sys +from pathlib import Path +import fnmatch + +import yaml + +from list_package_references import item_to_list + + +script_path = Path(__file__).resolve().parent +root_path = script_path.parent +recipes_path = root_path / "recipes" + + +def get_build_matrix(package_reference, conanfile, selection_config_path): + if selection_config_path is None: + selection_config = None + else: + with open(selection_config_path) as f: + selection_config = yaml.safe_load(f) + + build_matrix = { + "conanfile": str(conanfile), + "config": [], + } + + def check_and_append(platform, config): + if selection_config is None: + build_matrix["config"].append(config) + return + + if all( + not fnmatch.fnmatch(platform, pattern) + for pattern in item_to_list( + selection_config["platforms"].get("include", "*") + ) + ): + return + if any( + fnmatch.fnmatch(platform, pattern) + for pattern in item_to_list( + selection_config["platforms"].get("exclude", []) + ) + ): + return + + profile = config["host_profile"] + if all( + not fnmatch.fnmatch(profile, pattern) + for pattern in item_to_list( + selection_config["profiles"].get("include", "*") + ) + ): + return + if any( + fnmatch.fnmatch(profile, pattern) + for pattern in item_to_list(selection_config["profiles"].get("exclude", [])) + ): + return + + rule_excluded = False + for rule in selection_config["rules"]: + if all( + not fnmatch.fnmatch(package_reference, pattern) + for pattern in item_to_list(rule.get("packages", "*")) + ): + continue + if all( + not fnmatch.fnmatch(platform, pattern) + for pattern in item_to_list(rule.get("platforms", "*")) + ): + continue + if all( + not fnmatch.fnmatch(profile, pattern) + for pattern in item_to_list(rule.get("profiles", "*")) + ): + continue + + if rule["type"] == "include": + rule_excluded = False + elif rule["type"] == "exclude": + rule_excluded = True + if rule_excluded: + return + + build_matrix["config"].append(config) + + # android + for platform in [ + "android-23-armv8", + "android-23-armv7", + "android-23-x86", + "android-23-x86_64", + "android-21-armv8", + "android-21-armv7", + "android-21-x86", + "android-21-x86_64", + ]: + check_and_append( + platform, + { + "build_machine": "ubuntu-22.04", + "host_profile": platform, + "ndk_version": "26.3.11579264", + }, + ) + + # macos + check_and_append( + "macos-13", + { + "build_machine": "macos-13", + "host_profile": "apple-clang-14", + }, + ) + check_and_append( + "macos-14", + { + "build_machine": "macos-14", + "host_profile": "apple-armv8-clang-14", + }, + ) + + # ubuntu + check_and_append( + "ubuntu-24.04", + { + "build_machine": "ubuntu-24.04", + "host_profile": "gcc-13", + }, + ) + check_and_append( + "ubuntu-24.04", + { + "build_machine": "ubuntu-24.04", + "host_profile": "gcc-14", + }, + ) + check_and_append( + "ubuntu-24.04", + { + "build_machine": "ubuntu-24.04", + "host_profile": "clang-18", + }, + ) + + # windows + check_and_append( + "windows-2022", + { + "build_machine": "windows-2022", + "host_profile": "msvc-1940", + }, + ) + + return build_matrix + + +def get_cli_args(): + parser = argparse.ArgumentParser(description="List build matrix") + parser.add_argument( + "conanfile", + type=Path, + help="Path to conanfile.py", + ) + parser.add_argument( + "version", + type=str, + help="Version of the package", + ) + parser.add_argument( + "--selection-config", + type=Path, + help="Path to selection config file", + ) + parser.add_argument( + "--github-output", + type=Path, + help="Output file for GitHub action", + ) + args = parser.parse_args() + + return args + + +def get_github_args(): + event = json.loads(os.environ.get("GITHUB_EVENT", "{}")) + inputs = event.get("inputs", {}) + + conanfile = inputs.get("conanfile") + version = inputs.get("package_version") + + selection_config = root_path / "defaults.yaml" + + github_output = Path(os.environ.get("GITHUB_OUTPUT")) + + return argparse.Namespace( + conanfile=conanfile, + version=version, + selection_config=selection_config, + github_output=github_output, + ) + + +def get_is_github(): + return bool(os.environ.get("GITHUB_ACTIONS", False)) + + +def main(): + is_github = get_is_github() + + if is_github: + args = get_github_args() + else: + args = get_cli_args() + + # TODO that does not look great + package_name = args.conanfile.parent.parent.name + package_reference = f"{package_name}/{args.version}" + + build_matrix = get_build_matrix( + package_reference, args.conanfile, args.selection_config + ) + + print(json.dumps(build_matrix, indent=4)) + + if is_github: + with open(args.github_output, "w") as out: + print(f"packages={json.dumps(build_matrix)}", file=out) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/list_package_references.py b/scripts/list_package_references.py index ba1f11d..e4e22df 100644 --- a/scripts/list_package_references.py +++ b/scripts/list_package_references.py @@ -16,6 +16,12 @@ recipes_path = root_path / "recipes" +def item_to_list(item_or_list): + if isinstance(item_or_list, list): + return item_or_list + return [item_or_list] + + def get_package_infos(): package_infos = {} @@ -77,12 +83,12 @@ def get_selected_packages(package_references, config): for package_reference in package_references: if all( not fnmatch.fnmatch(package_reference, pattern) - for pattern in package_selection.get("include", ["*"]) + for pattern in item_to_list(package_selection.get("include", "*")) ): continue if any( fnmatch.fnmatch(package_reference, pattern) - for pattern in package_selection.get("exclude", []) + for pattern in item_to_list(package_selection.get("exclude", [])) ): continue result.append(package_reference) @@ -149,6 +155,7 @@ def get_cli_args(): def get_github_args(): event = json.loads(os.environ.get("GITHUB_EVENT", "{}")) + inputs = event.get("inputs", {}) selection_config = root_path / "defaults.yaml"