From b82a4405869029dfaedcbf3a96a0c1cd7ed3edeb Mon Sep 17 00:00:00 2001 From: Adrien Cacciaguerra Date: Wed, 17 Apr 2024 11:06:05 -0400 Subject: [PATCH] feat: add CodSpeed to the project --- .github/workflows/README.md | 16 - .github/workflows/auto_approve.yml | 41 -- .github/workflows/benchmark.yml | 58 ++ .github/workflows/checkboxes.yml | 42 -- .github/workflows/clear_cache.yml | 61 --- .github/workflows/contrib_checks.yml | 178 ------- .github/workflows/contrib_rerun_py.yml | 74 --- .github/workflows/cpp_matrix_full.json | 29 - .github/workflows/labels.yml | 42 -- .github/workflows/nightly.yml | 453 ---------------- .github/workflows/on_gh_release.yml | 36 -- .github/workflows/on_pr_comment.yml | 122 ----- .github/workflows/on_pull_request.yml | 265 --------- .github/workflows/on_pull_request_contrib.yml | 34 -- .../on_pull_request_target_contrib.yml | 39 -- .github/workflows/on_push_docs.yml | 100 ---- .github/workflows/on_push_main.yml | 151 ------ .github/workflows/release.yml | 502 ------------------ .github/workflows/reusable_bench.yml | 164 ------ .../reusable_build_and_upload_rerun_c.yml | 188 ------- .../reusable_build_and_upload_rerun_cli.yml | 208 -------- .../reusable_build_and_upload_wheels.yml | 241 --------- .github/workflows/reusable_build_examples.yml | 106 ---- .github/workflows/reusable_build_js.yml | 74 --- .github/workflows/reusable_build_web.yml | 93 ---- .../reusable_bundle_and_upload_rerun_cpp.yml | 63 --- .github/workflows/reusable_checks.yml | 214 -------- .github/workflows/reusable_checks_cpp.yml | 120 ----- .github/workflows/reusable_checks_python.yml | 72 --- .github/workflows/reusable_checks_rust.yml | 104 ---- .github/workflows/reusable_deploy_docs.yml | 239 --------- .../reusable_deploy_landing_preview.yml | 64 --- .github/workflows/reusable_pip_index.yml | 64 --- .github/workflows/reusable_pr_summary.yml | 60 --- .github/workflows/reusable_publish_js.yml | 74 --- .../workflows/reusable_publish_rerun_c.yml | 75 --- .../workflows/reusable_publish_rerun_cli.yml | 66 --- .github/workflows/reusable_publish_web.yml | 140 ----- .github/workflows/reusable_publish_wheels.yml | 182 ------- .github/workflows/reusable_release_crates.yml | 43 -- .github/workflows/reusable_run_notebook.yml | 88 --- .../reusable_sync_release_assets.yml | 61 --- .github/workflows/reusable_test_wheels.yml | 204 ------- .github/workflows/reusable_track_size.yml | 198 ------- .github/workflows/reusable_update_pr_body.yml | 43 -- .../workflows/reusable_upload_examples.yml | 138 ----- .github/workflows/reusable_upload_web.yml | 139 ----- Cargo.lock | 56 +- Cargo.toml | 2 +- crates/store/re_query/benches/latest_at.rs | 8 +- 50 files changed, 107 insertions(+), 5727 deletions(-) delete mode 100644 .github/workflows/README.md delete mode 100644 .github/workflows/auto_approve.yml create mode 100644 .github/workflows/benchmark.yml delete mode 100644 .github/workflows/checkboxes.yml delete mode 100644 .github/workflows/clear_cache.yml delete mode 100644 .github/workflows/contrib_checks.yml delete mode 100644 .github/workflows/contrib_rerun_py.yml delete mode 100644 .github/workflows/cpp_matrix_full.json delete mode 100644 .github/workflows/labels.yml delete mode 100644 .github/workflows/nightly.yml delete mode 100644 .github/workflows/on_gh_release.yml delete mode 100644 .github/workflows/on_pr_comment.yml delete mode 100644 .github/workflows/on_pull_request.yml delete mode 100644 .github/workflows/on_pull_request_contrib.yml delete mode 100644 .github/workflows/on_pull_request_target_contrib.yml delete mode 100644 .github/workflows/on_push_docs.yml delete mode 100644 .github/workflows/on_push_main.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/reusable_bench.yml delete mode 100644 .github/workflows/reusable_build_and_upload_rerun_c.yml delete mode 100644 .github/workflows/reusable_build_and_upload_rerun_cli.yml delete mode 100644 .github/workflows/reusable_build_and_upload_wheels.yml delete mode 100644 .github/workflows/reusable_build_examples.yml delete mode 100644 .github/workflows/reusable_build_js.yml delete mode 100644 .github/workflows/reusable_build_web.yml delete mode 100644 .github/workflows/reusable_bundle_and_upload_rerun_cpp.yml delete mode 100644 .github/workflows/reusable_checks.yml delete mode 100644 .github/workflows/reusable_checks_cpp.yml delete mode 100644 .github/workflows/reusable_checks_python.yml delete mode 100644 .github/workflows/reusable_checks_rust.yml delete mode 100644 .github/workflows/reusable_deploy_docs.yml delete mode 100644 .github/workflows/reusable_deploy_landing_preview.yml delete mode 100644 .github/workflows/reusable_pip_index.yml delete mode 100644 .github/workflows/reusable_pr_summary.yml delete mode 100644 .github/workflows/reusable_publish_js.yml delete mode 100644 .github/workflows/reusable_publish_rerun_c.yml delete mode 100644 .github/workflows/reusable_publish_rerun_cli.yml delete mode 100644 .github/workflows/reusable_publish_web.yml delete mode 100644 .github/workflows/reusable_publish_wheels.yml delete mode 100644 .github/workflows/reusable_release_crates.yml delete mode 100644 .github/workflows/reusable_run_notebook.yml delete mode 100644 .github/workflows/reusable_sync_release_assets.yml delete mode 100644 .github/workflows/reusable_test_wheels.yml delete mode 100644 .github/workflows/reusable_track_size.yml delete mode 100644 .github/workflows/reusable_update_pr_body.yml delete mode 100644 .github/workflows/reusable_upload_examples.yml delete mode 100644 .github/workflows/reusable_upload_web.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md deleted file mode 100644 index deb5a77c0565..000000000000 --- a/.github/workflows/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Overview - -Our CI workflows make heavy usage of [Reusable Workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows). These reusable workflows can then be tested manually via the `manual_dispatch.yml` workflow. -Or integrated into CI jobs such has `on_pull_request.yml` or `on_main.yml`. - -By convention: - -- All reusable workflows start with the `reusable_` prefix. -- All workflows that are triggered via `workflow_dispatch` start with the `manual_` prefix. -- All workflows that are triggered via an event start with the `on_` prefix. - - `on_pull_request` is triggered on pull requests. - - `on_push_main` is triggered on pushes to the main branch. - -If you are going to be doing any editing of workflows, the -[VS Code extension](https://marketplace.visualstudio.com/items?itemName=cschleiden.vscode-github-actions) -for GitHub Actions is highly recommended. diff --git a/.github/workflows/auto_approve.yml b/.github/workflows/auto_approve.yml deleted file mode 100644 index 86f791f16bc8..000000000000 --- a/.github/workflows/auto_approve.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "Approve Workflow Runs" - -on: - pull_request_target: - issue_comment: - types: [created, edited] - -defaults: - run: - shell: bash - -permissions: - contents: "read" - actions: "write" - -jobs: - approve-workflow-runs: - name: "Check for approval" - runs-on: ubuntu-latest - if: | - github.event.pull_request.head.repo.owner.login != 'rerun-io' && - (github.event_name == 'pull_request_target' || github.event.issue.pull_request) - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Wait a few seconds - run: | - # Give GitHub a bit of time to synchronize everything - sleep 5s - - - name: Approve workflow runs - run: | - pixi run python scripts/ci/approve_workflow_runs.py \ - --github-token "${{ secrets.GITHUB_TOKEN }}" \ - --github-repository "rerun-io/rerun" \ - --pr-number "${{ github.event.pull_request.number || github.event.issue.number }}" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000000..037024ec26f1 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,58 @@ +name: codspeed-benchmarks + +on: + push: + branches: + - "main" # or "master" + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +env: + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all + # these incremental artifacts when running on CI. + CARGO_INCREMENTAL: "0" + +jobs: + benchmarks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup rust toolchain, cache and cargo-codspeed binary + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-codspeed + cache-base: main + + - name: Build the benchmark target(s) + # have to run one at a time since multiple packages is not yet supported + # `--all-features` is not supported + run: | + # cargo codspeed build -p re_int_histogram # not used in `.github/workflows/reusable_bench.yml` + # cargo codspeed build -p re_log_types # not used in `.github/workflows/reusable_bench.yml` + # cargo codspeed build -p re_query_cache # not used in `.github/workflows/reusable_bench.yml` + # cargo codspeed build -p re_query_cache2 # not used in `.github/workflows/reusable_bench.yml` + # cargo codspeed build -p re_space_view_spatial # not used in `.github/workflows/reusable_bench.yml` + # cargo codspeed build -p re_types_core # not used in `.github/workflows/reusable_bench.yml` + + # cargo codspeed build -p re_entity_db # no benchmarks defined in this package + + cargo codspeed build -p re_log_encoding --features=decoder,encoder + cargo codspeed build -p re_query -p re_tuid + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: cargo codspeed run diff --git a/.github/workflows/checkboxes.yml b/.github/workflows/checkboxes.yml deleted file mode 100644 index 2f4890ae9c4c..000000000000 --- a/.github/workflows/checkboxes.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Checks that all checkboxes in a PR are checked - -name: Pull Request Checkboxes - -on: - pull_request_target: - types: - - opened - - synchronize - - reopened - - edited - -concurrency: - group: ${{ github.event.pull_request.number }}-pr-checkboxes - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - pull-requests: "write" - -jobs: - pr-checkboxes: - name: Check PR checkboxes - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Check PR checkboxes - run: | - pixi run ./scripts/ci/check_pr_checkboxes.py \ - --github-token ${{ secrets.GITHUB_TOKEN }} \ - --github-repository ${{ github.repository }} \ - --pr-number ${{ github.event.pull_request.number }} diff --git a/.github/workflows/clear_cache.yml b/.github/workflows/clear_cache.yml deleted file mode 100644 index fa081e73609b..000000000000 --- a/.github/workflows/clear_cache.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Clear cache - -on: - workflow_dispatch: - -permissions: - actions: write - -jobs: - clear-cache: - runs-on: ubuntu-latest - steps: - - name: Clear cache - uses: actions/github-script@v6 - with: - script: | - const opts = { - owner: context.repo.owner, - repo: context.repo.repo, - }; - - const cache_list = await github.rest.actions.getActionsCacheList({ - per_page: 1, - ...opts, - }); - const per_page = 100; - const pages = Math.ceil(cache_list.data.total_count / per_page); - - const total_count = cache_list.data.total_count; - let deleted = 0; - const _id = setInterval(() => { - console.log(`${deleted}/${total_count}`); - }, 500); - - let promises = []; - for (let page = 1; page <= pages; page++) { - const cache_page = await github.rest.actions.getActionsCacheList({ - per_page, - page, - ...opts, - }); - for (const cache of cache_page.data.actions_caches) { - promises.push( - github.rest.actions - .deleteActionsCacheById({ - cache_id: cache.id, - ...opts, - }) - .then(() => (deleted += 1)) - ); - } - await Promise.all(promises); - promises = []; - - // wait 60 seconds every 4 pages (secondary rate limit) - if (page % 4 === 0) { - await new Promise(f => setTimeout(f, 60_000)); - } - } - - clearInterval(_id); diff --git a/.github/workflows/contrib_checks.yml b/.github/workflows/contrib_checks.yml deleted file mode 100644 index 8476df0b0731..000000000000 --- a/.github/workflows/contrib_checks.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: "Checks: Lints, Tests, Docs" - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PR_NUMBER: - required: false - type: string - default: "" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-checks - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Do *not* use sscache since on contributor ci we don't have access to the gcloud stored cache. - #RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - -jobs: - py-lints: - name: Python lints (ruff, mypy, …) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Python format check - run: pixi run py-fmt-check - - - name: Lint Python - run: pixi run py-lint - - py-test-docs: - name: Test Python Docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install Python dependencies - shell: bash - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-doc.txt - - - name: Build via mkdocs - shell: bash - run: | - mkdocs build --strict -f rerun_py/mkdocs.yml - - no-codegen-changes: - name: Check if running codegen would produce any changes - runs-on: ubuntu-latest-16-cores - steps: - # Note: We explicitly don't override `ref` here. We need to see if changes would be made - # in a context where we have merged with main. Otherwise we might miss changes such as one - # PR introduces a new type and another PR changes the codegen. - - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Codegen check - run: pixi run codegen --force --check - - rs-lints: - name: Rust lints (fmt, check, clippy, tests, doc) - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Rust checks & tests - run: pixi run rs-check --skip individual_crates - - misc-rerun-lints: - name: Rerun lints - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Rerun lints - run: pixi run lint-rerun - - - name: Toml format check - run: pixi run toml-fmt-check - - - name: Check for too large files - run: pixi run check-large-files - - - name: Check Python example thumbnails - run: pixi run ./scripts/ci/thumbnails.py check - - spell-check: - name: Spell Check - runs-on: ubuntu-latest - steps: - - name: Checkout Actions Repository - uses: actions/checkout@v4 - - - name: Check spelling of entire workspace - uses: crate-ci/typos@v1.18.0 - - cpp-formatting: - name: C++ formatting check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Run clang format on all relevant files - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: "16" - # Only check c/cpp/h/hpp (default checks also .proto and others) - include-regex: ^.*\.(c|cpp|h|hpp)$ - - cpp-tests: - name: C++ tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - environments: cpp - - # TODO(emilk): make this work somehow. Right now this just results in - # > Compiler: GNU 12.3.0 (/__w/rerun/rerun/.pixi/env/bin/x86_64-conda-linux-gnu-c++) - # 😭 - # - name: Build and run C++ tests with clang++ - # run: | - # pixi run -e cpp cpp-clean - # RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=clang++ pixi run -e cpp cpp-build-all - # RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=clang++ pixi run -e cpp cpp-test - - - name: Build and run C++ tests with g++ - run: | - pixi run -e cpp cpp-clean - RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=g++ pixi run -e cpp cpp-build-all - RERUN_WERROR=ON RERUN_USE_ASAN=ON CXX=g++ pixi run -e cpp cpp-test diff --git a/.github/workflows/contrib_rerun_py.yml b/.github/workflows/contrib_rerun_py.yml deleted file mode 100644 index ab742d368e9a..000000000000 --- a/.github/workflows/contrib_rerun_py.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Reusable Build and Test Wheels - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - MATURIN_FEATURE_FLAGS: - required: false - type: string - default: "--no-default-features --features pypi" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-build-wheels - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - - # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac - # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - RUSTFLAGS: --cfg=web_sys_unstable_apis - - RUSTDOCFLAGS: --deny warnings - - # Do *not* use sscache since on contributor ci we don't have access to the gcloud stored cache. - #RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - -jobs: - build-wheels: - name: Build Wheels - runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.14.0 - steps: - - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - environments: wheel-test-min - - - name: Run e2e test - run: pixi run -e wheel-test-min RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - - - name: Run tests/roundtrips.py - # --release so we can inherit from some of the artifacts that maturin has just built before - # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow… - # --no-py-build because rerun-sdk is already built and installed - run: | - pixi run -e wheel-test-min RUST_LOG=debug tests/roundtrips.py --release --target x86_64-unknown-linux-gnu --no-py-build - - - name: Run docs/snippets/compare_snippet_output.py - # --release so we can inherit from some of the artifacts that maturin has just built before - # --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache… even though this is the target anyhow… - # --no-py-build because rerun-sdk is already built and installed - run: | - pixi run -e wheel-test-min RUST_LOG=debug docs/snippets/compare_snippet_output.py --release --target x86_64-unknown-linux-gnu --no-py-build diff --git a/.github/workflows/cpp_matrix_full.json b/.github/workflows/cpp_matrix_full.json deleted file mode 100644 index 480a153e1074..000000000000 --- a/.github/workflows/cpp_matrix_full.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "include": [ - { - "name": "Linux x64, C++17", - "runs_on": "ubuntu-latest-16-cores", - "cache_key": "build-linux", - "extra_env_vars": "RERUN_USE_ASAN=1 RERUN_SET_CXX_VERSION=17", - "additional_commands": "pixi run -e cpp cpp-docs" - }, - { - "name": "Linux x64, C++20", - "runs_on": "ubuntu-latest-16-cores", - "cache_key": "build-linux", - "extra_env_vars": "RERUN_USE_ASAN=1 RERUN_SET_CXX_VERSION=20" - }, - { - "name": "Windows x64", - "runs_on": "windows-latest-8-cores", - "cache_key": "build-windows", - "extra_env_vars": "" - }, - { - "name": "Mac aarch64", - "runs_on": "macos-latest-large", - "cache_key": "build-macos-arm64", - "extra_env_vars": "RERUN_USE_ASAN=1" - } - ] -} diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml deleted file mode 100644 index 7f448548c843..000000000000 --- a/.github/workflows/labels.yml +++ /dev/null @@ -1,42 +0,0 @@ -# https://github.com/marketplace/actions/require-labels -# Check for existence of labels -# See all our labels at https://github.com/rerun-io/rerun/issues/labels - -name: Pull Request Labels - -on: - pull_request_target: - types: - - opened - - synchronize - - reopened - - labeled - - unlabeled - -# No permissions needed here -# permissions: - -jobs: - label: - runs-on: ubuntu-latest - steps: - - name: Check for a "do-not-merge" label - uses: mheap/github-action-required-labels@v3 - with: - mode: exactly - count: 0 - labels: "do-not-merge" - - - name: Require label "include in changelog" or "exclude from changelog" - uses: mheap/github-action-required-labels@v3 - with: - mode: minimum - count: 1 - labels: "exclude from changelog, include in changelog" - - - name: Require at least one label - uses: mheap/github-action-required-labels@v3 - with: - mode: minimum - count: 1 - labels: "📊 analytics, 🟦 blueprint, 🪳 bug, 🌊 C++ API, CLI, codegen/idl, 🧑‍💻 dev experience, dependencies, 📖 documentation, 💬 discussion, examples, exclude from changelog, 🪵 Log-API, 📉 performance, 🐍 Python API, ⛃ re_datastore, 📺 re_viewer, 🔺 re_renderer, 🚜 refactor, ⛴ release, 🦀 Rust API, 🔨 testing, ui, 🕸️ web" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index ff7ef9cf5e83..000000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,453 +0,0 @@ -name: Nightly - -on: - workflow_dispatch: - schedule: - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule - # 12:15 UTC, every day - - cron: "15 12 * * *" - -defaults: - run: - shell: bash - -permissions: - contents: "write" - id-token: "write" - deployments: "write" - -jobs: - checks: - name: Checks - uses: ./.github/workflows/reusable_checks.yml - with: - CONCURRENCY: nightly - CHANNEL: nightly - secrets: inherit - - checks-cpp: - name: Checks - uses: ./.github/workflows/reusable_checks_cpp.yml - with: - CONCURRENCY: nightly - CHANNEL: nightly - secrets: inherit - - checks-rust: - name: Checks - uses: ./.github/workflows/reusable_checks_rust.yml - with: - CONCURRENCY: nightly - CHANNEL: nightly - secrets: inherit - - checks-python: - name: Checks - uses: ./.github/workflows/reusable_checks_python.yml - with: - CONCURRENCY: nightly - secrets: inherit - - # Check that a CLEAN container with just `cargo` on it can build rerun: - clean-build: - name: cargo build on clean container - strategy: - matrix: - os: [ubuntu-latest-16-cores, macos-latest, windows-latest-8-cores] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: 1.76.0 - - - run: cargo build -p rerun - - build-web: - name: "Build web viewer" - uses: ./.github/workflows/reusable_build_web.yml - with: - CONCURRENCY: nightly - CHANNEL: nightly - secrets: inherit - - upload-web: - name: "Upload Web" - needs: [build-web] - uses: ./.github/workflows/reusable_upload_web.yml - with: - CONCURRENCY: nightly - NIGHTLY: true - secrets: inherit - - # ----------------------------------------------------------------------------------- - # Build rerun_c library binaries: - - build-rerun_c-and-upload-linux-arm64: - needs: [checks] - name: "Linux-Arm64: Build & Upload rerun_c" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: nightly-linux-arm64 - PLATFORM: linux-arm64 - secrets: inherit - - build-rerun_c-and-upload-linux-x64: - needs: [checks] - name: "Linux-x64: Build & Upload rerun_c" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: nightly-linux-x64 - PLATFORM: linux-x64 - secrets: inherit - - build-rerun_c-and-upload-macos-x64: - needs: [checks] - name: "Mac-Intel: Build & Upload rerun_c" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: nightly-macos-x64 - PLATFORM: macos-x64 - secrets: inherit - - build-rerun_c-and-upload-macos-arm64: - needs: [checks] - name: "Mac-Arm64: Build & Upload rerun_c" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: nightly-macos-arm64 - PLATFORM: macos-arm64 - secrets: inherit - - build-rerun_c-and-upload-windows-x64: - needs: [checks] - name: "Windows-x64: Build & Upload rerun_c" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: nightly-windows-x64 - PLATFORM: windows-x64 - secrets: inherit - - # ----------------------------------------------------------------------------------- - # Build rerun-cli (rerun binaries): - - build-rerun-cli-and-upload-linux-arm64: - needs: [checks] - name: "Linux-arm64: Build & Upload rerun-cli" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: nightly-linux-arm64 - PLATFORM: linux-arm64 - secrets: inherit - - build-rerun-cli-and-upload-linux-x64: - needs: [checks] - name: "Linux-x64: Build & Upload rerun-cli" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: nightly-linux-x64 - PLATFORM: linux-x64 - secrets: inherit - - build-rerun-cli-and-upload-macos-x64: - needs: [checks] - name: "Mac-x64: Build & Upload rerun-cli" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: nightly-macos-x64 - PLATFORM: macos-x64 - secrets: inherit - - build-rerun-cli-and-upload-macos-arm64: - needs: [checks] - name: "Mac-arm64: Build & Upload rerun-cli" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: nightly-macos-arm64 - PLATFORM: macos-arm64 - secrets: inherit - - build-rerun-cli-and-upload-windows-x64: - needs: [checks] - name: "Windows-x64: Build & Upload rerun-cli" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: nightly-windows-x64 - PLATFORM: windows-x64 - secrets: inherit - - # --------------------------------------------------------------------------- - # Build wheels: - - build-wheel-linux-arm64: - needs: [checks, build-rerun-cli-and-upload-linux-arm64] - name: "Linux-arm64: Build & Upload Wheels" - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: nightly-linux-arm64 - PLATFORM: linux-arm64 - WHEEL_ARTIFACT_NAME: linux-arm64-wheel - MODE: "pypi" - secrets: inherit - - build-wheel-linux-x64: - needs: [checks, build-rerun-cli-and-upload-linux-x64] - name: "Linux-x64: Build & Upload Wheels" - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: nightly-linux-x64 - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: linux-x64-wheel - MODE: "pypi" - secrets: inherit - - build-wheel-macos-arm64: - needs: [checks, build-rerun-cli-and-upload-macos-arm64] - name: "Macos-arm64: Build & Upload Wheels" - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: nightly-macos-arm64 - PLATFORM: macos-arm64 - WHEEL_ARTIFACT_NAME: macos-arm64-wheel - MODE: "pypi" - secrets: inherit - - build-wheel-macos-x64: - needs: [checks, build-rerun-cli-and-upload-macos-x64] - name: "Macos-x64: Build & Upload Wheels" - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: nightly-macos-x64 - PLATFORM: macos-x64 - WHEEL_ARTIFACT_NAME: "macos-x64-wheel" - MODE: "pypi" - secrets: inherit - - build-wheel-windows-x64: - needs: [checks, build-rerun-cli-and-upload-windows-x64] - name: "Windows-x64: Build & Upload Wheels" - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: nightly-windows-x64 - PLATFORM: windows-x64 - WHEEL_ARTIFACT_NAME: windows-x64-wheel - MODE: "pypi" - secrets: inherit - - # --------------------------------------------------------------------------- - # Test wheels: - - test-wheel-linux-arm64: - needs: [checks, build-wheel-linux-arm64] - name: "linux-arm64: Test Wheels" - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: nightly-linux-arm64 - PLATFORM: linux-arm64 - WHEEL_ARTIFACT_NAME: linux-arm64-wheel - secrets: inherit - - test-wheel-linux-x64: - needs: [checks, build-wheel-linux-x64] - name: "Linux-x64: Test Wheels" - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: nightly-linux-x64 - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: linux-x64-wheel - secrets: inherit - - test-wheel-macos-arm64: - needs: [checks, build-wheel-macos-arm64] - name: "macos-arm64: Test Wheels" - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: nightly-macos-arm64 - PLATFORM: macos-arm64 - WHEEL_ARTIFACT_NAME: macos-arm64-wheel - secrets: inherit - - test-wheel-macos-x64: - needs: [checks, build-wheel-macos-x64] - name: "macos-x64: Test Wheels" - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: nightly-macos-x64 - PLATFORM: macos-x64 - WHEEL_ARTIFACT_NAME: macos-x64-wheel - secrets: inherit - - test-wheel-windows-x64: - needs: [checks, build-wheel-windows-x64] - name: "Windows-x64: Test Wheels" - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: nightly-windows-x64 - PLATFORM: windows-x64 - WHEEL_ARTIFACT_NAME: windows-x64-wheel - secrets: inherit - - # --------------------------------------------------------------------------- - - run-notebook: - name: "Run Notebook" - needs: [build-wheel-linux-x64] - uses: ./.github/workflows/reusable_run_notebook.yml - with: - CONCURRENCY: nightly - WHEEL_ARTIFACT_NAME: linux-x64-wheel - secrets: inherit - - build-examples: - name: "Build Examples" - needs: [build-wheel-linux-x64] - uses: ./.github/workflows/reusable_build_examples.yml - with: - CONCURRENCY: nightly - CHANNEL: nightly - WHEEL_ARTIFACT_NAME: linux-x64-wheel - secrets: inherit - - upload-examples: - name: "Upload Examples" - needs: [build-examples] - uses: ./.github/workflows/reusable_upload_examples.yml - with: - CONCURRENCY: nightly - NIGHTLY: true - secrets: inherit - - benches: - name: Benchmarks - uses: ./.github/workflows/reusable_bench.yml - with: - CONCURRENCY: nightly - SAVE_BENCHES: true - BENCH_NAME: main # We currently only run benches nightly, but we used to run them on main - COMPARE_TO: main # We currently only run benches nightly, but we used to run them on main - secrets: inherit - - # -------------------------------------------------------------------------- - # Release: - - generate-pip-index: - name: "Generate Pip Index" - needs: - [ - build-wheel-linux-arm64, - build-wheel-linux-x64, - build-wheel-macos-arm64, - build-wheel-macos-x64, - build-wheel-windows-x64, - ] - uses: ./.github/workflows/reusable_pip_index.yml - with: - CONCURRENCY: nightly - secrets: inherit - - bundle-and-upload-rerun_cpp: - name: "Bundle and upload rerun_cpp_sdk.zip" - needs: - [ - build-rerun_c-and-upload-linux-arm64, - build-rerun_c-and-upload-linux-x64, - build-rerun_c-and-upload-macos-arm64, - build-rerun_c-and-upload-macos-x64, - build-rerun_c-and-upload-windows-x64, - ] - uses: ./.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml - with: - CONCURRENCY: nightly - secrets: inherit - - pre-release: - name: Pre Release - concurrency: nightly - needs: - [ - build-rerun-cli-and-upload-linux-arm64, - build-rerun-cli-and-upload-linux-x64, - build-rerun-cli-and-upload-macos-arm64, - build-rerun-cli-and-upload-macos-x64, - build-rerun-cli-and-upload-windows-x64, - build-rerun_c-and-upload-linux-arm64, - build-rerun_c-and-upload-linux-x64, - build-rerun_c-and-upload-macos-arm64, - build-rerun_c-and-upload-macos-x64, - build-rerun_c-and-upload-windows-x64, - bundle-and-upload-rerun_cpp, - generate-pip-index, - upload-web, - ] - runs-on: "ubuntu-latest" - steps: - - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV - - # First delete the old prerelease. If we don't do this, we don't get things like - # proper source-archives and changelog info. - # https://github.com/dev-drprasad/delete-tag-and-release - - uses: dev-drprasad/delete-tag-and-release@v0.2.1 - with: - tag_name: prerelease - delete_release: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Create the actual prerelease - # https://github.com/ncipollo/release-action - - name: GitHub Release - uses: ncipollo/release-action@v1.12.0 - with: - body: | - This is a prerelease. It is not intended for production use. - Please report any issues you find. - - ## Example Hosted App - https://rerun.io/viewer/commit/${{ env.SHORT_SHA }} - - ## Wheels can be installed with: - ``` - pip install --pre --no-index -f https://build.rerun.io/commit/${{ env.SHORT_SHA }}/wheels --upgrade rerun-sdk - ``` - or - ``` - pip install --pre --no-index -f https://github.com/rerun-io/rerun/releases/download/prerelease --upgrade rerun-sdk - ``` - - ## CMake fetch-content for C++ SDK - ``` - include(FetchContent) - FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/${{ env.SHORT_SHA }}/rerun_cpp_sdk.zip) - FetchContent_MakeAvailable(rerun_sdk) - ``` - or - ``` - include(FetchContent) - FetchContent_Declare(rerun_sdk URL https://github.com/rerun-io/rerun/releases/download/prerelease/rerun_cpp_sdk.zip) - FetchContent_MakeAvailable(rerun_sdk) - ``` - - prerelease: true - # Be explicit about the commit we're releasing/tagging. - # Otherwise it can happen that there's a discrepancy between the tag and the commit for which we uploaded & linked artifacts. - # It seems to otherwise use the latest commit for the tag. From the actions's docs: - # > If the tag of the release you are creating does not yet exist, you should set both the tag and commit action inputs. - # We just deleted the previous tag, so this is the case! - commit: ${{github.sha}} - name: "Development Build" - tag: "prerelease" - token: ${{ secrets.GITHUB_TOKEN }} - generateReleaseNotes: false - allowUpdates: true - removeArtifacts: true - replacesArtifacts: true - - sync-release-assets: - needs: [pre-release] - name: "Sync pre-release assets & build.rerun.io" - uses: ./.github/workflows/reusable_sync_release_assets.yml - with: - CONCURRENCY: nightly - RELEASE_VERSION: prerelease - secrets: inherit diff --git a/.github/workflows/on_gh_release.yml b/.github/workflows/on_gh_release.yml deleted file mode 100644 index 837e657928ad..000000000000 --- a/.github/workflows/on_gh_release.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: "GitHub Release" - -on: - # Triggers when the `Publish release` button is pressed - release: - types: [published] - - # Manual trigger with `tag` input - workflow_dispatch: - inputs: - tag_name: - description: "Release tag" - type: string - required: true - -concurrency: - group: "release-${{ github.event.release.tag_name }}" - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - # required for updating the release - contents: write - id-token: write - -jobs: - sync-release-assets: - name: "Sync Release Assets" - uses: ./.github/workflows/reusable_sync_release_assets.yml - with: - CONCURRENCY: "${{ github.event.release.tag_name || inputs.tag_name }}" - RELEASE_VERSION: "${{ github.event.release.tag_name || inputs.tag_name }}" - secrets: inherit diff --git a/.github/workflows/on_pr_comment.yml b/.github/workflows/on_pr_comment.yml deleted file mode 100644 index 398982e01584..000000000000 --- a/.github/workflows/on_pr_comment.yml +++ /dev/null @@ -1,122 +0,0 @@ -# This workflow is triggered on any PR comment, and tries to find a `@rerun-bot` mention in it. -# If the mention is a command, such as `@rerun-bot full-check`, then it runs the command. -# -# Available commands: -# full-check Triggers a run of `on_push_main.yml` on the PR. - -name: "PR Comment" - -on: - issue_comment: - types: [created] - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - pull-requests: "write" - -jobs: - parse-command: - if: | - contains(github.event.comment.html_url, '/pull/') && - contains(github.event.comment.body, '@rerun-bot') && - contains( - fromJSON('["COLLABORATOR","MEMBER","OWNER"]'), - github.event.comment.author_association - ) - runs-on: ubuntu-latest - outputs: - command: ${{ steps.parse.outputs.command }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Parse comment - id: parse - env: - GITHUB_COMMENT_BODY: "${{ github.event.comment.body }}" - run: python ./scripts/ci/parse_bot_pr_comment.py - - full-check: - needs: [parse-command] - if: needs.parse-command.outputs.command == 'full-check' - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Dispatch main workflow - id: dispatch - env: - # NOTE: This uses `RERUN_BOT_TOKEN` instead of `GITHUB_TOKEN`, - # otherwise the recursive workflow protection prevents us from - # starting the `on_push_main` run. - # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow - GH_TOKEN: ${{ secrets.RERUN_BOT_TOKEN }} - run: | - get_latest_workflow_run () { - local workflow_name=$1 - local ref_name=$2 - local created_after=$3 - echo $( - # https://cli.github.com/manual/gh_run_list - # https://docs.github.com/en/search-github/getting-started-with-searching-on-github/understanding-the-search-syntax#query-for-dates - gh run list \ - --workflow $workflow_name \ - --event workflow_dispatch \ - --branch $ref_name \ - --created ">$created_after" \ - --json databaseId - ) - } - - dispatch_workflow () { - local workflow_name=$1 - local ref_name=$2 - local inputs=$3 - # https://cli.github.com/manual/gh_workflow_run - echo $inputs | gh workflow run $workflow_name --ref "refs/heads/$ref_name" --json - } - - workflow_name='on_push_main.yml' - ref_name=$(gh pr view ${{ github.event.issue.number }} --json headRefName | jq -r '.headRefName') - inputs='{"CONCURRENCY":"pr-${{ github.event.issue.number }}-full-check"}' - now=$(date --utc --iso-8601=seconds) - - echo "Dispatching workflow $workflow_name on branch $ref_name" - dispatch_workflow $workflow_name $ref_name $inputs - - # `gh workflow run` does NOT return the ID. - # In fact, it returns absolutely nothing: https://github.com/cli/cli/issues/4001 - # Instead, we have to wait for the workflow to start, and hope that nobody has - # started a workflow in parallel with us on the same branch. - - echo "Fetching workflow run id…" - run_info=$(get_latest_workflow_run $workflow_name $ref_name $now) - echo $run_info - run_id=$(echo $run_info | jq -r '.[0].databaseId') - while [ $run_id == 'null' ] - do - run_info=$(get_latest_workflow_run $workflow_name $ref_name $now) - echo $run_info - run_id=$(echo $run_info | jq -r '.[0].databaseId') - sleep 1 - done - echo "Workflow run: https://github.com/rerun-io/rerun/actions/runs/$run_id" - - echo "workflow_run_url=https://github.com/rerun-io/rerun/actions/runs/$run_id" >> "$GITHUB_OUTPUT" - - - name: Create PR comment - # https://github.com/mshick/add-pr-comment - uses: mshick/add-pr-comment@v2.8.2 - with: - # We use `GITHUB_TOKEN` here so there is no chance that we'll trigger another run of this workflow. - # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow - repo-token: ${{ secrets.GITHUB_TOKEN }} - message-id: "pr-${{ github.event.issue.number }}-${{ github.run_id }}" - message: | - Started a full build: ${{ steps.dispatch.outputs.workflow_run_url }} diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml deleted file mode 100644 index 17ae1f797f4d..000000000000 --- a/.github/workflows/on_pull_request.yml +++ /dev/null @@ -1,265 +0,0 @@ -# Jobs that only run for developers on the `rerun` team. -# We have to ensure that these jobs _only_ run for PRs inside the `rerun-io` organization -# this is done using the following check, added to every job: -# if: github.event.pull_request.head.repo.owner.login == 'rerun-io' -# (unfortunately this does not work on the trigger or the entire `jobs` category) - -name: Pull-Request - -on: - pull_request: - types: - - opened - - synchronize - -permissions: write-all - -# These jobs use fairly short names as they are a prefix in the display hierarchy -jobs: - checks: - name: Checks - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - uses: ./.github/workflows/reusable_checks.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - CHANNEL: pr - secrets: inherit - - cpp-paths-filter: - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - runs-on: ubuntu-latest - outputs: - cpp_changes: ${{ steps.filter.outputs.cpp_changes }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - cpp_changes: - # - .github/**/*.yml - this is tempting, but leads to constant rebuilds - - pixi.lock # maybe out build commands have changed - - pixi.toml # maybe out build commands have changed - - scripts/ci/* - - '**/*.hpp' - - '**/*.cpp' - - '**/CMakeLists.txt' - - '**/*cmake' - - docs-paths-filter: - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - runs-on: ubuntu-latest - outputs: - docs_changes: ${{ steps.filter.outputs.docs_changes }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - docs_changes: - # - .github/**/*.yml - this is tempting, but leads to constant rebuilds - - pixi.lock # maybe out build commands have changed - - pixi.toml # maybe out build commands have changed - - scripts/ci/* - - 'docs/content/**/*.md' - - 'examples/**/*.md' - - 'examples/manifest.toml' - - python-paths-filter: - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - runs-on: ubuntu-latest - outputs: - python_changes: ${{ steps.filter.outputs.python_changes }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - python_changes: - # - .github/**/*.yml - this is tempting, but leads to constant rebuilds - - pixi.lock # maybe out build commands have changed - - pixi.toml # maybe out build commands have changed - - scripts/ci/* - - '**/*.py' - - '**/requirements.txt' - - '**/pyproject.toml' - - rust-paths-filter: - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - runs-on: ubuntu-latest - outputs: - rust_changes: ${{ steps.filter.outputs.rust_changes }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - rust_changes: - # - .github/**/*.yml - this is tempting, but leads to constant rebuilds - - Cargo.lock - - pixi.lock # maybe out build commands have changed - - pixi.toml # maybe out build commands have changed - - scripts/ci/* - - "**/*.rs" - - "**/*.toml" - - web-paths-filter: - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - runs-on: ubuntu-latest - outputs: - web_changes: ${{ steps.filter.outputs.web_changes }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - web_changes: - # - .github/**/*.yml - this is tempting, but leads to constant rebuilds - - Cargo.lock - - pixi.lock # maybe out build commands have changed - - pixi.toml # maybe out build commands have changed - - scripts/ci/* - - "**/*.html" - - "**/*.js" - - "**/*.json" - - "**/*.rs" - - "**/*.toml" - - "**/yarn.lock" - - "crates/viewer/re_ui/data/**" - - rust-checks: - name: "Rust Checks" - needs: rust-paths-filter - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && needs.rust-paths-filter.outputs.rust_changes == 'true' - uses: ./.github/workflows/reusable_checks_rust.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - CHANNEL: pr - secrets: inherit - - python-checks: - name: "Python Checks" - needs: python-paths-filter - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && needs.python-paths-filter.outputs.python_changes == 'true' - uses: ./.github/workflows/reusable_checks_python.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - secrets: inherit - - cpp-tests: - name: "C++ tests" - needs: cpp-paths-filter - if: needs.cpp-paths-filter.outputs.cpp_changes == 'true' - uses: ./.github/workflows/reusable_checks_cpp.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - CHANNEL: pr - secrets: inherit - - min-cli-build: - name: "Minimum CLI Build" - needs: [python-paths-filter, rust-paths-filter] - if: needs.python-paths-filter.outputs.python_changes == 'true' || needs.rust-paths-filter.outputs.rust_changes == 'true' - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - PLATFORM: linux-x64 - secrets: inherit - - # Build and test a single wheel to limit CI cost. We use linux-x64 because it's fast. linux-arm64 would also be a good - # choice, but reusable_test_wheels.yml is broken for that target (https://github.com/rerun-io/rerun/issues/5525) - min-wheel-build: - name: "Minimum Wheel Build" - needs: [min-cli-build, python-paths-filter, rust-paths-filter] - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && (needs.python-paths-filter.outputs.python_changes == 'true' || needs.rust-paths-filter.outputs.rust_changes == 'true') - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - MODE: "pr" - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: "linux-x64-wheel-fast" - secrets: inherit - - min-wheel-test: - name: "Minimum Wheel Test" - needs: [min-wheel-build, python-paths-filter, rust-paths-filter] - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && (needs.python-paths-filter.outputs.python_changes == 'true' || needs.rust-paths-filter.outputs.rust_changes == 'true') - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: "linux-x64-wheel-fast" - FAST: true - secrets: inherit - - build-js: - name: "Build rerun_js" - needs: [web-paths-filter] - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && needs.web-paths-filter.outputs.web_changes == 'true' - uses: ./.github/workflows/reusable_build_js.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - secrets: inherit - - build-web: - name: "Build web viewer" - needs: [web-paths-filter] - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && needs.web-paths-filter.outputs.web_changes == 'true' - uses: ./.github/workflows/reusable_build_web.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - CHANNEL: main - secrets: inherit - - upload-web: - name: "Upload Web" - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - needs: [build-web] - uses: ./.github/workflows/reusable_upload_web.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - PR_NUMBER: ${{ github.event.pull_request.number }} - secrets: inherit - - save-pr-summary: - name: "Save PR Summary" - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - needs: [upload-web] - uses: ./.github/workflows/reusable_pr_summary.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - PR_NUMBER: ${{ github.event.pull_request.number }} - secrets: inherit - - update-pr-body: - name: "Update PR Body" - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - uses: ./.github/workflows/reusable_update_pr_body.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - PR_NUMBER: ${{ github.event.pull_request.number }} - secrets: inherit - - deploy-landing-preview: - name: "Deploy Landing Preview" - if: github.event.pull_request.head.repo.owner.login == 'rerun-io' && needs.docs-paths-filter.outputs.docs_changes == 'true' - needs: docs-paths-filter - uses: ./.github/workflows/reusable_deploy_landing_preview.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - PR_NUMBER: ${{ github.event.pull_request.number }} - secrets: inherit diff --git a/.github/workflows/on_pull_request_contrib.yml b/.github/workflows/on_pull_request_contrib.yml deleted file mode 100644 index 4569aa39b12d..000000000000 --- a/.github/workflows/on_pull_request_contrib.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Jobs that only run for external contributors. -# These have to be carefully sanitized, we don't want to leak secrets. -# - We can't use caching, outside of really rare scenarios, because our caching largely depends on GCS -# - We have to ensure that these jobs _only_ run for PRs outside of the `rerun-io` organization -# this is done using the following check, added to every job: -# if: github.event.pull_request.head.repo.owner.login != 'rerun-io' - -name: Pull-Request (Contrib) - -on: - pull_request: - types: - - opened - - synchronize - -permissions: - contents: "read" - -jobs: - checks: - name: "Checks" - if: github.event.pull_request.head.repo.owner.login != 'rerun-io' - uses: ./.github/workflows/contrib_checks.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - PR_NUMBER: ${{ github.event.pull_request.number }} - - python: - name: "Python" - if: github.event.pull_request.head.repo.owner.login != 'rerun-io' - uses: ./.github/workflows/contrib_rerun_py.yml - with: - CONCURRENCY: pr-${{ github.event.pull_request.number }} - MATURIN_FEATURE_FLAGS: "--no-default-features --features extension-module" diff --git a/.github/workflows/on_pull_request_target_contrib.yml b/.github/workflows/on_pull_request_target_contrib.yml deleted file mode 100644 index 51a5706a78a1..000000000000 --- a/.github/workflows/on_pull_request_target_contrib.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Jobs that only run for external contributors. -# These have to be carefully sanitized, we don't want to leak secrets. -name: Pull-Request-Target (Contrib) - -on: - # This will run workflows triggered by a pull request from the _base_ branch. - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target - pull_request_target: - types: - - opened - - synchronize - -defaults: - run: - shell: bash - -permissions: - contents: "read" - pull-requests: "write" # Updates PR body - -jobs: - update-pr-body: - name: Update PR body - if: github.event.pull_request.head.repo.owner.login != 'rerun-io' - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Update PR description - run: | - pixi run ./scripts/ci/update_pr_body.py \ - --github-token '${{ secrets.GITHUB_TOKEN }}' \ - --github-repository '${{ github.repository }}' \ - --pr-number '${{ github.event.pull_request.number }}' diff --git a/.github/workflows/on_push_docs.yml b/.github/workflows/on_push_docs.yml deleted file mode 100644 index f8cae71b23e2..000000000000 --- a/.github/workflows/on_push_docs.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: "Push Docs" - -on: - push: - branches: [docs-latest] - -concurrency: - group: on-push-docs - cancel-in-progress: true - -permissions: - contents: "read" - id-token: "write" - -defaults: - run: - shell: bash - -jobs: - # Get latest release version from crates.io - # This excludes any prerelease builds, e.g. `0.15.0-alpha.1` or `rc` or similar. - # We get it from crates.io because it's the strongest indicator of the latest - # fully released version we have available, and there is no better way to retrieve - # that in the context of this branch, because we don't want to rely on the contents - # of a local `Cargo.toml` or the git branch name. - get-version: - runs-on: ubuntu-latest - outputs: - version: ${{ steps.versioning.outputs.crate_version }} - steps: - - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Get version - id: versioning - run: | - crate_version=$(pixi run python scripts/ci/crates.py get-version --from=cratesio --skip-prerelease) - echo "crate_version=$crate_version" >> "$GITHUB_OUTPUT" - - build-search-index: - runs-on: ubuntu-latest-16-cores - needs: [get-version] - steps: - - uses: actions/checkout@v4 - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-linux" - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - toolchains: "nightly" - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Install rerun-sdk - run: | - - - name: Build search index - env: - # Here we disable: - # - All warnings - nightly toolchain has a different set of warnings enabled by default. - # We already test with `-D warnings` elsewhere, and the output is really noisy with the nightly warnings. - # - The web viewer server - we don't want to waste time building the web viewer, - # because it is not actually going to run - RUSTFLAGS: "-Awarnings --cfg disable_web_viewer_server" - run: | - # Install latest release version of the Rerun SDK - # We don't actually want to build it, because then we'd be installing the dev version. - pixi run pip install -r rerun_py/requirements-doc.txt - pixi run pip install rerun-sdk==${{ needs.get-version.outputs.version }} - - pixi run search-index build \ - landing \ - --url "https://edge.meilisearch.com" \ - --master-key "${{ secrets.MEILISEARCH_TOKEN }}" \ - --release-version "${{ needs.get-version.outputs.version }}" - - redeploy-rerun-io: - runs-on: ubuntu-latest - needs: [get-version] - steps: - - uses: actions/checkout@v4 - - - name: Re-deploy rerun.io - uses: ./.github/actions/vercel - with: - command: "deploy" - vercel_token: "${{ secrets.VERCEL_TOKEN }}" - vercel_team_name: "${{ vars.VERCEL_TEAM_NAME }}" - vercel_project_name: "${{ vars.VERCEL_PROJECT_NAME }}" - release_commit: "docs-latest" - release_version: "${{ needs.get-version.outputs.version }}" - target: "production" diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml deleted file mode 100644 index 6b61b147501c..000000000000 --- a/.github/workflows/on_push_main.yml +++ /dev/null @@ -1,151 +0,0 @@ -name: Push To Main - -on: - push: - branches: - - "main" - - # Can be triggered manually from within the UI or using the GH CLI, - # e.g. `gh workflow run on_push_main.yml --ref main` - workflow_dispatch: - inputs: - CONCURRENCY: - required: true - type: string - -permissions: write-all - -jobs: - checks: - name: Checks - uses: ./.github/workflows/reusable_checks.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - CHANNEL: main - secrets: inherit - - cpp_checks: - name: Checks - uses: ./.github/workflows/reusable_checks_cpp.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - CHANNEL: main - secrets: inherit - - rust_checks: - name: Checks - uses: ./.github/workflows/reusable_checks_rust.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - CHANNEL: main - secrets: inherit - - python_checks: - name: Checks - uses: ./.github/workflows/reusable_checks_python.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - secrets: inherit - - deploy-docs: - needs: [checks] - name: Deploy Docs - uses: ./.github/workflows/reusable_deploy_docs.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - PY_DOCS_VERSION_NAME: "main" - CPP_DOCS_VERSION_NAME: "main" - RS_DOCS_VERSION_NAME: "head" - UPDATE_LATEST: false - secrets: inherit - - build-web: - name: "Build web viewer" - uses: ./.github/workflows/reusable_build_web.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - CHANNEL: main - secrets: inherit - - upload-web: - name: "Upload Web" - needs: [build-web] - uses: ./.github/workflows/reusable_upload_web.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - secrets: inherit - - build-examples: - name: "Build Examples" - needs: [build-wheel-linux-x64] - uses: ./.github/workflows/reusable_build_examples.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - CHANNEL: main - WHEEL_ARTIFACT_NAME: linux-x64-wheel - secrets: inherit - - track-sizes: - name: "Track Sizes" - needs: [build-web, build-examples] - uses: ./.github/workflows/reusable_track_size.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - WITH_EXAMPLES: true - secrets: inherit - - upload-examples: - name: "Upload Examples" - needs: [build-examples] - uses: ./.github/workflows/reusable_upload_examples.yml - with: - CONCURRENCY: push-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - secrets: inherit - - # ----------------------------------------------------------------------------------- - # TODO(emilk): build and test one additional platform, picked at random - - build-rerun_c-and-upload-linux-x64: - needs: [checks] - name: "Linux-x64: Build & Upload rerun_c" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: push-linux-x64-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - PLATFORM: linux-x64 - secrets: inherit - - # ----------------------------------------------------------------------------------- - # TODO(emilk): build and test one additional platform, picked at random - - build-rerun-cli-and-upload-linux-x64: - needs: [checks] - name: "Linux-x64: Build & Upload rerun-cli" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: push-linux-x64-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - PLATFORM: linux-x64 - secrets: inherit - - # ----------------------------------------------------------------------------------- - # TODO(emilk): build and test one additional platform, picked at random - - build-wheel-linux-x64: - needs: [checks, build-rerun-cli-and-upload-linux-x64] - name: "Linux-x64: Build & Upload Wheels" - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: push-linux-x64-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: linux-x64-wheel - MODE: "pypi" - secrets: inherit - - test-wheel-linux-x64: - needs: [checks, build-wheel-linux-x64] - name: "Linux-x64: Test Wheels" - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: push-linux-x64-${{ github.ref_name }}-${{ inputs.CONCURRENCY }} - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: linux-x64-wheel - secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index b0ccc4c8663a..000000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,502 +0,0 @@ -name: Release - -on: - workflow_dispatch: - inputs: - release-type: - description: "What kind of release is this?" - type: choice - options: - - alpha - - rc - - final - required: true - -concurrency: - group: ${{ github.ref_name }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -# wants to push commits and create a PR -permissions: write-all - -jobs: - # Re-entrancy: - # - `version` is re-entrant because it doesn't commit/create PR if the version doesn't change, - # and the version doesn't change if we're already on the final version specified by the branch name. - # - `update-docs` is re-entrant because it overwrites history of the `gh-pages` branch, so any - # previous partial update will just be overwritten by the next successful run. - # - `publish-crates` is re-entrant because the `crates.py` script correctly handles publish failures - # by first checking if a crate has already been published before attempting to publish it. - # - `build-and-publish-wheels` is re-entrant because all the uploaded artifacts will be overwritten - # by any subsequent runs, and the final upload to PyPI has the `--skip-existing` flag, which ignores - # any wheels already uploaded. - # - `build-and-publish-web` is re-entrant for the same reason as `build-and-publish-wheels`, - # except that uploads are done to GCS instead of PyPI. - - checks: - name: "Checks" - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - - name: Check links for `?speculative-link` - # This checks that we have no links with `?speculative-link` in its query params. - # We use those markers to get our link checker to ignore links to unreleased docs. - # - # NOTE: For alpha releases, we won't fully publish all our docs, - # so we skip the check here, because we won't be able to - # remove the markers yet. - run: | - if [ ${{ inputs.release-type }} != "alpha" ]; then - python3 scripts/ci/check_speculative_links.py - fi - - # NOTE: When updating this job, also remember to update `post-release-version-bump`. - version: - name: "Versioning" - runs-on: ubuntu-latest - outputs: - previous: ${{ steps.versioning.outputs.previous }} - current: ${{ steps.versioning.outputs.current }} - final: ${{ steps.versioning.outputs.final }} - # will be set to `github.sha` if the pull request already exists - # this is the last (and not merge) commit in the release branch - release-commit: ${{ steps.commit.outputs.version_bump_commit_sha || github.sha }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.RERUN_BOT_TOKEN }} - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Update crate versions - id: versioning - run: | - echo Check that the release version matches expected format… - pixi run python scripts/ci/crates.py check-git-branch-name - - echo Parse the release version from the branch name… - # `release-0.8.1-meta.N` -> `0.8.1` - release_version=$(pixi run python scripts/ci/crates.py get-version --from git --finalize) - - echo "release_version: $release_version" - - echo Store version before the update, so we can later detect if it changed… - previous=$(pixi run python scripts/ci/crates.py get-version) - - echo If the version minus prerelease/build metadata is not the same as the release version, then update it.… - if [ $(pixi run python scripts/ci/crates.py get-version --finalize) != $release_version ]; then - pixi run python scripts/ci/crates.py version --exact $release_version - fi - - echo If this is an 'rc', additionally set add '-rc.N'. This will also bump the 'N' if '-rc.N' is already set… - if [ ${{ inputs.release-type }} = "rc" ]; then - pixi run python scripts/ci/crates.py version --bump prerelease --pre-id=rc - fi - - echo If this is an 'alpha', set the version to whatever is in the git branch name.… - if [ ${{ inputs.release-type }} = "alpha" ]; then - pixi run python scripts/ci/crates.py version --exact $(pixi run python scripts/ci/crates.py get-version --from git) - fi - - echo If this is a 'final', set the version to the final release version… - if [ ${{ inputs.release-type }} = "final" ]; then - pixi run python scripts/ci/crates.py version --exact $release_version - fi - - echo Store version after the update, and the expected "final" release version… - current=$(pixi run python scripts/ci/crates.py get-version) - final=$(pixi run python scripts/ci/crates.py get-version --finalize) - - echo Output everything for use in other steps… - echo "previous=$previous" - echo "current=$current" - echo "final=$final" - - echo "previous=$previous" >> "$GITHUB_OUTPUT" - echo "current=$current" >> "$GITHUB_OUTPUT" - echo "final=$final" >> "$GITHUB_OUTPUT" - - - name: Update rerun_notebook package version - run: | - pixi run python scripts/ci/update_rerun_notebook_version.py "${{ steps.versioning.outputs.current }}" - - - name: Update JS package versions - run: | - pixi run node rerun_js/scripts/version.mjs "${{ steps.versioning.outputs.current }}" - - - name: Update rerun_c version - - # Configuring CMake is enough to change the version on rerun.h! - run: | - cmake -B build -S . - - - run: pixi run toml-fmt - - - name: Commit new version - id: commit - if: steps.versioning.outputs.previous != steps.versioning.outputs.current - run: | - git pull - git config --global user.name "rerun-bot" - git config --global user.email "bot@rerun.io" - git commit -am "Bump versions to ${{ steps.versioning.outputs.current }}" - git push - echo "version_bump_commit_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - - name: Create pull request - env: - GH_TOKEN: ${{ secrets.RERUN_BOT_TOKEN }} - run: | - set +e - pr=$(gh pr view --json headRefName 2>/dev/null || echo "{}") - if echo "$pr" | jq '. | has("headRefName")' | grep -q 'true'; then - echo "PR already exists" - exit 0 - fi - set -e - - echo "PR does not exist, creating…" - - cat < pr-body.txt - ### Next steps - - Test the release - - If this is an 'alpha' release, you can just merge the pull request. - - Otherwise: - - For any added commits, run the release workflow in 'rc' mode again - - After testing, _ensure that this PR is mergeable to `main`_, then run the release workflow in 'release' mode - - Once the final release workflow finishes it will create a GitHub release for you. Then: - - [ ] Sanity check the build artifacts: - - [ ] pip install: does it install and run? - - [ ] cargo install of cli tool: does it install and run? - - [ ] C++ SDK zip: does it contain rerun_c for all platforms? - - [ ] Populate the release with the changelog and a nice header video/picture, check `Set as latest release`, then click `Publish release`. - - [ ] Update the [google colab notebooks](https://drive.google.com/drive/folders/0AC0q24MFKh3fUk9PVA) to install this version and re-execute the notebook. - - A few hours after the GitHub release is created, `regro-cf-autotick-bot` will create a - [conda feedstock PR](https://github.com/conda-forge/rerun-sdk-feedstock/pulls). - Make sure Jeremy is on top of it! - - - [ ] Tests - - [ ] Windows - - [ ] Linux - - [ ] MacOS - EOF - - gh pr create \ - --base main \ - --head $(git branch --show-current) \ - --title "Release ${{ (inputs.release-type == 'alpha' && steps.versioning.outputs.current) || steps.versioning.outputs.final }}" \ - --label "⛴ release" \ - --label "exclude from changelog" \ - --fill \ - --body-file pr-body.txt - - update-docs: - name: "Update Docs" - needs: [version, publish-web] - uses: ./.github/workflows/reusable_deploy_docs.yml - with: - CONCURRENCY: ${{ github.ref_name }} - PY_DOCS_VERSION_NAME: ${{ inputs.release-type == 'final' && needs.version.outputs.final || 'dev' }} - CPP_DOCS_VERSION_NAME: ${{ inputs.release-type == 'final' && 'stable' || 'dev' }} - RS_DOCS_VERSION_NAME: ${{ inputs.release-type == 'final' && 'stable' || 'dev' }} - RELEASE_COMMIT: ${{ needs.version.outputs.release-commit }} - RELEASE_VERSION: ${{ needs.version.outputs.final }} - UPDATE_LATEST: ${{ inputs.release-type == 'final' }} - secrets: inherit - - publish-crates: - name: "Publish Crates" - needs: [version] - uses: ./.github/workflows/reusable_release_crates.yml - with: - CONCURRENCY: ${{ github.ref_name }} - RELEASE_COMMIT: ${{ needs.version.outputs.release-commit }} - secrets: inherit - - publish-rerun_c: - name: "Build and Publish rerun_c" - needs: [version] - uses: ./.github/workflows/reusable_publish_rerun_c.yml - with: - release-version: ${{ needs.version.outputs.current }} - release-commit: ${{ needs.version.outputs.release-commit }} - concurrency: ${{ github.ref_name }} - secrets: inherit - - publish-rerun-cli: - name: "Publish rerun-cli" - needs: [version] - uses: ./.github/workflows/reusable_publish_rerun_cli.yml - with: - release-version: ${{ needs.version.outputs.current }} - release-commit: ${{ needs.version.outputs.release-commit }} - concurrency: ${{ github.ref_name }} - secrets: inherit - - publish-wheels: - name: "Build and Publish Wheels" - needs: [version, publish-rerun-cli] - uses: ./.github/workflows/reusable_publish_wheels.yml - with: - release-version: ${{ needs.version.outputs.current }} - concurrency: ${{ github.ref_name }} - release-commit: ${{ needs.version.outputs.release-commit }} - secrets: inherit - - publish-web: - name: "Build and Publish Web" - needs: [version, publish-wheels] - uses: ./.github/workflows/reusable_publish_web.yml - with: - release-version: ${{ needs.version.outputs.current }} - release-commit: ${{ needs.version.outputs.release-commit }} - concurrency: ${{ github.ref_name }} - wheel-artifact-name: linux-x64-wheel - update-latest: ${{ inputs.release-type == 'final' }} - secrets: inherit - - publish-js: - name: "Publish JS" - needs: [version] - uses: ./.github/workflows/reusable_publish_js.yml - with: - release-commit: ${{ needs.version.outputs.release-commit }} - concurrency: ${{ github.ref_name }} - secrets: inherit - - # Force-pushes `latest` and `docs-latest` to the contents of the release branch. - # The push to `docs-latest` also triggers a re-deploy of `rerun.io`. - update-latest-branch: - name: "Update Latest Branch" - if: inputs.release-type == 'final' - needs: - [ - version, - update-docs, - publish-crates, - publish-wheels, - publish-web, - publish-rerun_c, - publish-rerun-cli, - publish-js, - ] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.RERUN_BOT_TOKEN }} - ref: ${{ needs.version.outputs.release-commit }} - - - name: Update latest branch - run: | - git config --global user.name "rerun-bot" - git config --global user.email "bot@rerun.io" - git fetch - git checkout ${{ github.ref_name }} - git push --force origin refs/heads/${{ github.ref_name }}:refs/heads/latest - git push --force origin refs/heads/${{ github.ref_name }}:refs/heads/docs-latest - - github-release: - name: "GitHub Release" - if: inputs.release-type == 'final' - needs: - [ - version, - update-docs, - publish-crates, - publish-wheels, - publish-web, - publish-rerun_c, - publish-rerun-cli, - publish-js, - ] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.RERUN_BOT_TOKEN }} - - - name: Release tag - env: - GH_TOKEN: ${{ secrets.RERUN_BOT_TOKEN }} - run: | - version="${{ needs.version.outputs.final }}" - commit="${{ needs.version.outputs.release-commit }}" - git tag $version $commit - git push origin $version - gh release create $version --verify-tag --draft --title $version - - - name: Create comment - env: - GH_TOKEN: ${{ secrets.RERUN_BOT_TOKEN }} - run: | - pr_number=$(gh pr view --json number | jq '.number') - version="${{ needs.version.outputs.final }}" - - cat < comment-body.txt - GitHub release draft: [$version](https://github.com/rerun-io/rerun/releases/tag/$version) - - Do NOT create a GitHub release yet! - - The release will be automatically un-drafted by the "Sync Release Assets" job, which will run automatically. - EOF - - gh pr comment $pr_number --body-file comment-body.txt - - # Bump versions to next minor+alpha after the release has finished, - # so that the release PR can be merged. - post-release-version-bump: - name: "Post-Release Version Bump" - # We don't need to bump versions for `rc` releases, because we don't merge those. - if: inputs.release-type == 'alpha' || inputs.release-type == 'final' - needs: - [ - version, - update-docs, - publish-crates, - publish-wheels, - publish-web, - publish-rerun_c, - publish-rerun-cli, - publish-js, - ] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.RERUN_BOT_TOKEN }} - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: git config - run: | - git config --global user.name "rerun-bot" - git config --global user.email "bot@rerun.io" - git checkout ${{ github.ref_name }} - git pull --rebase - - - name: Update crate versions - id: crates - run: | - pixi run python scripts/ci/crates.py version --bump auto - version="$(pixi run python scripts/ci/crates.py get-version)" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Update rerun_notebook package version - run: | - pixi run python scripts/ci/update_rerun_notebook_version.py "${{ steps.crates.outputs.version }}" - - - name: Update JS package versions - run: | - pixi run node rerun_js/scripts/version.mjs "${{ steps.crates.outputs.version }}" - - - name: Update rerun_c version - # Configuring CMake is enough to change the version on rerun.h! - run: | - cmake -B build -S . - - - run: pixi run toml-fmt - - - name: Commit new version - run: | - git commit -am "Bump versions to ${{ steps.crates.outputs.version }}" - git push - - comment-artifact-links: - name: "Link to artifacts" - needs: - [ - version, - update-docs, - publish-crates, - publish-wheels, - publish-web, - publish-rerun_c, - publish-rerun-cli, - publish-js, - ] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.RERUN_BOT_TOKEN }} - - - name: Create comment - env: - GH_TOKEN: ${{ secrets.RERUN_BOT_TOKEN }} - run: | - pr_number=$(gh pr view --json number | jq '.number') - echo "pr_number: $pr_number" - short_commit_hash=$(echo ${{ needs.version.outputs.release-commit }} | cut -c1-7) - - if [ ${{ inputs.release-type }} = "final" ]; then - web_app_link="https://rerun.io/viewer/version/${{ needs.version.outputs.final }}" - rerun_io_docs_link="https://rerun.io/docs" - py_docs_link="https://ref.rerun.io/docs/python/${{ needs.version.outputs.final }}" - else - web_app_link="https://rerun.io/viewer/commit/$short_commit_hash" - rerun_io_docs_link="https://rerun.io/preview/$short_commit_hash/docs" - py_docs_link="https://ref.rerun.io/docs/python/dev" - fi - wheels_link="https://pypi.org/project/rerun-sdk/${{ needs.version.outputs.current }}" - crates_link="https://crates.io/crates/rerun/${{ needs.version.outputs.current }}" - npm_link="https://www.npmjs.com/package/@rerun-io/web-viewer/v/${{ needs.version.outputs.current }}" - rs_docs_link="https://docs.rs/rerun/${{ needs.version.outputs.current }}" - cpp_sdk_zip_link="https://build.rerun.io/commit/$short_commit_hash/rerun_cpp_sdk.zip" - - pip_install="pip install rerun-sdk==${{ needs.version.outputs.current }}" - cargo_install="cargo install rerun-cli@${{ needs.version.outputs.current }}" - npm_install="npm install @rerun-io/web-viewer@${{ needs.version.outputs.current }}" - - cat < comment-body.txt - Version ${{ needs.version.outputs.current }} published successfully. - - | artifact | install | - | --------------------------------- | -------------- | - | [web app]($web_app_link) | | - | [wheels]($wheels_link) | $pip_install | - | [crates]($crates_link) | $cargo_install | - | [npm]($npm_link) | $npm_install | - | [docs]($rerun_io_docs_link) | | - | [py docs]($py_docs_link) | | - | [rs docs]($rs_docs_link) | | - | [cpp_sdk zip]($cpp_sdk_zip_link) | | - EOF - - gh pr comment $pr_number --body-file comment-body.txt diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml deleted file mode 100644 index 091192663ad0..000000000000 --- a/.github/workflows/reusable_bench.yml +++ /dev/null @@ -1,164 +0,0 @@ -name: Reusable Bench - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - SAVE_BENCHES: - required: false - type: boolean - default: false - BENCH_NAME: - required: false - type: string - default: "" - COMPARE_TO: - required: false - type: string - default: "" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-bench - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - # contents permission to update benchmark contents in gh-pages branch - contents: write - id-token: "write" - # deployments permission to deploy GitHub pages website - deployments: write - -jobs: - # --------------------------------------------------------------------------- - - rs-benchmarks: - name: Rust Criterion benchmarks - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # we need full history - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-linux" - # Cache will be produced by `reusable_checks/rs-lints` - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV - - - name: Run benchmark - # Use bash shell so we get pipefail behavior with tee - run: | - cargo bench \ - --all-features \ - -p re_entity_db \ - -p re_log_encoding \ - -p re_query \ - -p re_tuid \ - -- --output-format=bencher | tee /tmp/${{ env.SHORT_SHA }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - # TODO(jleibs) make this whole thing a python script - - name: "Upload bench to GCS based on SHA" - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: /tmp/${{ env.SHORT_SHA }} - destination: "rerun-builds/benches/" - process_gcloudignore: false - - - name: Download comparison bench from GCS - if: ${{ inputs.COMPARE_TO != '' }} - run: | - mkdir /tmp/compare/ - gsutil cp gs://rerun-builds/benches/${{inputs.COMPARE_TO}} /tmp/compare/${{ inputs.COMPARE_TO }} - - - name: Install cargo-benchcmp - run: cargo install --quiet cargo-benchcmp - - - name: Compare results with benchcmp - if: ${{ inputs.COMPARE_TO != '' }} - run: cargo benchcmp /tmp/compare/${{ inputs.COMPARE_TO }} /tmp/${{ env.SHORT_SHA }} > /tmp/bench_results.txt - - - name: "Upload bench-results to GCS" - if: ${{ inputs.COMPARE_TO != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: /tmp/bench_results.txt - destination: "rerun-builds/commit/${{env.SHORT_SHA}}/" - process_gcloudignore: false - - - name: "Copy bench to named file" - if: ${{ inputs.BENCH_NAME != '' }} - run: cp /tmp/${{ env.SHORT_SHA }} /tmp/${{ inputs.BENCH_NAME }} - - # Don't upload the new named bench until the end in case the names are the same - - name: "Upload named bench to GCS" - if: ${{ inputs.BENCH_NAME != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: /tmp/${{ inputs.BENCH_NAME }} - destination: "rerun-builds/benches/" - process_gcloudignore: false - - - name: Alert on regression - # https://github.com/benchmark-action/github-action-benchmark - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Rust Benchmark - tool: "cargo" - output-file-path: /tmp/${{ env.SHORT_SHA }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Show alert with commit comment on detecting possible performance regression - comment-on-alert: true - alert-threshold: "125%" - fail-on-alert: false - comment-always: false # Generates too much GitHub notification spam - - # Don't push to gh-pages - save-data-file: false - auto-push: false - - - name: Render benchmark result - if: github.ref == 'refs/heads/main' - run: | - python3 -m pip install google-cloud-storage==2.9.0 - scripts/ci/render_bench.py crates \ - --after $(date -d"30 days ago" +%Y-%m-%d) \ - --output "gs://rerun-builds/graphs" diff --git a/.github/workflows/reusable_build_and_upload_rerun_c.yml b/.github/workflows/reusable_build_and_upload_rerun_c.yml deleted file mode 100644 index 18564deb05cd..000000000000 --- a/.github/workflows/reusable_build_and_upload_rerun_c.yml +++ /dev/null @@ -1,188 +0,0 @@ -name: Reusable Rerun-c Build - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PLATFORM: - required: true - type: string - ADHOC_NAME: - required: false - type: string - default: "" - RELEASE_COMMIT: - required: false - type: string - default: "" - - workflow_dispatch: - inputs: - ADHOC_NAME: - required: false - type: string - description: "Name of the adhoc build, used for upload directory" - default: "" - PLATFORM: - type: choice - options: - - linux-arm64 - - linux-x64 - - windows-x64 - - macos-arm64 - - macos-x64 - description: "Platform to build for" - required: true - CONCURRENCY: - required: false - type: string - default: "adhoc" - description: "Concurrency group to use" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-build-rerun_c - cancel-in-progress: true - -env: - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - set-config: - name: Set Config (${{ inputs.PLATFORM }}) - runs-on: ubuntu-latest-16-cores - outputs: - RUNNER: ${{ steps.set-config.outputs.runner }} - TARGET: ${{ steps.set-config.outputs.target }} - CONTAINER: ${{ steps.set-config.outputs.container }} - LIB_NAME: ${{ steps.set-config.outputs.lib_name }} - steps: - - name: Set runner and target based on platform - id: set-config - run: | - case "${{ inputs.PLATFORM }}" in - linux-arm64) - runner="buildjet-8vcpu-ubuntu-2204-arm" - target="aarch64-unknown-linux-gnu" - container="'rerunio/ci_docker:0.14.0'" - lib_name="librerun_c.a" - ;; - linux-x64) - runner="ubuntu-latest-16-cores" - target="x86_64-unknown-linux-gnu" - container="'rerunio/ci_docker:0.14.0'" - lib_name="librerun_c.a" - ;; - windows-x64) - runner="windows-latest-8-cores" - target="x86_64-pc-windows-msvc" - container="null" - lib_name="rerun_c.lib" - ;; - macos-arm64) - runner="macos-latest" # Small runners, because building rerun_c is fast - target="aarch64-apple-darwin" - container="null" - lib_name="librerun_c.a" - ;; - macos-x64) - runner="macos-latest" # Small runners, because building rerun_c is fast - target="x86_64-apple-darwin" - container="null" - lib_name="librerun_c.a" - ;; - *) echo "Invalid platform" && exit 1 ;; - esac - echo "runner=$runner" >> "$GITHUB_OUTPUT" - echo "target=$target" >> "$GITHUB_OUTPUT" - echo "container=$container" >> "$GITHUB_OUTPUT" - echo "lib_name=$lib_name" >> "$GITHUB_OUTPUT" - - rs-build-rerun_c: - name: Build rerun_c (${{ needs.set-config.outputs.RUNNER }}) - - needs: [set-config] - - runs-on: ${{ needs.set-config.outputs.RUNNER }} - container: - image: ${{ fromJson(needs.set-config.outputs.CONTAINER) }} - credentials: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '') }} - - - name: Set up Rust and Authenticate to GCS - uses: ./.github/actions/setup-rust - with: - cache_key: "build-${{ inputs.PLATFORM }}" - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - targets: ${{ needs.set-config.outputs.TARGET }} - - - name: Build rerun_c (release) - run: cargo build --locked -p rerun_c --release --target ${{ needs.set-config.outputs.TARGET }} - - - name: Get sha - id: get-sha - run: | - full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha) }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: "Upload rerun_c (commit)" - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.LIB_NAME }}" - destination: "rerun-builds/commit/${{ steps.get-sha.outputs.sha }}/rerun_c/${{ inputs.PLATFORM }}" - parent: false - process_gcloudignore: false - - - name: "Upload rerun_c (adhoc)" - if: ${{ inputs.ADHOC_NAME != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.LIB_NAME }}" - destination: "rerun-builds/adhoc/${{inputs.ADHOC_NAME}}/rerun_c/${{ inputs.PLATFORM }}" - parent: false - process_gcloudignore: false diff --git a/.github/workflows/reusable_build_and_upload_rerun_cli.yml b/.github/workflows/reusable_build_and_upload_rerun_cli.yml deleted file mode 100644 index f3279346b271..000000000000 --- a/.github/workflows/reusable_build_and_upload_rerun_cli.yml +++ /dev/null @@ -1,208 +0,0 @@ -name: Reusable Rerun CLI build & upload - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PLATFORM: - required: true - type: string - ADHOC_NAME: - required: false - type: string - default: "" - RELEASE_COMMIT: - required: false - type: string - default: "" - - workflow_dispatch: - inputs: - ADHOC_NAME: - required: false - type: string - default: "" - description: "Name of the adhoc build, used for upload directory" - PLATFORM: - type: choice - options: - - linux-arm64 - - linux-x64 - - windows-x64 - - macos-arm64 - - macos-x64 - description: "Platform to build for" - required: true - CONCURRENCY: - required: false - type: string - default: "adhoc" - description: "Concurrency group to use" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-build-rerun-cli - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - set-config: - name: Set Config (${{ inputs.PLATFORM }}) - runs-on: ubuntu-latest - outputs: - RUNNER: ${{ steps.set-config.outputs.runner }} - TARGET: ${{ steps.set-config.outputs.target }} - CONTAINER: ${{ steps.set-config.outputs.container }} - BIN_NAME: ${{ steps.set-config.outputs.bin_name }} - steps: - - name: Set runner and target based on platform - id: set-config - run: | - case "${{ inputs.PLATFORM }}" in - linux-arm64) - runner="buildjet-8vcpu-ubuntu-2204-arm" - target="aarch64-unknown-linux-gnu" - container="'rerunio/ci_docker:0.14.0'" - bin_name="rerun" - ;; - linux-x64) - runner="ubuntu-latest-16-cores" - target="x86_64-unknown-linux-gnu" - container="'rerunio/ci_docker:0.14.0'" - bin_name="rerun" - ;; - windows-x64) - runner="windows-latest-8-cores" - target="x86_64-pc-windows-msvc" - container="null" - bin_name="rerun.exe" - ;; - macos-arm64) - runner="macos-latest-large" # See https://github.blog/2023-10-02-introducing-the-new-apple-silicon-powered-m1-macos-larger-runner-for-github-actions/ - target="aarch64-apple-darwin" - container="null" - bin_name="rerun" - ;; - macos-x64) - runner="macos-latest-large" # See https://github.blog/2023-10-02-introducing-the-new-apple-silicon-powered-m1-macos-larger-runner-for-github-actions/ - target="x86_64-apple-darwin" - container="null" - bin_name="rerun" - ;; - *) echo "Invalid platform" && exit 1 ;; - esac - echo "runner=$runner" >> "$GITHUB_OUTPUT" - echo "target=$target" >> "$GITHUB_OUTPUT" - echo "container=$container" >> "$GITHUB_OUTPUT" - echo "bin_name=$bin_name" >> "$GITHUB_OUTPUT" - - build-rerun-cli: - name: Build rerun-cli (${{ needs.set-config.outputs.RUNNER }}) - - needs: [set-config] - - runs-on: ${{ needs.set-config.outputs.RUNNER }} - container: - image: ${{ fromJson(needs.set-config.outputs.CONTAINER) }} - credentials: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '') }} - - - name: Set up Rust and Authenticate to GCS - uses: ./.github/actions/setup-rust - with: - cache_key: "build-${{ inputs.PLATFORM }}" - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - targets: ${{ needs.set-config.outputs.TARGET }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Build web-viewer (release) - run: pixi run rerun-build-web-release - - # This does not run in the pixi environment, doing so - # causes it to select the wrong compiler on macos-arm64 - - name: Build rerun-cli - env: - # this stops `re_web_viewer_server/build.rs` from running - RERUN_IS_PUBLISHING: true - run: | - cargo build \ - --locked \ - -p rerun-cli \ - --no-default-features \ - --features native_viewer,web_viewer \ - --release \ - --target ${{ needs.set-config.outputs.TARGET }} - - - name: Get sha - id: get-sha - run: | - full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha) }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: "Upload rerun-cli (commit)" - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.BIN_NAME }}" - destination: "rerun-builds/commit/${{ steps.get-sha.outputs.sha }}/rerun-cli/${{ inputs.PLATFORM }}" - parent: false - process_gcloudignore: false - - - name: "Upload rerun-cli (adhoc)" - if: ${{ inputs.ADHOC_NAME != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.BIN_NAME }}" - destination: "rerun-builds/adhoc/${{inputs.ADHOC_NAME}}/rerun-cli/${{ inputs.PLATFORM }}" - parent: false - process_gcloudignore: false diff --git a/.github/workflows/reusable_build_and_upload_wheels.yml b/.github/workflows/reusable_build_and_upload_wheels.yml deleted file mode 100644 index 7d3a914a4cf3..000000000000 --- a/.github/workflows/reusable_build_and_upload_wheels.yml +++ /dev/null @@ -1,241 +0,0 @@ -name: Reusable Build Wheels - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - MODE: - description: "The build mode, either `pypi` or `pr`" - required: true - type: string - PLATFORM: - required: true - type: string - WHEEL_ARTIFACT_NAME: - required: false - type: string - default: "" - RELEASE_COMMIT: - required: false - type: string - default: "" - - workflow_dispatch: - inputs: - PLATFORM: - type: choice - options: - - linux-arm64 - - linux-x64 - - windows-x64 - - macos-arm64 - - macos-x64 - description: "Platform to build for" - required: true - MODE: - type: choice - required: false - options: - - pypi - - pr - description: "The build mode (`pypi` includes the web viewer, `pr` does not)" - CONCURRENCY: - required: false - type: string - default: "adhoc" - description: "Concurrency group to use" - WHEEL_ARTIFACT_NAME: - required: false - type: string - default: "" - description: "If set, will be saved under that name in the workflow artifacts" - RELEASE_COMMIT: - required: false - type: string - default: "" - description: "Release commit" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-build-wheels - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - - # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac - # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - RUSTFLAGS: --cfg=web_sys_unstable_apis - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - # --------------------------------------------------------------------------- - - set-config: - name: Set Config (${{ inputs.PLATFORM }}) - runs-on: ubuntu-latest - outputs: - RUNNER: ${{ steps.set-config.outputs.runner }} - TARGET: ${{ steps.set-config.outputs.target }} - CONTAINER: ${{ steps.set-config.outputs.container }} - COMPAT: ${{ steps.set-config.outputs.compat }} - steps: - - name: Set runner and target based on platform - id: set-config - run: | - case "${{ inputs.PLATFORM }}" in - linux-arm64) - runner="buildjet-8vcpu-ubuntu-2204-arm" - target="aarch64-unknown-linux-gnu" - container="'rerunio/ci_docker:0.14.0'" # Required to be manylinux compatible - compat="manylinux_2_31" - ;; - linux-x64) - runner="ubuntu-latest-16-cores" - target="x86_64-unknown-linux-gnu" - compat="manylinux_2_31" - container="'rerunio/ci_docker:0.14.0'" # Required to be manylinux compatible - ;; - windows-x64) - runner="windows-latest-8-cores" - target="x86_64-pc-windows-msvc" - container="null" - compat="manylinux_2_31" - ;; - macos-arm64) - runner="macos-latest-large" # See https://github.blog/2023-10-02-introducing-the-new-apple-silicon-powered-m1-macos-larger-runner-for-github-actions/ - target="aarch64-apple-darwin" - container="null" - compat="manylinux_2_31" - ;; - macos-x64) - runner="macos-latest-large" # See https://github.blog/2023-10-02-introducing-the-new-apple-silicon-powered-m1-macos-larger-runner-for-github-actions/ - target="x86_64-apple-darwin" - container="null" - compat="manylinux_2_31" - ;; - *) echo "Invalid platform" && exit 1 ;; - esac - echo "runner=$runner" >> "$GITHUB_OUTPUT" - echo "target=$target" >> "$GITHUB_OUTPUT" - echo "container=$container" >> "$GITHUB_OUTPUT" - echo "compat=$compat" >> "$GITHUB_OUTPUT" - - # --------------------------------------------------------------------------- - - build-wheels: - name: Build Wheels (${{ needs.set-config.outputs.RUNNER }}) - - needs: [set-config] - - runs-on: ${{ needs.set-config.outputs.RUNNER }} - container: - image: ${{ fromJson(needs.set-config.outputs.CONTAINER) }} - credentials: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '') }} - - - name: Set up Rust and Authenticate to GCS - uses: ./.github/actions/setup-rust - with: - cache_key: "build-${{ inputs.PLATFORM }}" - # Cache will be produced by `reusable_checks/rs-lints` - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - targets: ${{ needs.set-config.outputs.TARGET }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Get sha - id: get-sha - run: | - full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha) }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: "Download rerun-cli" - run: | - pixi run fetch-artifact \ - --commit-sha ${{ steps.get-sha.outputs.sha }} \ - --artifact rerun-cli \ - --platform ${{ inputs.PLATFORM }} \ - --dest rerun_py/rerun_sdk/rerun_cli - - - name: Build - run: | - pixi run python scripts/ci/build_and_upload_wheels.py \ - --mode ${{ inputs.MODE }} \ - --target ${{ needs.set-config.outputs.TARGET }} \ - --dir commit/${{ steps.get-sha.outputs.sha }}/wheels \ - --compat ${{ needs.set-config.outputs.COMPAT }} \ - --upload-gcs - - - name: Save wheel artifact - if: ${{ inputs.WHEEL_ARTIFACT_NAME != '' }} - uses: actions/upload-artifact@v4 - with: - name: ${{inputs.WHEEL_ARTIFACT_NAME}} - path: dist/${{ needs.set-config.outputs.TARGET }} - - # --------------------------------------------------------------------------- - # rerun_notebook support - - - name: "Build rerun_notebook" - # only build the notebook if we are building for pypi and running linux-x64 - if: ${{ inputs.MODE == 'pypi' && inputs.PLATFORM == 'linux-x64' }} - run: | - rm -rf dist - pixi run js-build-base - pixi run python scripts/ci/build_and_upload_rerun_notebook.py \ - --dir commit/${{ steps.get-sha.outputs.sha }}/wheels - - - name: Save rerun_notebook wheel artifact - if: ${{ inputs.MODE == 'pypi' && inputs.PLATFORM == 'linux-x64' }} - uses: actions/upload-artifact@v4 - with: - name: rerun_notebook_wheel - path: dist diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml deleted file mode 100644 index 4536aab49d07..000000000000 --- a/.github/workflows/reusable_build_examples.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Reusable Build Examples - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - WHEEL_ARTIFACT_NAME: - required: true - type: string - CHANNEL: # `nightly`/`main`/`pr` - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-build-examples - cancel-in-progress: true - -env: - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - rs-build-examples: - name: Build Examples - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '' }} - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-web" - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - environments: wheel-test - - - name: Download Wheel - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.WHEEL_ARTIFACT_NAME }} - path: wheel - - - name: Install built wheel - run: | - pixi run python scripts/ci/pixi_install_wheel.py --feature python-pypi --package rerun-sdk --dir wheel - - - name: Print wheel version - run: | - pixi list -e wheel-test | grep rerun_sdk - pixi run -e wheel-test python -m rerun --version - pixi run -e wheel-test which rerun - - - name: Get sha - id: get-sha - run: | - full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: Build examples - run: | - pixi run -e wheel-test build-examples rrd \ - --channel ${{ inputs.CHANNEL }} \ - example_data - - - name: Build & run snippets - run: | - pixi run -e wheel-test build-examples snippets \ - example_data/snippets - - - name: Upload assets - uses: actions/upload-artifact@v4 - with: - name: example_data - path: example_data diff --git a/.github/workflows/reusable_build_js.yml b/.github/workflows/reusable_build_js.yml deleted file mode 100644 index f5a606f4a3aa..000000000000 --- a/.github/workflows/reusable_build_js.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Reusable Build rerun_js - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-build-js - cancel-in-progress: true - -env: - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - build: - name: Build rerun_js - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: actions/setup-node@v4 - with: - node-version: "20.x" - - - name: Install Yarn - run: npm install -g yarn - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-web" - # Cache will be produced by `reusable_checks/rs-check-wasm` - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Install yarn dependencies - run: pixi run yarn --cwd rerun_js install - - - name: Build rerun_js package - run: pixi run yarn --cwd rerun_js workspaces run build diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml deleted file mode 100644 index 87378ea37bdc..000000000000 --- a/.github/workflows/reusable_build_web.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Reusable Build web viewer - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - RELEASE_VERSION: - required: false - type: string - default: "prerelease" - CHANNEL: # `nightly` or `main` - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-build-web - cancel-in-progress: true - -env: - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - rs-build-web-viewer: - name: Build web viewer - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-web" - # Cache will be produced by `reusable_checks/rs-check-wasm` - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Build web-viewer (release) - run: | - if [ ${{ inputs.CHANNEL }} = "nightly" ]; then - export DEFAULT_EXAMPLES_MANIFEST_URL="https://app.rerun.io/version/nightly/examples_manifest.json" - fi - pixi run rerun-build-web-release - - # We build a single manifest pointing to the `commit` - # All the `pr`, `main`, release tag, etc. variants will always just point to the resolved commit - - name: Build examples manifest - run: | - full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" - sha="$(echo $full_commit | cut -c1-7)" - - pixi run build-examples manifest \ - --base-url "https://app.rerun.io/commit/$sha" \ - --channel "${{ inputs.CHANNEL }}" \ - web_viewer/examples_manifest.json - - - name: Upload web viewer - uses: actions/upload-artifact@v4 - with: - name: web_viewer - path: web_viewer diff --git a/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml b/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml deleted file mode 100644 index a6f4f10ded45..000000000000 --- a/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Reusable C++ bundling and upload - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PLATFORM_FILTER: - required: false - type: string - RELEASE_COMMIT: - required: false - type: string - default: "" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-bundle-and-upload-rerun-cpp - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - bundle-and-upload-rerun_cpp: - name: Bundle and upload rerun_cpp_sdk.zip - runs-on: ubuntu-latest - container: - image: rerunio/ci_docker:0.14.0 # Need container for arrow dependency. - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '') }} - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - - name: Install python gcs library - run: | - python3 -m pip install google-cloud-storage - - - name: Get sha - id: get-sha - run: | - full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha) }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: "Bundle and upload rerun_cpp_sdk.zip" - run: python3 ./scripts/ci/bundle_and_upload_rerun_cpp.py --git-hash ${{ steps.get-sha.outputs.sha }} --platform-filter=${{ inputs.PLATFORM_FILTER }} diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml deleted file mode 100644 index e549eab37efe..000000000000 --- a/.github/workflows/reusable_checks.yml +++ /dev/null @@ -1,214 +0,0 @@ -name: "General checks: Lints, Tests, Docs" - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - CHANNEL: # `nightly`/`main`/`pr` - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-checks - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - py-test-docs: - name: Test Python Docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - cache: "pip" - cache-dependency-path: "rerun_py/requirements-doc.txt" - - - name: Install Python dependencies - shell: bash - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-doc.txt - - - name: Build via mkdocs - shell: bash - run: | - mkdocs build --strict -f rerun_py/mkdocs.yml - - # --------------------------------------------------------------------------- - - no-codegen-changes: - name: Check if running codegen would produce any changes - runs-on: ubuntu-latest-16-cores - steps: - # Note: We explicitly don't override `ref` here. We need to see if changes would be made - # in a context where we have merged with main. Otherwise we might miss changes such as one - # PR introduces a new type and another PR changes the codegen. - - uses: actions/checkout@v4 - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-linux" - save_cache: true - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Codegen check - run: pixi run codegen --force --check - - # --------------------------------------------------------------------------- - - misc-rerun-lints: - name: Rerun lints - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - - name: Rerun lints - run: pixi run lint-rerun - - - name: Toml format check - run: pixi run toml-fmt-check - - # NOTE: We don't want spurious failures caused by issues being closed, so this does not run on CI, - # at least for the time being. - # - name: Check for zombie TODOs - # shell: bash - # run: | - # pixi run ./scripts/zombie_todos.py --token ${{ secrets.GITHUB_TOKEN }} - - - name: Check for too large files - run: pixi run check-large-files - - - name: Check Python example thumbnails - run: pixi run ./scripts/ci/thumbnails.py check - - - name: Check example manifest coverage - run: pixi run ./scripts/check_example_manifest_coverage.py - - lint-md: - name: Lint markdown - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Run linter - run: | - # double quoting because pixi does its own glob expansion - pixi run mdlint --glob "'docs/content/**/*.md'" - pixi run mdlint --glob "'examples/python/*/README.md'" - pixi run mdlint --glob "'examples/cpp/*/README.md'" - pixi run mdlint --glob "'examples/rust/*/README.md'" - - # --------------------------------------------------------------------------- - - spell-check: - name: Spell Check - runs-on: ubuntu-latest - steps: - - name: Checkout Actions Repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Check spelling of entire workspace - uses: crate-ci/typos@v1.18.0 - - # --------------------------------------------------------------------------- - - misc-formatting: - name: Misc formatting - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: prettier --check - run: pixi run misc-fmt-check - - # --------------------------------------------------------------------------- - - link-checker: - name: Check links - runs-on: ubuntu-latest - # do not fail entire workflow (e.g. nightly) if this is the only failing check - continue-on-error: true - steps: - - uses: actions/checkout@v4 - - # Check https://github.com/lycheeverse/lychee on how to run locally. - - name: Link Checker - id: lychee - uses: lycheeverse/lychee-action@v1.10.0 - with: - fail: ${{ inputs.CHANNEL == 'nightly' }} - lycheeVersion: "0.15.1" - # When given a directory, lychee checks only markdown, html and text files, everything else we have to glob in manually. - # Pass --verbose, so that all considered links are printed, making it easier to debug. - args: | - --verbose --base . "**/*.md" "**/*.rs" "**/*.toml" "**/*.hpp" "**/*.cpp" "**/CMakeLists.txt" "**/*.py" "**/*.yml" - - - name: Warn because of broken links - if: ${{ steps.lychee.outputs.exit_code != '0' }} - run: echo "::warning title="Link checker"::Link checker detected broken links!" diff --git a/.github/workflows/reusable_checks_cpp.yml b/.github/workflows/reusable_checks_cpp.yml deleted file mode 100644 index 63ccf4babf5c..000000000000 --- a/.github/workflows/reusable_checks_cpp.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: "C++ Tests on all platforms & compilers" - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - CHANNEL: - required: false - type: string # enum: 'nightly', 'main', or 'pr' - - workflow_dispatch: - inputs: - CONCURRENCY: - required: false - type: string - default: "adhoc" - CHANNEL: - required: false - type: string # enum: 'nightly', 'main', or 'pr' - -concurrency: - group: ${{ inputs.CONCURRENCY }}-checks_cpp - cancel-in-progress: true - -env: - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - matrix_prep: - runs-on: ubuntu-latest - outputs: - MATRIX: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - name: Load C++ test matrix - id: set-matrix - run: echo "matrix=$(jq -c . < ./.github/workflows/cpp_matrix_full.json)" >> $GITHUB_OUTPUT - - cpp-tests: - name: C++ build & test - ${{ matrix.name }} - needs: matrix_prep - strategy: - matrix: ${{ fromJson(needs.matrix_prep.outputs.MATRIX) }} - runs-on: ${{ matrix.runs_on }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - environments: cpp - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: ${{ matrix.cache_key }} - # Cache will be produced by `reusable_checks/rs-lints` - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - # Workaround for ASAN issues on Github images https://github.com/actions/runner-images/issues/9491 - - name: Fix kernel mmap rnd bits - if: runner.os == 'Linux' - # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with - # high-entropy ASLR in much newer kernels that GitHub runners are - # using leading to random crashes: https://reviews.llvm.org/D148280 - run: sudo sysctl vm.mmap_rnd_bits=28 - - - name: pixi run -e cpp cpp-clean - run: pixi run -e cpp cpp-clean - - - name: pixi run -e cpp cpp-build-all - run: ${{ matrix.extra_env_vars }} RERUN_WERROR=ON pixi run -e cpp cpp-build-all - - - name: pixi run -e cpp cpp-test - run: ${{ matrix.extra_env_vars }} RERUN_WERROR=ON pixi run -e cpp cpp-test - - - name: pixi run -e cpp cpp-build-all-shared-libs - if: ${{ inputs.CHANNEL == 'nightly' }} - run: ${{ matrix.extra_env_vars }} RERUN_WERROR=ON pixi run -e cpp cpp-build-all-shared-libs - - - name: additional_commands - run: ${{ matrix.additional_commands }} - - cpp-formatting: - name: C++ formatting check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Run clang format on all relevant files - uses: jidicula/clang-format-action@v4.11.0 - with: - clang-format-version: "16" - # Only check c/cpp/h/hpp (default checks also .proto and others) - include-regex: ^.*\.(c|cpp|h|hpp)$ diff --git a/.github/workflows/reusable_checks_python.yml b/.github/workflows/reusable_checks_python.yml deleted file mode 100644 index 0d47ca61e773..000000000000 --- a/.github/workflows/reusable_checks_python.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: "Python Checks: Lints & Docs" - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-checks_python - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - # --------------------------------------------------------------------------- - - py-lints: - name: Python lints (ruff, mypy, …) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Python format check - run: pixi run py-fmt-check - - - name: Lint Python - run: pixi run py-lint - - # --------------------------------------------------------------------------- - - py-test-docs: - name: Test Python Docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.8" - cache: "pip" - cache-dependency-path: "rerun_py/requirements-doc.txt" - - - name: Install Python dependencies - shell: bash - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-doc.txt - - - name: Build via mkdocs - shell: bash - run: | - mkdocs build --strict -f rerun_py/mkdocs.yml diff --git a/.github/workflows/reusable_checks_rust.yml b/.github/workflows/reusable_checks_rust.yml deleted file mode 100644 index b89ef812556c..000000000000 --- a/.github/workflows/reusable_checks_rust.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: "Rust Checks: Lints, Tests, Docs" - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - CHANNEL: - required: false - type: string # enum: 'nightly', 'main', or 'pr' - -concurrency: - group: ${{ inputs.CONCURRENCY }}-checks_rust - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - # --------------------------------------------------------------------------- - - rs-lints: - name: Rust lints (fmt, check, clippy, tests, doc) - runs-on: ubuntu-latest-16-cores - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-linux" - save_cache: true - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Rust checks & tests - if: ${{ inputs.CHANNEL == 'pr' }} - run: pixi run rs-check --skip individual_crates tests - - - name: Rust checks & tests - if: ${{ inputs.CHANNEL == 'main' }} - run: pixi run rs-check --skip individual_crates - - - name: Rust all checks & tests - if: ${{ inputs.CHANNEL == 'nightly' }} - run: pixi run rs-check - - # Run some basics tests on Mac and Windows - mac-windows-tests: - name: Test on macOS and Windows - if: ${{ inputs.CHANNEL == 'nightly' }} - strategy: - matrix: - include: - - os: macos-latest - name: macos - - os: windows-latest-8-cores - name: windows - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-${{ matrix.name }}" - save_cache: true - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: cargo test --all-targets --all-features - run: cargo test --all-targets --all-features diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml deleted file mode 100644 index 34854b9b510b..000000000000 --- a/.github/workflows/reusable_deploy_docs.yml +++ /dev/null @@ -1,239 +0,0 @@ -name: "Reusable Deploy Docs" - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PY_DOCS_VERSION_NAME: - required: true - type: string - CPP_DOCS_VERSION_NAME: - required: true - type: string - RS_DOCS_VERSION_NAME: - required: true - type: string - RELEASE_VERSION: - required: false - type: string - RELEASE_COMMIT: - required: false - type: string - UPDATE_LATEST: - required: false - type: boolean - default: false - -concurrency: - group: ${{ inputs.CONCURRENCY }}-deploy-docs - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "write" - id-token: "write" - -env: - PYTHON_VERSION: "3.8" - - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -jobs: - # --------------------------------------------------------------------------- - - py-deploy-docs: - name: Python - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Don't do a shallow clone - ref: ${{ inputs.RELEASE_COMMIT || (github.event_name == 'pull_request' && github.event.pull_request.head.ref || '') }} - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - cache: "pip" - cache-dependency-path: "rerun_py/requirements-doc.txt" - - - name: Install Python dependencies - shell: bash - run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-doc.txt - - - name: Set up git author - run: | - remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Mike will incrementally update the existing gh-pages branch - # We then check it out, and reset it to a new orphaned branch, which we force-push to origin - # to make sure we don't accumulate unnecessary history in gh-pages branch - - name: Deploy via mike # https://github.com/jimporter/mike - if: ${{ inputs.UPDATE_LATEST }} - run: | - git fetch - mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.PY_DOCS_VERSION_NAME}} stable - git checkout gh-pages - git checkout --orphan gh-pages-orphan - git commit -m "Update docs for ${GITHUB_SHA}" - git push origin gh-pages-orphan:gh-pages -f - - # Mike will incrementally update the existing gh-pages branch - # We then check it out, and reset it to a new orphaned branch, which we force-push to origin - # to make sure we don't accumulate unnecessary history in gh-pages branch - - name: Deploy tag via mike # https://github.com/jimporter/mike - if: ${{ ! inputs.UPDATE_LATEST }} - run: | - git fetch - mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python ${{inputs.PY_DOCS_VERSION_NAME}} - git checkout gh-pages - git checkout --orphan gh-pages-orphan - git commit -m "Update docs for ${GITHUB_SHA}" - git push origin gh-pages-orphan:gh-pages -f - - # --------------------------------------------------------------------------- - - rs-deploy-docs: - name: Rust - needs: [py-deploy-docs] - runs-on: ubuntu-latest-16-cores - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages - ref: ${{ inputs.RELEASE_COMMIT || (github.event_name == 'pull_request' && github.event.pull_request.head.ref || '') }} - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-linux" - # Cache will be produced by `reusable_checks/rs-lints` - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: Delete existing /target/doc - run: rm -rf ./target/doc - - - name: cargo doc --document-private-items - run: cargo doc --document-private-items --no-deps --all-features --workspace --exclude rerun-cli - - - name: Set up git author - run: | - remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up ghp-import - run: pip install ghp-import - - - name: Patch in a redirect page - run: echo "" > target/doc/index.html - env: - REDIRECT_CRATE: rerun - - # See: https://github.com/c-w/ghp-import - - name: Deploy the docs - run: | - git fetch - python3 -m ghp_import -n -p -x docs/rust/${{ inputs.RS_DOCS_VERSION_NAME }} target/doc/ -m "Update the rust docs" - - cpp-deploy-docs: - name: Cpp - needs: [rs-deploy-docs] - runs-on: ubuntu-latest - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages - ref: ${{ inputs.RELEASE_COMMIT || (github.event_name == 'pull_request' && github.event.pull_request.head.ref || '') }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Doxygen C++ docs - run: pixi run -e cpp cpp-docs - - - name: Set up git author - run: | - remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up ghp-import - run: pip install ghp-import - - # TODO(andreas): Do we need this? - # - name: Patch in a redirect page - # shell: bash - # run: echo "" > target/doc/index.html - # env: - # REDIRECT_CRATE: rerun - - # See: https://github.com/c-w/ghp-import - - name: Deploy the docs (versioned) - if: ${{ inputs.RELEASE_VERSION }} - run: | - git fetch - python3 -m ghp_import -n -p -x docs/cpp/${{ inputs.RELEASE_VERSION }} rerun_cpp/docs/html/ -m "Update the C++ docs (versioned)" - - - name: Deploy the docs (named) - run: | - git fetch - python3 -m ghp_import -n -p -x docs/cpp/${{ inputs.CPP_DOCS_VERSION_NAME }} rerun_cpp/docs/html/ -m "Update the C++ docs" diff --git a/.github/workflows/reusable_deploy_landing_preview.yml b/.github/workflows/reusable_deploy_landing_preview.yml deleted file mode 100644 index 3ae87315b530..000000000000 --- a/.github/workflows/reusable_deploy_landing_preview.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Reusable Deploy Landing Preview - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PR_NUMBER: - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-deploy-landing-preview - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "write" - id-token: "write" - pull-requests: "write" - -jobs: - deploy: - name: Deploy - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '' }} - - - name: Get sha - id: get-sha - run: | - full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" - echo "sha=$full_commit" >> "$GITHUB_OUTPUT" - - - name: Re-deploy rerun.io - id: vercel - uses: ./.github/actions/vercel - with: - command: "deploy" - vercel_token: ${{ secrets.VERCEL_TOKEN }} - vercel_team_name: ${{ vars.VERCEL_TEAM_NAME }} - vercel_project_name: ${{ vars.VERCEL_PROJECT_NAME }} - release_commit: ${{ steps.get-sha.outputs.sha }} - target: "preview" - - - name: Create PR comment - # https://github.com/mshick/add-pr-comment - uses: mshick/add-pr-comment@v2.8.2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - message: | - ## Deployed docs - - | Commit | Link | - | ------- | ----- | - | ${{ steps.get-sha.outputs.sha }} | https://${{ steps.vercel.outputs.vercel_preview_url }}/docs | diff --git a/.github/workflows/reusable_pip_index.yml b/.github/workflows/reusable_pip_index.yml deleted file mode 100644 index 7e02b2b924a2..000000000000 --- a/.github/workflows/reusable_pip_index.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Reusable Pip Index - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - COMMIT: - required: false - type: string - default: "" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-pip-index - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - pr-summary: - name: Create a Pip Index file - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ inputs.COMMIT || (github.event_name == 'pull_request' && github.event.pull_request.head.ref || '') }} - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - - name: Install deps - run: pip install google-cloud-storage Jinja2 - - - name: Render pip index and upload to gcloud - run: | - full_commit="${{ inputs.COMMIT || (github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha) }}" - commit=$(echo $full_commit | cut -c1-7) - - python scripts/ci/generate_prerelease_pip_index.py \ - --title "Commit: $commit" \ - --dir "commit/$commit/wheels" \ - --upload \ - --check diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml deleted file mode 100644 index 8b46b818621d..000000000000 --- a/.github/workflows/reusable_pr_summary.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Reusable PR Summary - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PR_NUMBER: - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-pr-summary - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - pull-requests: "write" - -jobs: - pr-summary: - name: Create HTML summary for PR - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - - name: Render HTML template - run: | - pixi run python scripts/ci/generate_pr_summary.py \ - --github-token ${{secrets.GITHUB_TOKEN}} \ - --github-repository ${GITHUB_REPOSITORY} \ - --pr-number ${{ inputs.PR_NUMBER }} \ - --upload - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ inputs.PR_NUMBER }} diff --git a/.github/workflows/reusable_publish_js.yml b/.github/workflows/reusable_publish_js.yml deleted file mode 100644 index af70d65081cc..000000000000 --- a/.github/workflows/reusable_publish_js.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Build and publish JS - -on: - workflow_call: - inputs: - concurrency: - type: string - required: true - release-commit: - description: "Commit to release" - type: string - required: true - -concurrency: - group: ${{ inputs.concurrency }}-publish-js - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "write" - id-token: "write" - -jobs: - get-commit-sha: - name: Get Commit Sha - runs-on: ubuntu-latest - outputs: - short-sha: ${{ steps.get-short-sha.outputs.short-sha }} - full-sha: ${{ steps.get-full-sha.outputs.full-sha }} - steps: - - name: "Set short-sha" - id: get-short-sha - run: echo "short-sha=$(echo ${{ inputs.release-commit }} | cut -c1-7)" >> $GITHUB_OUTPUT - - - name: "Set full-sha" - id: get-full-sha - run: echo "full-sha=${{ inputs.release-commit }}" >> $GITHUB_OUTPUT - - build: - runs-on: ubuntu-latest-16-cores - needs: [get-commit-sha] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.release-commit }} - - - uses: actions/setup-node@v4 - with: - node-version: "20.x" - registry-url: "https://registry.npmjs.org" - - - name: Install Yarn - run: npm install -g yarn - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-web" - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Publish packages - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - pixi run node rerun_js/scripts/publish.mjs diff --git a/.github/workflows/reusable_publish_rerun_c.yml b/.github/workflows/reusable_publish_rerun_c.yml deleted file mode 100644 index e36d38e973df..000000000000 --- a/.github/workflows/reusable_publish_rerun_c.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Build and publish C++ SDK - -on: - workflow_call: - inputs: - concurrency: - type: string - required: true - release-version: - description: "Release Version Number (Must match Cargo.toml)" - type: string - required: true - release-commit: - description: "Which commit to build+publish" - type: string - required: true - -permissions: - contents: "read" - id-token: "write" - -jobs: - linux-arm64: - name: "Linux-Arm64" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: publish-rerun-c-linux-arm64-${{ github.ref_name }} - PLATFORM: linux-arm64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - linux-x64: - name: "Linux-x64" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: publish-rerun-c-linux-x64-${{ github.ref_name }} - PLATFORM: linux-x64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - macos-x64: - name: "Mac-Intel" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: publish-rerun-c-macos-x64-${{ github.ref_name }} - PLATFORM: macos-x64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - macos-arm64: - name: "Mac-Arm" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: publish-rerun-c-macos-arm64-${{ github.ref_name }} - PLATFORM: macos-arm64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - windows-x64: - name: "Windows-x64" - uses: ./.github/workflows/reusable_build_and_upload_rerun_c.yml - with: - CONCURRENCY: publish-rerun-c-windows-${{ github.ref_name }} - PLATFORM: windows-x64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - bundle-and-upload-rerun_cpp: - name: "Bundle and upload rerun_cpp_sdk.zip" - needs: [linux-arm64, linux-x64, macos-x64, macos-arm64, windows-x64] - uses: ./.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml - with: - CONCURRENCY: bundle-rerun-c-${{ github.ref_name }} - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit diff --git a/.github/workflows/reusable_publish_rerun_cli.yml b/.github/workflows/reusable_publish_rerun_cli.yml deleted file mode 100644 index 440c1ce53a40..000000000000 --- a/.github/workflows/reusable_publish_rerun_cli.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Build and publish wheels - -on: - workflow_call: - inputs: - concurrency: - type: string - required: true - release-version: - description: "Release Version Number (Must match Cargo.toml)" - type: string - required: true - release-commit: - description: "Which commit to build+publish" - type: string - required: true - -permissions: - contents: "read" - id-token: "write" - -jobs: - linux-arm64: - name: "Linux-arm64" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: publish-rerun-cli-linux-arm64-${{ github.ref_name }} - PLATFORM: linux-arm64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - linux-x64: - name: "Linux-x64" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: publish-rerun-cli-linux-x64-${{ github.ref_name }} - PLATFORM: linux-x64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - macos-x64: - name: "Mac-Intel" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: publish-rerun-cli-macos-x64-${{ github.ref_name }} - PLATFORM: macos-x64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - macos-arm64: - name: "Mac-Arm" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: publish-rerun-cli-macos-arm64-${{ github.ref_name }} - PLATFORM: macos-arm64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - windows-x64: - name: "Windows-x64" - uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml - with: - CONCURRENCY: publish-rerun-cli-windows-x64-${{ github.ref_name }} - PLATFORM: windows-x64 - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit diff --git a/.github/workflows/reusable_publish_web.yml b/.github/workflows/reusable_publish_web.yml deleted file mode 100644 index 088a06d154ca..000000000000 --- a/.github/workflows/reusable_publish_web.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: Build and publish web - -on: - workflow_call: - inputs: - concurrency: - type: string - required: true - release-version: - description: "Release Version Number (Must match Cargo.toml)" - type: string - required: true - release-commit: - description: "Commit to release" - type: string - required: true - wheel-artifact-name: - description: "Name of the wheel to use when running examples" - type: string - required: true - update-latest: - description: "Whether to update the latest version of app" - type: boolean - required: true - -defaults: - run: - shell: bash - -permissions: - contents: "write" - id-token: "write" - -jobs: - get-commit-sha: - name: Get Commit Sha - runs-on: ubuntu-latest - outputs: - short-sha: ${{ steps.get-short-sha.outputs.short-sha }} - full-sha: ${{ steps.get-full-sha.outputs.full-sha }} - steps: - - name: "Set short-sha" - id: get-short-sha - run: echo "short-sha=$(echo ${{ inputs.release-commit }} | cut -c1-7)" >> $GITHUB_OUTPUT - - - name: "Set full-sha" - id: get-full-sha - run: echo "full-sha=${{ inputs.release-commit }}" >> $GITHUB_OUTPUT - - build-web: - runs-on: ubuntu-latest-16-cores - needs: [get-commit-sha] - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.release-commit }} - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-web" - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - environments: wheel-test - - # built by `reusable_build_and_publish_wheels` - - name: Download Wheel - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.wheel-artifact-name }} - path: wheel - - - name: Install built wheel - run: | - pixi run python scripts/ci/pixi_install_wheel.py --feature python-pypi --package rerun-sdk --dir wheel - - - name: Print wheel version - run: | - pixi list -e wheel-test | grep rerun_sdk - pixi run -e wheel-test python -m rerun --version - pixi run -e wheel-test which rerun - - - name: Build web-viewer (release) - run: | - pixi run -e wheel-test rerun-build-web-release - - - name: Build examples - run: | - pixi run -e wheel-test build-examples rrd \ - --channel "release" \ - web_viewer/examples - - - name: Build & run snippets - run: | - pixi run -e wheel-test build-examples snippets \ - web_viewer/examples/snippets - - - name: Build examples manifest - run: | - pixi run -e wheel-test build-examples manifest \ - --base-url "https://app.rerun.io/version/${{inputs.release-version}}" \ - --channel "release" \ - web_viewer/examples_manifest.json - - - name: Upload app.rerun.io (versioned) - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/version/${{ inputs.release-version }}" - parent: false - process_gcloudignore: false - - - name: Upload app.rerun.io (commit) - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/commit/${{ needs.get-commit-sha.outputs.short-sha }}" - parent: false - process_gcloudignore: false - - - name: Publish app.rerun.io - if: inputs.update-latest - run: | - gsutil -m cp -r 'gs://rerun-web-viewer/version/${{ inputs.release-version }}/*' gs://rerun-web-viewer/version/latest diff --git a/.github/workflows/reusable_publish_wheels.yml b/.github/workflows/reusable_publish_wheels.yml deleted file mode 100644 index d2bec3303060..000000000000 --- a/.github/workflows/reusable_publish_wheels.yml +++ /dev/null @@ -1,182 +0,0 @@ -name: Build and publish wheels - -# To run this manually: -# 1. Build each platform using `scripts/ci/build_and_upload_wheels.py` -# 2. (optional) Generate index using `scripts/ci/generate_prerelease_pip_index.py` -# 3. Publish to PyPI using `scripts/ci/publish_wheels.py` - -on: - workflow_call: - inputs: - concurrency: - type: string - required: true - release-version: - description: "Release Version Number (Must match Cargo.toml)" - type: string - required: true - release-commit: - description: "Which commit to build+publish" - type: string - required: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - get-commit-sha: - name: Get Commit Sha - runs-on: ubuntu-latest - outputs: - short-sha: ${{ steps.get-short-sha.outputs.short-sha }} - full-sha: ${{ steps.get-full-sha.outputs.full-sha }} - steps: - - name: "Set short-sha" - id: get-short-sha - run: echo "short-sha=$(echo ${{ inputs.release-commit }} | cut -c1-7)" >> $GITHUB_OUTPUT - - - name: "Set full-sha" - id: get-full-sha - run: echo "full-sha=${{ inputs.release-commit }}" >> $GITHUB_OUTPUT - - ## Build - - # Note: this also builds `rerun_notebook` - build-linux-x64: - name: "Linux-x64: Build Wheels" - needs: [get-commit-sha] - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: wheels-build-linux-x64-${{ inputs.concurrency }} - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: linux-x64-wheel - RELEASE_COMMIT: ${{ inputs.release-commit }} - MODE: "pypi" - secrets: inherit - - build-linux-arm64: - name: "Linux-arm64: Build Wheels" - needs: [get-commit-sha] - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: wheels-build-linux-arm64-${{ inputs.concurrency }} - PLATFORM: linux-arm64 - WHEEL_ARTIFACT_NAME: linux-arm64-wheel - RELEASE_COMMIT: ${{ inputs.release-commit }} - MODE: "pypi" - secrets: inherit - - build-windows-x64: - name: "Windows-x64: Build Wheels" - needs: [get-commit-sha] - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: wheels-build-windows-x64-${{ inputs.concurrency }} - PLATFORM: windows-x64 - WHEEL_ARTIFACT_NAME: windows-x64-wheel - RELEASE_COMMIT: ${{ inputs.release-commit }} - MODE: "pypi" - secrets: inherit - - build-macos-arm64: - name: "Macos-arm64: Build Wheels" - needs: [get-commit-sha] - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: wheels-build-macos-arm64-${{ inputs.concurrency }} - PLATFORM: macos-arm64 - WHEEL_ARTIFACT_NAME: macos-arm64-wheel - RELEASE_COMMIT: ${{ inputs.release-commit }} - MODE: "pypi" - secrets: inherit - - build-macos-x64: - name: "Macos-x64: Build Wheels" - needs: [get-commit-sha] - uses: ./.github/workflows/reusable_build_and_upload_wheels.yml - with: - CONCURRENCY: wheels-build-macos-x64-${{ inputs.concurrency }} - PLATFORM: macos-x64 - WHEEL_ARTIFACT_NAME: "macos-x64-wheel" - RELEASE_COMMIT: ${{ inputs.release-commit }} - MODE: "pypi" - secrets: inherit - - ## Test - - test-windows-x64: - name: "Windows-x64: Test Wheels" - needs: [build-windows-x64] - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: wheels-test-windows-x64-${{ inputs.concurrency }} - PLATFORM: windows-x64 - WHEEL_ARTIFACT_NAME: windows-x64-wheel - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - test-linux: - name: "Linux-x64: Test Wheels" - needs: [build-linux-x64] - uses: ./.github/workflows/reusable_test_wheels.yml - with: - CONCURRENCY: wheels-test-linux-x64-${{ inputs.concurrency }} - PLATFORM: linux-x64 - WHEEL_ARTIFACT_NAME: linux-x64-wheel - RELEASE_COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - generate-wheel-index: - name: "Generate Pip Index" - needs: - [ - build-linux-x64, - build-linux-arm64, - build-windows-x64, - build-macos-arm64, - build-macos-x64, - ] - uses: ./.github/workflows/reusable_pip_index.yml - with: - CONCURRENCY: generate-wheel-index-${{ inputs.concurrency }} - COMMIT: ${{ inputs.release-commit }} - secrets: inherit - - publish-wheels: - name: "Publish Wheels" - needs: [get-commit-sha, generate-wheel-index] - runs-on: ubuntu-latest-16-cores - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Don't do a shallow clone since we need it for finding the full commit hash - ref: ${{ inputs.release-commit }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - - name: Publish to PyPI - run: | - pixi run python scripts/ci/publish_wheels.py \ - --version ${{ inputs.release-version }} \ - --dir "commit/${{ needs.get-commit-sha.outputs.short-sha }}/wheels" \ - --repository "${{ vars.PYPI_REPOSITORY }}" \ - --token "${{ secrets.MATURIN_PYPI_TOKEN }}" \ diff --git a/.github/workflows/reusable_release_crates.yml b/.github/workflows/reusable_release_crates.yml deleted file mode 100644 index e4e74000057e..000000000000 --- a/.github/workflows/reusable_release_crates.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Release crates - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - RELEASE_COMMIT: - required: false - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-release-crates - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - publish-crates: - name: "Publish Crates" - runs-on: ubuntu-latest-16-cores - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ inputs.RELEASE_COMMIT || (github.event_name == 'pull_request' && github.event.pull_request.head.ref || '') }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Build web-viewer (release) - run: pixi run rerun-build-web-release - - - name: Publish - run: pixi run python scripts/ci/crates.py publish --token ${{ secrets.CRATES_IO_TOKEN }} diff --git a/.github/workflows/reusable_run_notebook.yml b/.github/workflows/reusable_run_notebook.yml deleted file mode 100644 index 8cf05a27ac62..000000000000 --- a/.github/workflows/reusable_run_notebook.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Reusable Build and Upload Notebook - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - WHEEL_ARTIFACT_NAME: - required: false - type: string - default: "" - -concurrency: - group: ${{ inputs.CONCURRENCY }}-run-notebook - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - run-notebook: - name: Run notebook - runs-on: ubuntu-latest-16-cores # Note that as of writing we need the additional storage page (14 gb of the ubunut-latest runner is not enough). - container: - image: rerunio/ci_docker:0.14.0 # Required to run the wheel or we get "No matching distribution found for attrs>=23.1.0" during `pip install rerun-sdk` - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - environments: wheel-test - - - name: Download Wheel - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.WHEEL_ARTIFACT_NAME }} - path: wheel - - - name: Download Notebook Wheel - uses: actions/download-artifact@v4 - with: - name: rerun_notebook_wheel - path: wheel - - - name: Get version - id: get-version - run: | - pixi run -e wheel-test 'echo "wheel_version=$(python scripts/ci/crates.py get-version)"' >> "$GITHUB_OUTPUT" - - - name: Install built wheel - run: | - pixi run python scripts/ci/pixi_install_wheel.py --feature python-pypi --package rerun-sdk --dir wheel - pixi run python scripts/ci/pixi_install_wheel.py --feature python-pypi --package rerun-notebook --dir wheel --platform-independent - - - name: Install Deps - run: pixi run -e wheel-test pip install -r examples/python/notebook/requirements.txt - - - name: Create notebook - run: pixi run -e wheel-test jupyter nbconvert --to=html --ExecutePreprocessor.enabled=True examples/python/notebook/cube.ipynb --output /tmp/cube.html - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: Get sha - id: get-sha - run: | - full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: "Upload Notebook" - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "/tmp/cube.html" - destination: "rerun-builds/commit/${{ steps.get-sha.outputs.sha }}/notebooks" - parent: false - process_gcloudignore: false diff --git a/.github/workflows/reusable_sync_release_assets.yml b/.github/workflows/reusable_sync_release_assets.yml deleted file mode 100644 index 4a7de14289fd..000000000000 --- a/.github/workflows/reusable_sync_release_assets.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Sync assets with release - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - RELEASE_VERSION: - required: true - type: string - default: "" - WAIT_TIME_SECS: - required: false - type: number - default: 0 - -concurrency: - group: ${{ inputs.CONCURRENCY }}-sync-assets - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "write" - id-token: "write" - -jobs: - sync-assets: - name: Upload assets from build.rerun.io - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - - name: Sync release assets & build.rerun.io - run: | - pixi run python ./scripts/ci/sync_release_assets.py \ - --github-release ${{ inputs.RELEASE_VERSION }} \ - --github-token ${{ secrets.GITHUB_TOKEN }} \ - --wait ${{ inputs.WAIT_TIME_SECS }} \ - --remove --update diff --git a/.github/workflows/reusable_test_wheels.yml b/.github/workflows/reusable_test_wheels.yml deleted file mode 100644 index e1243ff5923c..000000000000 --- a/.github/workflows/reusable_test_wheels.yml +++ /dev/null @@ -1,204 +0,0 @@ -name: Reusable Test Wheels - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PLATFORM: - required: true - type: string - WHEEL_ARTIFACT_NAME: - required: false - type: string - default: "" - RELEASE_COMMIT: - required: false - type: string - default: "" - FAST: - required: false - type: boolean - default: false - -concurrency: - group: ${{ inputs.CONCURRENCY }}-test-wheels - cancel-in-progress: true - -env: - PYTHON_VERSION: "3.8" - # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses - # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html - # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html - - # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac - # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings - RUSTFLAGS: --cfg=web_sys_unstable_apis - - RUSTDOCFLAGS: --deny warnings - - # Disable the GHA backend (Github's 10GB storage) since we use our own GCS backend. - # See: https://github.com/marketplace/actions/sccache-action - SCCACHE_GHA_ENABLED: "false" - - # Wrap every `rustc` invocation in `sccache`. - RUSTC_WRAPPER: "sccache" - - # Not only `sccache` cannot cache incremental builds, it's counter-productive to generate all - # these incremental artifacts when running on CI. - CARGO_INCREMENTAL: "0" - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - # --------------------------------------------------------------------------- - - set-config: - name: Set Config (${{ inputs.PLATFORM }}) - runs-on: ubuntu-latest-16-cores - outputs: - RUNNER: ${{ steps.set-config.outputs.runner }} - TARGET: ${{ steps.set-config.outputs.target }} - CONTAINER: ${{ steps.set-config.outputs.container }} - steps: - - name: Set runner and target based on platform - id: set-config - - # TODO(#5525): at least this target is broken, maybe others — we currently only use linux-x64 and windows-x64 - run: | - case "${{ inputs.PLATFORM }}" in - linux-arm64) - runner="buildjet-8vcpu-ubuntu-2204-arm" - target="aarch64-unknown-linux-gnu" - container="'rerunio/ci_docker:0.14.0'" - ;; - linux-x64) - runner="ubuntu-latest-16-cores" - target="x86_64-unknown-linux-gnu" - container="'rerunio/ci_docker:0.14.0'" - ;; - windows-x64) - runner="windows-latest-8-cores" - target="x86_64-pc-windows-msvc" - container="null" - ;; - macos-arm64) - runner="macos-13-xlarge" # This is an Arm vm, https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners#about-macos-larger-runners - target="aarch64-apple-darwin" - container="null" - ;; - macos-x64) - runner="macos-13-large" # This is an Intel vm, see https://docs.github.com/en/actions/using-github-hosted-runners/about-larger-runners/about-larger-runners#about-macos-larger-runners - target="x86_64-apple-darwin" - container="null" - ;; - *) echo "Invalid platform" && exit 1 ;; - esac - echo "runner=$runner" >> "$GITHUB_OUTPUT" - echo "target=$target" >> "$GITHUB_OUTPUT" - echo "container=$container" >> "$GITHUB_OUTPUT" - - # --------------------------------------------------------------------------- - - test-wheels: - name: Test Wheels (${{ needs.set-config.outputs.RUNNER }}) - - needs: [set-config] - - runs-on: ${{ needs.set-config.outputs.RUNNER }} - container: - image: ${{ fromJson(needs.set-config.outputs.CONTAINER) }} - credentials: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - steps: - - name: Show context - run: | - echo "GITHUB_CONTEXT": $GITHUB_CONTEXT - echo "JOB_CONTEXT": $JOB_CONTEXT - echo "INPUTS_CONTEXT": $INPUTS_CONTEXT - echo "ENV_CONTEXT": $ENV_CONTEXT - env: - ENV_CONTEXT: ${{ toJson(env) }} - GITHUB_CONTEXT: ${{ toJson(github) }} - JOB_CONTEXT: ${{ toJson(job) }} - INPUTS_CONTEXT: ${{ toJson(inputs) }} - - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '') }} - - - name: Set up Rust - uses: ./.github/actions/setup-rust - with: - cache_key: "build-${{ inputs.PLATFORM }}" - # Cache will be produced by `reusable_checks/rs-lints` - save_cache: false - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - # Only has the deps for round-trips. Not all examples. - environments: wheel-test-min - - - name: Download Wheel - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.WHEEL_ARTIFACT_NAME }} - path: wheel - - - name: Get version - id: get-version - run: pixi run -e wheel-test-min 'echo "wheel_version=$(python scripts/ci/crates.py get-version)"' >> "$GITHUB_OUTPUT" - - - name: Install built wheel - run: | - pixi run python scripts/ci/pixi_install_wheel.py --feature python-pypi --package rerun-sdk --dir wheel - - - name: Print wheel version - run: | - pixi list -e wheel-test-min | grep rerun_sdk - pixi run -e wheel-test-min python -m rerun --version - pixi run -e wheel-test-min which rerun - pixi run -e wheel-test-min rerun-from-path --version - - - name: Run unit tests - run: cd rerun_py/tests && pixi run -e wheel-test-min pytest -c ../pyproject.toml - - - name: Run e2e test - run: pixi run -e wheel-test-min RUST_LOG=debug python scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed - - - name: Check for archetype roundtrip tests - if: ${{ inputs.FAST }} - # Only check that we have the archetype roundtrip tests, but don't spend time actually running them - run: pixi run -e wheel-test-min RUST_LOG=debug python tests/roundtrips.py --no-run - - - name: Run tests/roundtrips.py - if: ${{ !inputs.FAST }} - # --release so we can inherit from some of the artifacts that maturin has just built before - # explicit target because otherwise cargo loses the target cache… even though this is the target anyhow… - # --no-py-build because rerun-sdk is already built and installed - run: pixi run -e wheel-test-min RUST_LOG=debug python tests/roundtrips.py --release --target ${{ needs.set-config.outputs.TARGET }} --no-py-build - - - name: Build C++ examples - if: ${{ !inputs.FAST }} - # Separated out of compare_snippet_output.py run so we control the pixi environment. - # This used to cause issues on Windows during the setup of the pixi environment when running from inside `compare_snippet_output.py`. - run: pixi run -e wheel-test-min cpp-build-snippets - - - name: Run docs/snippets/compare_snippet_output.py - if: ${{ !inputs.FAST }} - # --release so we can inherit from some of the artifacts that maturin has just built before - # explicit target because otherwise cargo loses the target cache… even though this is the target anyhow… - # --no-py-build because rerun-sdk is already built and installed - run: pixi run -e wheel-test-min RUST_LOG=debug python docs/snippets/compare_snippet_output.py --release --target ${{ needs.set-config.outputs.TARGET }} --no-py-build --no-cpp-build diff --git a/.github/workflows/reusable_track_size.yml b/.github/workflows/reusable_track_size.yml deleted file mode 100644 index d5e83f412246..000000000000 --- a/.github/workflows/reusable_track_size.yml +++ /dev/null @@ -1,198 +0,0 @@ -name: "Track Size" - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PR_NUMBER: - required: false - type: number - WITH_EXAMPLES: - required: true - type: boolean - -defaults: - run: - shell: bash - -permissions: - contents: write - id-token: write - deployments: write - pull-requests: write - -jobs: - track-sizes: - name: "Track Sizes" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # we need full history - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Get context - id: context - run: | - echo "short_sha=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: "Set up Cloud SDK" - uses: "google-github-actions/setup-gcloud@v2" - with: - version: ">= 363.0.0" - - - name: Download web_viewer - uses: actions/download-artifact@v4 - with: - name: web_viewer - path: web_viewer - - - name: Download examples - if: ${{ inputs.WITH_EXAMPLES }} - uses: actions/download-artifact@v4 - with: - name: example_data - path: example_data - - - name: Download base results - run: | - # Get base commit: - # 1. From the index file - # 2. From the latest commit in the base branch of the PR - # 3. From the latest commit in the current branch - index_path="gs://rerun-builds/sizes/index" - if [ "$(gsutil -q stat $index_path ; echo $?)" = 0 ]; then - gsutil cp $index_path "/tmp/base_index" - base_commit=$(cat /tmp/base_index) - else - if [ -n ${{ inputs.PR_NUMBER }} ]; then - base_commit=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-7) - else - base_commit=${{ steps.context.outputs.short_sha }} - fi - fi - echo "base commit: $base_commit" - - # Download data for base commit, or default to empty file - data_path="gs://rerun-builds/sizes/commit/$base_commit/data.json" - if [ "$(gsutil -q stat $data_path ; echo $?)" = 0 ]; then - gsutil cp $data_path "/tmp/prev.json" - else - echo "[]" > "/tmp/prev.json" - fi - - - name: Measure sizes - id: measure - run: | - entries=() - - entries+=("Wasm:web_viewer/re_viewer_bg.wasm:MiB") - entries+=("JS:web_viewer/re_viewer.js:kiB") - - if [ -n ${{ inputs.WITH_EXAMPLES }} = "true" ]; then - for file in example_data/*.rrd; do - name=$(basename "$file") - entries+=("$name:$file:MiB") - done - fi - - python3 scripts/ci/count_bytes.py "${entries[@]}" > /tmp/sizes.json - - python3 scripts/ci/count_dependencies.py -p re_sdk --no-default-features > /tmp/deps1.json - python3 scripts/ci/count_dependencies.py -p re_viewer --all-features > /tmp/deps2.json - python3 scripts/ci/count_dependencies.py -p rerun --all-features > /tmp/deps3.json - - # Merge the results, putting dependencies first (on top): - jq -s '.[0] + .[1] + .[2] + .[3]' /tmp/deps1.json /tmp/deps2.json /tmp/deps3.json /tmp/sizes.json > /tmp/data.json - - comparison=$( - python3 scripts/ci/compare.py \ - --threshold=2% \ - --before-header=${{ (inputs.PR_NUMBER && github.event.pull_request.base.ref) || 'Before' }} \ - --after-header=${{ github.ref_name }} \ - "/tmp/prev.json" "/tmp/data.json" - ) - { - echo 'comparison<> "$GITHUB_OUTPUT" - - if [ -n "$comparison" ]; then - echo "is_comparison_set=true" >> "$GITHUB_OUTPUT" - else - echo "is_comparison_set=false" >> "$GITHUB_OUTPUT" - fi - - echo "$entries" - echo "previous: $(cat /tmp/prev.json)" - echo "current: $(cat /tmp/data.json)" - echo "$comparison" - echo "is comparison set: $is_comparison_set" - - - name: Upload data to GCS (commit) - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: /tmp/data.json - destination: "rerun-builds/sizes/commit/${{ steps.context.outputs.short_sha }}" - process_gcloudignore: false - - - name: Create index file - if: github.ref == 'refs/heads/main' - run: | - echo "${{ steps.context.outputs.short_sha }}" > "/tmp/index" - - - name: Upload index - if: github.ref == 'refs/heads/main' - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: /tmp/index - destination: "rerun-builds/sizes" - process_gcloudignore: false - - - name: Create PR comment - if: inputs.PR_NUMBER != '' && steps.measure.outputs.is_comparison_set == 'true' - # https://github.com/mshick/add-pr-comment - uses: mshick/add-pr-comment@v2.8.2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - message: | - # Size changes - - ${{ steps.measure.outputs.comparison }} - - - name: Alert on regression - if: github.ref == 'refs/heads/main' - # https://github.com/benchmark-action/github-action-benchmark - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Sizes - tool: customSmallerIsBetter - output-file-path: /tmp/data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Show alert with commit on detecting possible size regression - comment-on-alert: true - alert-threshold: "110%" - fail-on-alert: false - comment-always: false # Generates too much GitHub notification spam - - # Don't push to gh-pages - save-data-file: false - auto-push: false - - - name: Render benchmark result - if: github.ref == 'refs/heads/main' - run: | - python3 -m pip install google-cloud-storage==2.9.0 - scripts/ci/render_bench.py sizes \ - --after $(date -d"180 days ago" +%Y-%m-%d) \ - --output "gs://rerun-builds/graphs" diff --git a/.github/workflows/reusable_update_pr_body.yml b/.github/workflows/reusable_update_pr_body.yml deleted file mode 100644 index 7708107925d9..000000000000 --- a/.github/workflows/reusable_update_pr_body.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Update links in PR Body - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - PR_NUMBER: - required: true - type: string - -concurrency: - group: ${{ inputs.CONCURRENCY }}-pr-update-body - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - pull-requests: "write" - -jobs: - update-pr-body: - name: Update PR body - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.25.0 - - - name: Update PR description - run: | - pixi run ./scripts/ci/update_pr_body.py \ - --github-token '${{ secrets.GITHUB_TOKEN }}' \ - --github-repository '${{ github.repository }}' \ - --pr-number '${{ inputs.PR_NUMBER }}' diff --git a/.github/workflows/reusable_upload_examples.yml b/.github/workflows/reusable_upload_examples.yml deleted file mode 100644 index 1f7b35ce6e50..000000000000 --- a/.github/workflows/reusable_upload_examples.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: Reusable Upload Examples - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - ADHOC_NAME: - type: string - required: false - default: "" - MARK_TAGGED_VERSION: - required: false - type: boolean - default: false - RELEASE_VERSION: - required: false - type: string - default: "prerelease" - PR_NUMBER: - type: string - default: "" - NIGHTLY: - required: false - type: boolean - default: false - -concurrency: - group: ${{ inputs.CONCURRENCY }}-upload-examples - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - upload-web: - name: Upload Examples to Google Cloud - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Download assets - uses: actions/download-artifact@v4 - with: - name: example_data - path: example_data - - # Upload the wasm, html etc to a Google cloud bucket: - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: Get sha - id: get-sha - run: | - full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: "Upload examples (commit)" - if: ${{ !inputs.NIGHTLY }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "example_data" - destination: "rerun-web-viewer/commit/${{ steps.get-sha.outputs.sha }}/examples" - parent: false - process_gcloudignore: false - - - name: "Upload examples (tagged)" - if: inputs.MARK_TAGGED_VERSION - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "example_data" - destination: "rerun-web-viewer/version/${{ inputs.RELEASE_VERSION }}/examples" - parent: false - process_gcloudignore: false - - - name: "Upload examples (adhoc)" - if: ${{ inputs.ADHOC_NAME != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "example_data" - destination: "rerun-web-viewer/adhoc/${{ inputs.ADHOC_NAME }}/examples" - parent: false - process_gcloudignore: false - - - name: "Upload examples (prerelease)" - if: github.ref == 'refs/heads/main' - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "example_data" - destination: "rerun-web-viewer/prerelease/examples" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 - - - name: "Upload examples (main)" - if: github.ref == 'refs/heads/main' - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "example_data" - destination: "rerun-web-viewer/version/main/examples" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 - - - name: "Upload examples (pr)" - if: ${{ inputs.PR_NUMBER != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "example_data" - destination: "rerun-web-viewer/pr/${{ inputs.PR_NUMBER }}/examples" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 - - - name: "Upload examples (nightly)" - if: ${{ inputs.NIGHTLY }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "example_data" - destination: "rerun-web-viewer/version/nightly/examples" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml deleted file mode 100644 index 91df5ae54053..000000000000 --- a/.github/workflows/reusable_upload_web.yml +++ /dev/null @@ -1,139 +0,0 @@ -name: Reusable Upload Web - -on: - workflow_call: - inputs: - CONCURRENCY: - required: true - type: string - ADHOC_NAME: - type: string - required: false - default: "" - MARK_TAGGED_VERSION: - required: false - type: boolean - default: false - RELEASE_VERSION: - required: false - type: string - default: "prerelease" - PR_NUMBER: - required: false - type: string - default: "" - NIGHTLY: - required: false - type: boolean - default: false - -concurrency: - group: ${{ inputs.CONCURRENCY }}-upload-web - cancel-in-progress: true - -defaults: - run: - shell: bash - -permissions: - contents: "read" - id-token: "write" - -jobs: - upload-web: - name: Upload web build to google cloud (wasm32 + wasm-bindgen) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }} - - - name: Download Web Viewer - uses: actions/download-artifact@v4 - with: - name: web_viewer - path: web_viewer - - # Upload the wasm, html etc to a Google cloud bucket: - - id: "auth" - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} - - - name: Get sha - id: get-sha - run: | - full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" - echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: "Upload web-viewer (commit)" - if: ${{ !inputs.NIGHTLY }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/commit/${{ steps.get-sha.outputs.sha }}" - parent: false - process_gcloudignore: false - - - name: "Upload web-viewer (tagged)" - if: inputs.MARK_TAGGED_VERSION - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/version/${{inputs.RELEASE_VERSION}}" - parent: false - process_gcloudignore: false - - - name: "Upload web-viewer (adhoc)" - if: ${{ inputs.ADHOC_NAME != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/adhoc/${{inputs.ADHOC_NAME}}" - parent: false - process_gcloudignore: false - - - name: "Upload web-viewer (prerelease)" - if: github.ref == 'refs/heads/main' - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/prerelease" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 - - - name: "Upload web-viewer (main)" - if: github.ref == 'refs/heads/main' - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/version/main" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 - - - name: "Upload web-viewer (pr)" - if: ${{ inputs.PR_NUMBER != '' }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/pr/${{ inputs.PR_NUMBER }}" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 - - - name: "Upload web-viewer (nightly)" - if: ${{ inputs.NIGHTLY }} - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: "web_viewer" - destination: "rerun-web-viewer/version/nightly" - parent: false - process_gcloudignore: false - headers: |- - cache-control: no-cache, max-age=0 diff --git a/Cargo.lock b/Cargo.lock index 07f02659d20c..7034175be765 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1086,12 +1086,44 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "codspeed" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a104ac948e0188b921eb3fcbdd55dcf62e542df4c7ab7e660623f6288302089" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "722c36bdc62d9436d027256ce2627af81ac7a596dfc7d13d849d0d212448d7fe" +dependencies = [ + "codspeed", + "colored", + "criterion", +] + [[package]] name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "com" version = "0.6.0" @@ -1261,9 +1293,9 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f9c16c823fba76d9643cc387e9677d9771abe0827561381815215c47f808da9" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", "cast", @@ -4302,7 +4334,7 @@ dependencies = [ "ahash", "anyhow", "backtrace", - "criterion", + "codspeed-criterion-compat", "crossbeam", "document-features", "itertools 0.13.0", @@ -4333,7 +4365,7 @@ version = "0.18.0-alpha.1+dev" dependencies = [ "ahash", "anyhow", - "criterion", + "codspeed-criterion-compat", "document-features", "indent", "insta", @@ -4508,7 +4540,7 @@ version = "0.18.0-alpha.1+dev" dependencies = [ "ahash", "anyhow", - "criterion", + "codspeed-criterion-compat", "document-features", "emath", "getrandom", @@ -4564,7 +4596,7 @@ dependencies = [ name = "re_int_histogram" version = "0.18.0-alpha.1+dev" dependencies = [ - "criterion", + "codspeed-criterion-compat", "insta", "mimalloc", "smallvec", @@ -4588,7 +4620,7 @@ dependencies = [ name = "re_log_encoding" version = "0.18.0-alpha.1+dev" dependencies = [ - "criterion", + "codspeed-criterion-compat", "ehttp", "js-sys", "lz4_flex", @@ -4618,7 +4650,7 @@ dependencies = [ "anyhow", "backtrace", "clean-path", - "criterion", + "codspeed-criterion-compat", "crossbeam", "document-features", "fixed", @@ -4688,7 +4720,7 @@ dependencies = [ "ahash", "anyhow", "backtrace", - "criterion", + "codspeed-criterion-compat", "indent", "indexmap 2.1.0", "itertools 0.13.0", @@ -4946,7 +4978,7 @@ dependencies = [ "anyhow", "bitflags 2.5.0", "bytemuck", - "criterion", + "codspeed-criterion-compat", "egui", "glam", "hexasphere", @@ -5109,7 +5141,7 @@ dependencies = [ name = "re_tuid" version = "0.18.0-alpha.1+dev" dependencies = [ - "criterion", + "codspeed-criterion-compat", "document-features", "getrandom", "once_cell", @@ -5202,7 +5234,7 @@ dependencies = [ "anyhow", "backtrace", "bytemuck", - "criterion", + "codspeed-criterion-compat", "document-features", "itertools 0.13.0", "nohash-hasher", diff --git a/Cargo.toml b/Cargo.toml index c5333aa85ebd..2abb20e3317e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,7 +155,7 @@ clean-path = "0.2" comfy-table = { version = "7.0", default-features = false } console_error_panic_hook = "0.1.6" convert_case = "0.6" -criterion = "0.5" +criterion = { package = "codspeed-criterion-compat", version = "2.6.0" } crossbeam = "0.8" directories = "5" document-features = "0.2.8" diff --git a/crates/store/re_query/benches/latest_at.rs b/crates/store/re_query/benches/latest_at.rs index 521d48f42c67..af5b5f1d39a8 100644 --- a/crates/store/re_query/benches/latest_at.rs +++ b/crates/store/re_query/benches/latest_at.rs @@ -31,10 +31,10 @@ mod constants { #[cfg(not(debug_assertions))] mod constants { - pub const NUM_FRAMES_POINTS: u32 = 1_000; - pub const NUM_POINTS: u32 = 1_000; - pub const NUM_FRAMES_STRINGS: u32 = 1_000; - pub const NUM_STRINGS: u32 = 1_000; + pub const NUM_FRAMES_POINTS: u32 = 100; + pub const NUM_POINTS: u32 = 100; + pub const NUM_FRAMES_STRINGS: u32 = 100; + pub const NUM_STRINGS: u32 = 100; } #[allow(clippy::wildcard_imports)]