From e395a89bfe7be11db72398e14c29d6f6de8bd3ea Mon Sep 17 00:00:00 2001 From: Noa Oved <104720318+noaov1@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:39:05 +0300 Subject: [PATCH] Merge main-v0.13.2 into main (#981) * feat: add publish crates script (#893) * feat: implement unpublished crates verification (#894) * chore(ci): tune boostraping cache (#949) * chore(infra): add bootstrap flow to the ci (#941) * Merge remote-tracking branch 'origin/main-v0.13.2' into noa/merge-main-v0.13.2-into-main-1727176225 * No conflicts in main-v0.13.2 -> main merge, this commit is for any change needed to pass the CI. Co-Authored-By: Nadin Jbara <93648739+nadin-Starkware@users.noreply.github.com> Co-Authored-By: alon-dotan-starkware --- .github/actions/bootstrap/action.yml | 5 + .github/actions/install_rust/action.yml | 5 + .github/workflows/blockifier_ci.yml | 4 +- .../workflows/blockifier_compiled_cairo.yml | 2 +- .github/workflows/blockifier_post-merge.yml | 2 +- .github/workflows/committer_ci.yml | 8 +- .github/workflows/main.yml | 10 +- .github/workflows/papyrus_ci.yml | 10 +- .../workflows/papyrus_nightly-tests-call.yml | 2 +- .github/workflows/papyrus_nightly-tests.yml | 14 +-- .github/workflows/verify-deps.yml | 2 +- scripts/publish_crates.py | 115 ++++++++++++++++++ 12 files changed, 152 insertions(+), 27 deletions(-) create mode 100644 .github/actions/bootstrap/action.yml create mode 100644 scripts/publish_crates.py diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml new file mode 100644 index 0000000000..b8366c9836 --- /dev/null +++ b/.github/actions/bootstrap/action.yml @@ -0,0 +1,5 @@ +runs: + using: "composite" + steps: + - name: Install rust. + uses: ./.github/actions/install_rust diff --git a/.github/actions/install_rust/action.yml b/.github/actions/install_rust/action.yml index bf6287cdbe..c81c5ac569 100644 --- a/.github/actions/install_rust/action.yml +++ b/.github/actions/install_rust/action.yml @@ -1,4 +1,9 @@ +name: Bootsrap rust installation +description: Setup rust environment and its components, also caching the build results. + runs: using: "composite" steps: - uses: moonrepo/setup-rust@v1 + with: + cache-base: main(-v[0-9].*)? diff --git a/.github/workflows/blockifier_ci.yml b/.github/workflows/blockifier_ci.yml index 99d593fd5a..b7857358ee 100644 --- a/.github/workflows/blockifier_ci.yml +++ b/.github/workflows/blockifier_ci.yml @@ -42,7 +42,7 @@ jobs: runs-on: starkware-ubuntu-20-04-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: cargo build -p blockifier - run: cargo test -p blockifier @@ -50,7 +50,7 @@ jobs: runs-on: starkware-ubuntu-20-04-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - name: Build native blockifier run: ./build_native_in_docker.sh scripts/build_native_blockifier.sh diff --git a/.github/workflows/blockifier_compiled_cairo.yml b/.github/workflows/blockifier_compiled_cairo.yml index 8c6bb97f24..916837c897 100644 --- a/.github/workflows/blockifier_compiled_cairo.yml +++ b/.github/workflows/blockifier_compiled_cairo.yml @@ -35,7 +35,7 @@ jobs: runs-on: starkware-ubuntu-20-04-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap # Setup pypy and link to the location expected by .cargo/config.toml. - uses: actions/setup-python@v5 diff --git a/.github/workflows/blockifier_post-merge.yml b/.github/workflows/blockifier_post-merge.yml index c20135acfa..1afff5e93e 100644 --- a/.github/workflows/blockifier_post-merge.yml +++ b/.github/workflows/blockifier_post-merge.yml @@ -15,7 +15,7 @@ jobs: runs-on: starkware-ubuntu-20-04-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - uses: Noelware/setup-protoc@1.1.0 # Setup pypy and link to the location expected by .cargo/config.toml. diff --git a/.github/workflows/committer_ci.yml b/.github/workflows/committer_ci.yml index 083728c774..358c17f1c7 100644 --- a/.github/workflows/committer_ci.yml +++ b/.github/workflows/committer_ci.yml @@ -39,11 +39,11 @@ concurrency: jobs: run-regression-tests: - runs-on: starkware-ubuntu-latest-small + runs-on: starkware-ubuntu-latest-medium if: ${{ github.event_name == 'pull_request' }} steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - id: auth uses: "google-github-actions/auth@v2" @@ -55,7 +55,7 @@ jobs: - run: cargo test -p committer_cli --release -- --include-ignored test_regression benchmarking: - runs-on: starkware-ubuntu-latest-small + runs-on: starkware-ubuntu-latest-medium if: ${{ github.event_name == 'pull_request' }} steps: # Checkout the base branch to get the old code. @@ -123,7 +123,7 @@ jobs: runs-on: starkware-ubuntu-20-04-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap # Commit hash on pull request event would be the head commit of the branch. - name: Get commit hash prefix for PR update diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c29c0b7b3f..1de2c3603b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,7 +68,7 @@ jobs: run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV # Install rust components. - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap # Run tests. - name: "Run rustfmt and clippy" @@ -88,10 +88,10 @@ jobs: run-workspace-tests: - runs-on: starkware-ubuntu-latest-small + runs-on: starkware-ubuntu-latest-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: cargo test -p workspace_tests run-tests: @@ -101,7 +101,7 @@ jobs: with: # Fetch the entire history. fetch-depth: 0 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - uses: Noelware/setup-protoc@1.1.0 # Setup pypy and link to the location expected by .cargo/config.toml. @@ -166,7 +166,7 @@ jobs: with: # Fetch the entire history. fetch-depth: 0 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - name: Set-Up run: | sudo apt-get update diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml index e5218b4eb6..19e1b2199d 100644 --- a/.github/workflows/papyrus_ci.yml +++ b/.github/workflows/papyrus_ci.yml @@ -46,7 +46,7 @@ jobs: runs-on: starkware-ubuntu-latest-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - uses: Noelware/setup-protoc@1.1.0 with: version: ${{env.PROTOC_VERSION}} @@ -64,7 +64,7 @@ jobs: runs-on: starkware-ubuntu-latest-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - uses: Noelware/setup-protoc@1.1.0 with: version: ${{env.PROTOC_VERSION}} @@ -100,7 +100,7 @@ jobs: runs-on: starkware-ubuntu-latest-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - uses: Noelware/setup-protoc@1.1.0 with: version: ${{env.PROTOC_VERSION}} @@ -116,7 +116,7 @@ jobs: runs-on: starkware-ubuntu-latest-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - uses: Noelware/setup-protoc@1.1.0 - run: | @@ -163,7 +163,7 @@ jobs: filters: | target_directory: - 'crates/papyrus_storage/src/db/**' - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap # repeat this job 32 times. this is a random test for part of the code that may cause a corrupted database. - run: for run in {1..32}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done if: steps.changes.outputs.target_directory == 'true' diff --git a/.github/workflows/papyrus_nightly-tests-call.yml b/.github/workflows/papyrus_nightly-tests-call.yml index 3e11b21b1e..d05d1938e0 100644 --- a/.github/workflows/papyrus_nightly-tests-call.yml +++ b/.github/workflows/papyrus_nightly-tests-call.yml @@ -21,7 +21,7 @@ jobs: runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: sudo apt update; sudo apt -y install libclang-dev # Install libclang-dev that is not a part of the ubuntu vm in github actions. if: runner.os == 'Linux' diff --git a/.github/workflows/papyrus_nightly-tests.yml b/.github/workflows/papyrus_nightly-tests.yml index af5f385781..927dcb9b7b 100644 --- a/.github/workflows/papyrus_nightly-tests.yml +++ b/.github/workflows/papyrus_nightly-tests.yml @@ -13,7 +13,7 @@ jobs: GW-integration-test-ubuntu: uses: ./.github/workflows/papyrus_nightly-tests-call.yml with: - os: starkware-ubuntu-latest-small + os: starkware-ubuntu-latest-medium secrets: INTEGRATION_TESTNET_NODE_URL: ${{ secrets.INTEGRATION_TESTNET_NODE_URL }} INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: ${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }} @@ -37,7 +37,7 @@ jobs: if: github.event.schedule == '30 0 * * *' steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: mkdir data @@ -56,7 +56,7 @@ jobs: if: github.event.schedule == '30 0 * * *' steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: npm install -g ganache@7.4.3 - run: brew install protobuf@$PROTOC_VERSION @@ -71,7 +71,7 @@ jobs: if: github.event.schedule == '30 0 * * *' steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: cargo build -r -p papyrus_load_test integration-test: @@ -79,7 +79,7 @@ jobs: if: github.event.schedule == '30 0 * * *' steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: > cargo test -r --test latency_histogram @@ -91,8 +91,8 @@ jobs: # TODO(dvir): make this run only if the path 'crates/papyrus_storage/src/db/**' (same path as in the CI) was changed on the # last day and increase the number of repetitions. random-table-test: - runs-on: starkware-ubuntu-latest-small + runs-on: starkware-ubuntu-latest-medium steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - run: for run in {1..100}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done diff --git a/.github/workflows/verify-deps.yml b/.github/workflows/verify-deps.yml index e23ac29496..8582cb1b6b 100644 --- a/.github/workflows/verify-deps.yml +++ b/.github/workflows/verify-deps.yml @@ -11,7 +11,7 @@ jobs: continue-on-error: true steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/install_rust + - uses: ./.github/actions/bootstrap - name: Update Dependencies run: cargo update --verbose - name: Build diff --git a/scripts/publish_crates.py b/scripts/publish_crates.py new file mode 100644 index 0000000000..1111181561 --- /dev/null +++ b/scripts/publish_crates.py @@ -0,0 +1,115 @@ +import argparse +import subprocess +import sys +from typing import List, Tuple +import toml +import json +import asyncio + + +async def crate_version_exists(crate_name: str, version: str) -> bool: + + response = subprocess.run( + ["curl", "-s", f"https://crates.io/api/v1/crates/{crate_name}"], + capture_output=True, + text=True, + ) + + crate_metadata = json.loads(response.stdout) + already_published = version in [v["num"] for v in crate_metadata["versions"]] + print( + f"Crate {crate_name} version {version} " + f"{'exists' if already_published else 'does not exist'} on crates.io" + ) + return already_published + + +def get_workspace_version() -> str: + try: + cargo_data = toml.load("Cargo.toml") + + return cargo_data["workspace"]["package"]["version"] + except (KeyError, TypeError): + raise ValueError("Version key not found in Cargo.toml") + + +async def verify_unpublished(crates: List[str]): + """ + Asserts that none of the crates in the set have been published. + """ + + version = get_workspace_version() + + tasks = [crate_version_exists(crate_name=crate, version=version) for crate in crates] + results = await asyncio.gather(*tasks) + + already_published = [crate for crate, unpublished in zip(crates, results) if unpublished] + + assert ( + not already_published + ), f"Crates {already_published} have already been published with {version=}." + + +def get_package_and_dependencies_in_order(crate: str) -> List[str]: + """ + Returns a list of all local (member) crates that the input crate depends on, in topological + order. I.e, if crate A depends on crate B, then B will appear before A in the list. + The output list also includes the input crate (last element). + """ + # We use the `depth` prefix to easily sort the dependencies in topological order: higher depth + # means the crate is depended on by the crate at the lower depth. + prefixed_tree = ( + subprocess.check_output(["cargo", "tree", "-p", crate, "--prefix", "depth"]) + .decode() + .splitlines() + ) + # Clean up the lines to only keep the *local* crate names with their depth prefix. + # Skip all non-local crates ('(/home' should appear in lines describing local crates). + prefixed_local_crates = [line.split()[0].strip() for line in prefixed_tree if "(/home" in line] + + # Reverse order to iterate in descending depth order. + ordered_dependencies = [] + for dependency_with_depth in reversed(sorted(prefixed_local_crates)): + # Strip the leading depth number (package names do not start with integers). + dependency = dependency_with_depth.lstrip("0123456789") + # The same package may appear multiple times, and with different depths. Always keep the + # highest depth only. + if dependency not in ordered_dependencies: + ordered_dependencies.append(dependency) + return ordered_dependencies + + +async def publish_crate_and_dependencies(crate: str, dry_run: bool): + dependencies = get_package_and_dependencies_in_order(crate=crate) + assert crate == dependencies[-1], f"{crate} should be the last element of '{dependencies}'." + + # Do not attempt to publish anything if even one of the dependencies is already published. + await verify_unpublished(crates=dependencies) + + base_command_template = "cargo publish -p {crate}" + f"{' --dry-run' if dry_run else ''}" + # Publish order is important. + cmd = " && ".join( + [base_command_template.format(crate=dependency) for dependency in dependencies] + ) + + print(f"Publishing {crate} ({dry_run=}) and its dependencies: {dependencies}...") + print(cmd, flush=True) + subprocess.run(cmd, check=True, shell=True) + print(f"Done.") + + +async def main(): + parser = argparse.ArgumentParser( + description="Publish a crate and it's dependencies in the local workspace." + ) + parser.add_argument( + "--crate", type=str, help="Crate to publish (dependencies will also be published)." + ) + parser.add_argument("--dry_run", required=False, action="store_true", help="Dry run.") + args = parser.parse_args() + + await publish_crate_and_dependencies(crate=args.crate, dry_run=args.dry_run) + + +if __name__ == "__main__": + sys.exit(asyncio.run(main()))