From 6224bacc28b46aae75bc05cb385ae94843adcc67 Mon Sep 17 00:00:00 2001
From: Nadin Jbara <93648739+nadin-Starkware@users.noreply.github.com>
Date: Mon, 23 Sep 2024 18:43:08 +0300
Subject: [PATCH 1/5] feat: add publish crates script (#893)

---
 scripts/publish_crates.py | 75 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 scripts/publish_crates.py

diff --git a/scripts/publish_crates.py b/scripts/publish_crates.py
new file mode 100644
index 0000000000..8fb30c8942
--- /dev/null
+++ b/scripts/publish_crates.py
@@ -0,0 +1,75 @@
+import argparse
+import subprocess
+from typing import List, Set
+
+
+def verify_unpublished(crates: List[str]):
+    """
+    Asserts that none of the crates in the set have been published.
+    """
+    raise NotImplementedError("Not implemented yet.")
+
+
+def get_package_and_dependencies_in_order(crate: str) -> List[str]:
+    """
+    Returns a list of all local (member) crates that the input crate depends on, in topological
+    order. I.e, if crate A depends on crate B, then B will appear before A in the list.
+    The output list also includes the input crate (last element).
+    """
+    # We use the `depth` prefix to easily sort the dependencies in topological order: higher depth
+    # means the crate is depended on by the crate at the lower depth.
+    prefixed_tree = (
+        subprocess.check_output(["cargo", "tree", "-p", crate, "--prefix", "depth"])
+        .decode()
+        .splitlines()
+    )
+    # Clean up the lines to only keep the *local* crate names with their depth prefix.
+    # Skip all non-local crates ('(/home' should appear in lines describing local crates).
+    prefixed_local_crates = [line.split()[0].strip() for line in prefixed_tree if "(/home" in line]
+
+    # Reverse order to iterate in descending depth order.
+    ordered_dependencies = []
+    for dependency_with_depth in reversed(sorted(prefixed_local_crates)):
+        # Strip the leading depth number (package names do not start with integers).
+        dependency = dependency_with_depth.lstrip("0123456789")
+        # The same package may appear multiple times, and with different depths. Always keep the
+        # highest depth only.
+        if dependency not in ordered_dependencies:
+            ordered_dependencies.append(dependency)
+    return ordered_dependencies
+
+
+def publish_crate_and_dependencies(crate: str, dry_run: bool):
+    dependencies = get_package_and_dependencies_in_order(crate=crate)
+    assert crate == dependencies[-1], f"{crate} should be the last element of '{dependencies}'."
+
+    # Do not attempt to publish anything if even one of the dependencies is already published.
+    verify_unpublished(crates=dependencies)
+
+    base_command_template = "cargo publish -p {crate}" + f"{' --dry-run' if dry_run else ''}"
+    # Publish order is important.
+    cmd = " && ".join(
+        [base_command_template.format(crate=dependency) for dependency in dependencies]
+    )
+
+    print(f"Publishing {crate} ({dry_run=}) and its dependencies: {dependencies}...")
+    print(cmd, flush=True)
+    subprocess.run(cmd, check=True, shell=True)
+    print(f"Done.")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish a crate and it's dependencies in the local workspace."
+    )
+    parser.add_argument(
+        "--crate", type=str, help="Crate to publish (dependencies will also be published)."
+    )
+    parser.add_argument("--dry_run", required=False, action="store_true", help="Dry run.")
+    args = parser.parse_args()
+
+    publish_crate_and_dependencies(crate=args.crate, dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+    main()

From bac70cc422857046667d6f0229f936e80c57c873 Mon Sep 17 00:00:00 2001
From: Nadin Jbara <93648739+nadin-Starkware@users.noreply.github.com>
Date: Mon, 23 Sep 2024 18:55:26 +0300
Subject: [PATCH 2/5] feat: implement unpublished crates verification (#894)

---
 scripts/publish_crates.py | 56 +++++++++++++++++++++++++++++++++------
 1 file changed, 48 insertions(+), 8 deletions(-)

diff --git a/scripts/publish_crates.py b/scripts/publish_crates.py
index 8fb30c8942..1111181561 100644
--- a/scripts/publish_crates.py
+++ b/scripts/publish_crates.py
@@ -1,13 +1,53 @@
 import argparse
 import subprocess
-from typing import List, Set
+import sys
+from typing import List, Tuple
+import toml
+import json
+import asyncio
 
 
-def verify_unpublished(crates: List[str]):
+async def crate_version_exists(crate_name: str, version: str) -> bool:
+
+    response = subprocess.run(
+        ["curl", "-s", f"https://crates.io/api/v1/crates/{crate_name}"],
+        capture_output=True,
+        text=True,
+    )
+
+    crate_metadata = json.loads(response.stdout)
+    already_published = version in [v["num"] for v in crate_metadata["versions"]]
+    print(
+        f"Crate {crate_name} version {version} "
+        f"{'exists' if already_published else 'does not exist'} on crates.io"
+    )
+    return already_published
+
+
+def get_workspace_version() -> str:
+    try:
+        cargo_data = toml.load("Cargo.toml")
+
+        return cargo_data["workspace"]["package"]["version"]
+    except (KeyError, TypeError):
+        raise ValueError("Version key not found in Cargo.toml")
+
+
+async def verify_unpublished(crates: List[str]):
     """
     Asserts that none of the crates in the set have been published.
     """
-    raise NotImplementedError("Not implemented yet.")
+
+    version = get_workspace_version()
+
+    tasks = [crate_version_exists(crate_name=crate, version=version) for crate in crates]
+    results = await asyncio.gather(*tasks)
+
+    already_published = [crate for crate, unpublished in zip(crates, results) if unpublished]
+
+    assert (
+        not already_published
+    ), f"Crates {already_published} have already been published with {version=}."
 
 
 def get_package_and_dependencies_in_order(crate: str) -> List[str]:
@@ -39,12 +79,12 @@ def get_package_and_dependencies_in_order(crate: str) -> List[str]:
     return ordered_dependencies
 
 
-def publish_crate_and_dependencies(crate: str, dry_run: bool):
+async def publish_crate_and_dependencies(crate: str, dry_run: bool):
     dependencies = get_package_and_dependencies_in_order(crate=crate)
     assert crate == dependencies[-1], f"{crate} should be the last element of '{dependencies}'."
 
     # Do not attempt to publish anything if even one of the dependencies is already published.
-    verify_unpublished(crates=dependencies)
+    await verify_unpublished(crates=dependencies)
 
     base_command_template = "cargo publish -p {crate}" + f"{' --dry-run' if dry_run else ''}"
     # Publish order is important.
@@ -58,7 +98,7 @@ def publish_crate_and_dependencies(crate: str, dry_run: bool):
     print(f"Done.")
 
 
-def main():
+async def main():
     parser = argparse.ArgumentParser(
         description="Publish a crate and it's dependencies in the local workspace."
     )
@@ -68,8 +108,8 @@ def main():
     parser.add_argument("--dry_run", required=False, action="store_true", help="Dry run.")
     args = parser.parse_args()
 
-    publish_crate_and_dependencies(crate=args.crate, dry_run=args.dry_run)
+    await publish_crate_and_dependencies(crate=args.crate, dry_run=args.dry_run)
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(asyncio.run(main()))

From b212a7dd5dc553c1cf79346d7c08fd79c301cf69 Mon Sep 17 00:00:00 2001
From: alon-dotan-starkware <alon.dotan@starkware.co>
Date: Tue, 24 Sep 2024 13:28:11 +0300
Subject: [PATCH 3/5] chore(ci): tune boostraping cache (#949)

---
 .github/actions/install_rust/action.yml     | 5 +++++
 .github/workflows/committer_ci.yml          | 4 ++--
 .github/workflows/main.yml                  | 2 +-
 .github/workflows/papyrus_nightly-tests.yml | 4 ++--
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/actions/install_rust/action.yml b/.github/actions/install_rust/action.yml
index bf6287cdbe..c81c5ac569 100644
--- a/.github/actions/install_rust/action.yml
+++ b/.github/actions/install_rust/action.yml
@@ -1,4 +1,9 @@
+name: Bootsrap rust installation
+description: Setup rust environment and its components, also caching the build results.
+
 runs:
   using: "composite"
   steps:
     - uses: moonrepo/setup-rust@v1
+      with:
+        cache-base: main(-v[0-9].*)?
diff --git a/.github/workflows/committer_ci.yml b/.github/workflows/committer_ci.yml
index 5dbfa34f4d..41710d7248 100644
--- a/.github/workflows/committer_ci.yml
+++ b/.github/workflows/committer_ci.yml
@@ -37,7 +37,7 @@ concurrency:
 
 jobs:
   run-regression-tests:
-    runs-on: starkware-ubuntu-latest-small
+    runs-on: starkware-ubuntu-latest-medium
     if: ${{ github.event_name == 'pull_request' }}
     steps:
       - uses: actions/checkout@v4
@@ -53,7 +53,7 @@ jobs:
       - run: cargo test -p committer_cli --release -- --include-ignored test_regression
 
   benchmarking:
-    runs-on: starkware-ubuntu-latest-small
+    runs-on: starkware-ubuntu-latest-medium
     if: ${{ github.event_name == 'pull_request' }}
     steps:
       # Checkout the base branch to get the old code.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c29c0b7b3f..e8f663cd85 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -88,7 +88,7 @@ jobs:
 
 
   run-workspace-tests:
-    runs-on: starkware-ubuntu-latest-small
+    runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
diff --git a/.github/workflows/papyrus_nightly-tests.yml b/.github/workflows/papyrus_nightly-tests.yml
index af5f385781..e28179ab27 100644
--- a/.github/workflows/papyrus_nightly-tests.yml
+++ b/.github/workflows/papyrus_nightly-tests.yml
@@ -13,7 +13,7 @@ jobs:
   GW-integration-test-ubuntu:
     uses: ./.github/workflows/papyrus_nightly-tests-call.yml
     with:
-      os: starkware-ubuntu-latest-small
+      os: starkware-ubuntu-latest-medium
     secrets:
       INTEGRATION_TESTNET_NODE_URL: ${{ secrets.INTEGRATION_TESTNET_NODE_URL }}
       INTEGRATION_TESTNET_SENDER_PRIVATE_KEY: ${{ secrets.INTEGRATION_TESTNET_SENDER_PRIVATE_KEY }}
@@ -91,7 +91,7 @@ jobs:
   # TODO(dvir): make this run only if the path 'crates/papyrus_storage/src/db/**' (same path as in the CI)  was changed on the
   # last day and increase the number of repetitions.
   random-table-test:
-    runs-on: starkware-ubuntu-latest-small
+    runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust

From 009e4b1918268e024779dd26f4861f054bd24a8a Mon Sep 17 00:00:00 2001
From: alon-dotan-starkware <alon.dotan@starkware.co>
Date: Tue, 24 Sep 2024 13:51:13 +0300
Subject: [PATCH 4/5] chore(infra): add bootstrap flow to the ci (#941)

---
 .github/actions/bootstrap/action.yml             |  5 +++++
 .github/workflows/blockifier_ci.yml              |  4 ++--
 .github/workflows/blockifier_compiled_cairo.yml  |  2 +-
 .github/workflows/blockifier_post-merge.yml      |  2 +-
 .github/workflows/committer_ci.yml               |  4 ++--
 .github/workflows/main.yml                       |  8 ++++----
 .github/workflows/papyrus_ci.yml                 | 10 +++++-----
 .github/workflows/papyrus_nightly-tests-call.yml |  2 +-
 .github/workflows/papyrus_nightly-tests.yml      | 10 +++++-----
 .github/workflows/verify-deps.yml                |  2 +-
 10 files changed, 27 insertions(+), 22 deletions(-)
 create mode 100644 .github/actions/bootstrap/action.yml

diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml
new file mode 100644
index 0000000000..b8366c9836
--- /dev/null
+++ b/.github/actions/bootstrap/action.yml
@@ -0,0 +1,5 @@
+runs:
+  using: "composite"
+  steps:
+    - name: Install rust.
+      uses: ./.github/actions/install_rust
diff --git a/.github/workflows/blockifier_ci.yml b/.github/workflows/blockifier_ci.yml
index f64627d15a..9d802227b8 100644
--- a/.github/workflows/blockifier_ci.yml
+++ b/.github/workflows/blockifier_ci.yml
@@ -42,7 +42,7 @@ jobs:
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - run: cargo build -p blockifier
       - run: cargo test -p blockifier
 
@@ -50,7 +50,7 @@ jobs:
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - name: Build native blockifier
         run: ./build_native_in_docker.sh scripts/build_native_blockifier.sh
 
diff --git a/.github/workflows/blockifier_compiled_cairo.yml b/.github/workflows/blockifier_compiled_cairo.yml
index 835e04440a..0d69f5decd 100644
--- a/.github/workflows/blockifier_compiled_cairo.yml
+++ b/.github/workflows/blockifier_compiled_cairo.yml
@@ -31,7 +31,7 @@ jobs:
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
 
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5
diff --git a/.github/workflows/blockifier_post-merge.yml b/.github/workflows/blockifier_post-merge.yml
index c20135acfa..1afff5e93e 100644
--- a/.github/workflows/blockifier_post-merge.yml
+++ b/.github/workflows/blockifier_post-merge.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - uses: Noelware/setup-protoc@1.1.0
 
       # Setup pypy and link to the location expected by .cargo/config.toml.
diff --git a/.github/workflows/committer_ci.yml b/.github/workflows/committer_ci.yml
index 41710d7248..61a7969578 100644
--- a/.github/workflows/committer_ci.yml
+++ b/.github/workflows/committer_ci.yml
@@ -41,7 +41,7 @@ jobs:
     if: ${{ github.event_name == 'pull_request' }}
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
 
       - id: auth
         uses: "google-github-actions/auth@v2"
@@ -121,7 +121,7 @@ jobs:
     runs-on: starkware-ubuntu-20-04-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
 
         # Commit hash on pull request event would be the head commit of the branch.
       - name: Get commit hash prefix for PR update
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e8f663cd85..1de2c3603b 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -68,7 +68,7 @@ jobs:
         run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
 
       # Install rust components.
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
 
       # Run tests.
       - name: "Run rustfmt and clippy"
@@ -91,7 +91,7 @@ jobs:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - run: cargo test -p workspace_tests
 
   run-tests:
@@ -101,7 +101,7 @@ jobs:
         with:
           # Fetch the entire history.
           fetch-depth: 0
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - uses: Noelware/setup-protoc@1.1.0
 
       # Setup pypy and link to the location expected by .cargo/config.toml.
@@ -166,7 +166,7 @@ jobs:
         with:
           # Fetch the entire history.
           fetch-depth: 0
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - name: Set-Up
         run: |
           sudo apt-get update
diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml
index 140d58a146..a33a38dde5 100644
--- a/.github/workflows/papyrus_ci.yml
+++ b/.github/workflows/papyrus_ci.yml
@@ -46,7 +46,7 @@ jobs:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -64,7 +64,7 @@ jobs:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -82,7 +82,7 @@ jobs:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -98,7 +98,7 @@ jobs:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - uses: Noelware/setup-protoc@1.1.0
 
       - run: |
@@ -145,7 +145,7 @@ jobs:
           filters: |
             target_directory:
               - 'crates/papyrus_storage/src/db/**'
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       # repeat this job 32 times. this is a random test for part of the code that may cause a corrupted database.
       - run: for run in {1..32}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done
         if: steps.changes.outputs.target_directory == 'true'
diff --git a/.github/workflows/papyrus_nightly-tests-call.yml b/.github/workflows/papyrus_nightly-tests-call.yml
index 3e11b21b1e..d05d1938e0 100644
--- a/.github/workflows/papyrus_nightly-tests-call.yml
+++ b/.github/workflows/papyrus_nightly-tests-call.yml
@@ -21,7 +21,7 @@ jobs:
     runs-on: ${{ inputs.os }}
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - run: sudo apt update; sudo apt -y install libclang-dev
         # Install libclang-dev that is not a part of the ubuntu vm in github actions.
         if: runner.os == 'Linux'
diff --git a/.github/workflows/papyrus_nightly-tests.yml b/.github/workflows/papyrus_nightly-tests.yml
index e28179ab27..927dcb9b7b 100644
--- a/.github/workflows/papyrus_nightly-tests.yml
+++ b/.github/workflows/papyrus_nightly-tests.yml
@@ -37,7 +37,7 @@ jobs:
     if: github.event.schedule == '30 0 * * *'
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
 
       - run: mkdir data
 
@@ -56,7 +56,7 @@ jobs:
     if: github.event.schedule == '30 0 * * *'
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - run: npm install -g ganache@7.4.3
 
       - run: brew install protobuf@$PROTOC_VERSION
@@ -71,7 +71,7 @@ jobs:
     if: github.event.schedule == '30 0 * * *'
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - run: cargo build -r -p papyrus_load_test
 
   integration-test:
@@ -79,7 +79,7 @@ jobs:
     if: github.event.schedule == '30 0 * * *'
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - run: >
           cargo test -r
           --test latency_histogram
@@ -94,5 +94,5 @@ jobs:
     runs-on: starkware-ubuntu-latest-medium
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - run: for run in {1..100}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done
diff --git a/.github/workflows/verify-deps.yml b/.github/workflows/verify-deps.yml
index e23ac29496..8582cb1b6b 100644
--- a/.github/workflows/verify-deps.yml
+++ b/.github/workflows/verify-deps.yml
@@ -11,7 +11,7 @@ jobs:
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
+      - uses: ./.github/actions/bootstrap
       - name: Update Dependencies
         run: cargo update --verbose
       - name: Build

From 6ac652029c3bbfd97777a00cd3dd8fc0294e97a5 Mon Sep 17 00:00:00 2001
From: Noa Oved <noa@starkware.co>
Date: Tue, 24 Sep 2024 14:10:27 +0300
Subject: [PATCH 5/5] No conflicts in main-v0.13.2 -> main merge, this commit
 is for any change needed to pass the CI.