[pull] master from paritytech:master

.config/taplo.toml

@@ -33,3 +33,10 @@ keys = ["build"]
 
 [rule.formatting]
 reorder_arrays = false
+
+[[rule]]
+include = ["Cargo.toml"]
+keys = ["workspace.dependencies"]
+
+[rule.formatting]
+reorder_keys = true

.forklift/config-gitlab.toml

@@ -0,0 +1,33 @@
+[compression]
+type = "zstd"
+
+[compression.zstd]
+compressionLevel = 3
+
+[general]
+jobNameVariable = "CI_JOB_NAME"
+jobsBlackList = []
+logLevel = "warn"
+threadsCount = 6
+
+[cache]
+extraEnv = ["RUNTIME_METADATA_HASH"]
+
+[metrics]
+enabled = true
+pushEndpoint = "placeholder"
+
+[metrics.extraLabels]
+environment = "production"
+job_name = "$CI_JOB_NAME"
+project_name = "$CI_PROJECT_PATH"
+
+[storage]
+type = "s3"
+
+[storage.s3]
+accessKeyId = "placeholder"
+bucketName = "placeholder"
+concurrency = 10
+endpointUrl = "placeholder"
+secretAccessKey = "placeholder"

.forklift/config.toml

@@ -23,11 +23,7 @@ job_name = "$CI_JOB_NAME"
 project_name = "$CI_PROJECT_PATH"
 
 [storage]
-type = "s3"
+type = "gcs"
 
-[storage.s3]
-accessKeyId = "placeholder"
-bucketName = "placeholder"
-concurrency = 10
-endpointUrl = "placeholder"
-secretAccessKey = "placeholder"
+[storage.gcs]
+bucketName = "parity-ci-forklift"

.github/actions/set-up-gh/action.yml

@@ -0,0 +1,36 @@
+name: 'install gh'
+description: 'Install the gh cli in a debian based distro and switches to the PR branch.'
+inputs:
+  pr-number:
+    description: "Number of the PR"
+    required: true
+  GH_TOKEN:
+    description: "GitHub token"
+    required: true
+outputs:
+  branch:
+    description: 'Branch name for the PR'
+    value: ${{ steps.branch.outputs.branch }}
+runs:
+  using: "composite"
+  steps:
+      - name: Instal gh cli
+        shell: bash
+        # Here it would get the script from previous step
+        run: |
+          (type -p wget >/dev/null || (apt update && apt-get install wget -y))
+          mkdir -p -m 755 /etc/apt/keyrings
+          wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null 
+          chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          apt update
+          apt install gh -y
+          git config --global --add safe.directory '*'
+      - run: gh pr checkout ${{ inputs.pr-number }}
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ inputs.GH_TOKEN }}
+      - name: Export branch name
+        shell: bash
+        run: echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> "$GITHUB_OUTPUT"
+        id: branch

.github/commands-readme.md

@@ -0,0 +1,200 @@
+# Running commands
+
+Command bot has been migrated, it is no longer a comment parser and now it is a GitHub action that works as a [`workflow_dispatch`](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch) event.
+
+## How to run an action
+
+To run an action, you need to go to the [_actions tab_](https://github.com/paritytech/polkadot-sdk/actions) and pick the one you desire to run.
+
+The current available command actions are:
+
+- [Command FMT](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-fmt.yml)
+- [Command Update UI](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-update-ui.yml)
+- [Command Sync](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-sync.yml)
+- [Command Bench](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench.yml)
+- [Command Bench All](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench-all.yml)
+- [Command Bench Overhead](https://github.com/paritytech/polkadot-sdk/actions/workflows/command-bench-overhead.yml)
+
+You need to select the action, and click on the dropdown that says: `Run workflow`. It is located in the upper right.
+
+If this dropdown is not visible, you may not have permission to run the action. Contact IT for help.
+
+![command screenshot](command-screnshot.png)
+
+Each command will have the same two required values, but it could have more.
+
+GitHub's official documentation: [Manually running a workflow](https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow)
+
+### Number of the Pull Request
+
+The number of the pull request. Required so the action can fetch the correct branch and comment if it fails.
+
+## Action configurations
+
+### Bench
+
+Runs `benchmark pallet` or `benchmark overhead` against your PR and commits back updated weights.
+
+Posible combinations based on the `benchmark` dropdown.
+
+- `substrate-pallet`: Pallet Benchmark for Substrate for specific pallet
+  - Requires `Subcommand` to be `pallet`
+  - Requires `Runtime` to be `dev`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Target Directory` to be `substrate`
+- `polkadot-pallet`: Pallet Benchmark for Polkadot for specific pallet
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one of the following:
+    - `rococo`
+    - `westend`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Target Directory` to be `polkadot`
+- `cumulus-assets`: Pallet Benchmark for Cumulus assets
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one of the following:
+    - `asset-hub-westend`
+    - `asset-hub-rococo`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `assets`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-collectives`: Pallet Benchmark for Cumulus collectives
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be `collectives-westend`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `collectives`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-coretime`: Pallet Benchmark for Cumulus coretime
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one of the following:
+    - `coretime-rococo`
+    - `coretime-westend`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `coretime`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-bridge-hubs`: Pallet Benchmark for Cumulus bridge-hubs
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one of the following:
+    - `bridge-hub-rococo`
+    - `bridge-hub-westend`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `bridge-hub`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-contracts`: Pallet Benchmark for Cumulus contracts
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one `contracts-rococo`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `contracts`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-glutton`: Pallet Benchmark for Cumulus glutton
+  - Requires `Subcommand` to be `pallet`
+  - Requires `Runtime` to be one of the following:
+    - `glutton-westend`
+    - `glutton-westend-dev-1300`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `glutton`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-starters`: Pallet Benchmark for Cumulus starters
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one of the following:
+    - `seedling`
+    - `shell`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `starters`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-people`: Pallet Benchmark for Cumulus people
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one of the following:
+    - `people-westend`
+    - `people-rococo`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `people`
+  - Requires `Target Directory` to be `cumulus`
+- `cumulus-testing`: Pallet Benchmark for Cumulus testing
+  - Requires `Subcommand` to be one of the following:
+    - `pallet`
+    - `xcm`
+  - Requires `Runtime` to be one of the following:
+    - `penpal`
+    - `rococo-parachain`
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+  - Requires `Runtime Dir` to be `testing`
+  - Requires `Target Directory` to be `cumulus`
+
+### Bench-all
+
+This is a wrapper to run `bench` for all pallets.
+
+Posible combinations based on the `benchmark` dropdown.
+
+- `pallet`: Benchmark for Substrate/Polkadot/Cumulus/Trappist for specific pallet
+  - Requires field `Pallet` to have an input that applies to `^([a-z_]+)([:]{2}[a-z_]+)?$`
+- `substrate`: Pallet + Overhead + Machine Benchmark for Substrate for all pallets
+  - Requires `Target Directory` to be `substrate`
+- `polkadot`: Pallet + Overhead Benchmark for Polkadot
+  - Requires `Runtime` to be one of the following:
+    - `rococo`
+    - `westend`
+  - Requires `Target Directory` to be `polkadot`
+- `cumulus`: Pallet Benchmark for Cumulus
+  - Requires `Runtime` to be one of the following:
+    - `rococo`
+    - `westend`
+    - `asset-hub-kusama`
+    - `asset-hub-polkadot`
+    - `asset-hub-rococo`
+    - `asset-hub-westend`
+    - `bridge-hub-kusama`
+    - `bridge-hub-polkadot`
+    - `bridge-hub-rococo`
+    - `bridge-hub-westend`
+    - `collectives-polkadot`
+    - `collectives-westend`
+    - `coretime-rococo`
+    - `coretime-westend`
+    - `contracts-rococo`
+    - `glutton-kusama`
+    - `glutton-westend`
+    - `people-rococo`
+    - `people-westend`
+  - Requires `Target Directory` to be `cumulus`
+
+### Bench-overhead
+
+Run benchmarks overhead and commit back results to PR.
+
+Posible combinations based on the `benchmark` dropdown.
+
+- `default`: Runs `benchmark overhead` and commits back to PR the updated `extrinsic_weights.rs` files
+  - Requires `Runtime` to be one of the following:
+    - `rococo`
+    - `westend`
+  - Requires `Target directory` to be `polkadot`
+- `substrate`: Runs `benchmark overhead` and commits back to PR the updated `extrinsic_weights.rs` files
+  - Requires `Target directory` to be `substrate`
+- `cumulus`: Runs `benchmark overhead` and commits back to PR the updated `extrinsic_weights.rs` files
+  - Requires `Runtime` to be one of the following:
+    - `asset-hub-rococo`
+    - `asset-hub-westend`
+  - Requires `Target directory` to be `cumulus`
+
+## How to modify an action
+
+If you want to modify an action and test it, you can do by simply pushing your changes and then selecting your branch in the `Use worflow from` option.
+
+This will use a file from a specified branch.

.github/env

@@ -1 +1 @@
-IMAGE="docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v20240408"
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v202407161507"

.github/review-bot.yml

@@ -9,6 +9,7 @@ rules:
         - ^\.gitlab/.*
         - ^\.config/nextest.toml
         - ^\.cargo/.*
+        - ^\.forklift/.*
       exclude:
         - ^\.gitlab/pipeline/zombienet.*
     type: "or"
@@ -33,6 +34,7 @@ rules:
         - ^docker/.*
         - ^\.github/.*
         - ^\.gitlab/.*
+        - ^\.forklift/.*
         - ^\.config/nextest.toml
         - ^\.cargo/.*
     minApprovals: 2

.github/scripts/check-workspace.py

@@ -135,8 +135,12 @@ def check_deps(deps):
 				if dep_name in all_crates:
 					links.append((name, dep_name))
 
-					if not 'path' in deps[dep]:
-						broken.append((name, dep_name, "crate must be linked via `path`"))
+					if name == 'polkadot-sdk':
+						if not 'path' in deps[dep]:
+							broken.append((name, dep_name, "crate must use path"))
+							return
+					elif not 'workspace' in deps[dep] or not deps[dep]['workspace']:
+						broken.append((name, dep_name, "crate must use workspace inheritance"))
 						return
 
 		def check_crate(deps):
@@ -154,8 +158,6 @@ def check_crate(deps):
 
 		check_crate(manifest)
 
-
-
 	links.sort()
 	broken.sort()
 

.github/scripts/common/lib.sh

@@ -444,3 +444,16 @@ get_latest_release_tag() {
     latest_release_tag=$(curl -s -H "$TOKEN" $api_base/paritytech/polkadot-sdk/releases/latest | jq -r '.tag_name')
     printf $latest_release_tag
 }
+
+function get_polkadot_node_version_from_code() {
+   # list all the files with node version
+  git grep -e "\(NODE_VERSION[^=]*= \)\".*\"" |
+  # fetch only the one we need
+  grep  "primitives/src/lib.rs:" |
+  # Print only the version
+  awk '{ print $7 }' |
+  # Remove the quotes
+  sed 's/"//g' |
+  # Remove the semicolon
+  sed 's/;//g'
+}

.github/scripts/deny-git-deps.py

@@ -19,6 +19,7 @@
 
 root = sys.argv[1] if len(sys.argv) > 1 else os.getcwd()
 workspace = Workspace.from_path(root)
+errors = []
 
 def check_dep(dep, used_by):
 	if dep.location != DependencyLocation.GIT:
@@ -27,14 +28,23 @@ def check_dep(dep, used_by):
 	if used_by in KNOWN_BAD_GIT_DEPS.get(dep.name, []):
 		print(f'🤨 Ignoring git dependency {dep.name} in {used_by}')
 	else:
-		print(f'🚫 Found git dependency {dep.name} in {used_by}')
-		sys.exit(1)	
+		errors.append(f'🚫 Found git dependency {dep.name} in {used_by}')
 
 # Check the workspace dependencies that can be inherited:
 for dep in workspace.dependencies:
 	check_dep(dep, "workspace")
 
+	if workspace.crates.find_by_name(dep.name):
+		if dep.location != DependencyLocation.PATH:
+			errors.append(f'🚫 Workspace must use path to link local dependency {dep.name}')
+
 # And the dependencies of each crate:
 for crate in workspace.crates:
 	for dep in crate.dependencies:
 		check_dep(dep, crate.name)
+
+if errors:
+	print('❌ Found errors:')
+	for error in errors:
+		print(error)
+	sys.exit(1)