Skip to content

Commit

Permalink
Merge branch 'master' into giles-sp-std
Browse files Browse the repository at this point in the history
  • Loading branch information
gilescope committed May 5, 2024
2 parents aa3e2f0 + 3836376 commit d5d1fe5
Show file tree
Hide file tree
Showing 1,281 changed files with 25,013 additions and 17,528 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# - Multiple owners are supported.
# - Either handle (e.g, @github_user or @github/team) or email can be used. Keep in mind,
# that handles might work better because they are more recognizable on GitHub,
# eyou can use them for mentioning unlike an email.
# you can use them for mentioning unlike an email.
# - The latest matching rule, if multiple, takes precedence.

# CI
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/common/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ fetch_release_artifacts() {
popd > /dev/null
}

# Fetch the release artifacts like binary and sigantures from S3. Assumes the ENV are set:
# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
# - RELEASE_ID
# - GITHUB_TOKEN
# - REPO in the form paritytech/polkadot
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/check-links.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ name: Check links
on:
pull_request:
paths:
- "*.rs"
- "*.prdoc"
- "**.rs"
- "**.prdoc"
- ".github/workflows/check-links.yml"
- ".config/lychee.toml"
types: [opened, synchronize, reopened, ready_for_review]
Expand Down
54 changes: 54 additions & 0 deletions .github/workflows/subsystem-benchmarks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# The actions takes json file as input and runs github-action-benchmark for it.

on:
workflow_dispatch:
inputs:
benchmark-data-dir-path:
description: "Path to the benchmark data directory"
required: true
type: string
output-file-path:
description: "Path to the benchmark data file"
required: true
type: string

jobs:
subsystem-benchmarks:
runs-on: ubuntu-latest
environment: subsystem-benchmarks
steps:
- name: Validate inputs
run: |
echo "${{ github.event.inputs.benchmark-data-dir-path }}" | grep -P '^[a-z\-]'
echo "${{ github.event.inputs.output-file-path }}" | grep -P '^[a-z\-]+\.json'
- name: Checkout Sources
uses: actions/checkout@v4.1.2
with:
fetch-depth: 0
ref: "gh-pages"

- name: Copy bench results
id: step_one
run: |
cp bench/gitlab/${{ github.event.inputs.output-file-path }} ${{ github.event.inputs.output-file-path }}
- name: Switch branch
id: step_two
run: |
git checkout master --
- uses: actions/create-github-app-token@v1
id: app-token
with:
app-id: ${{ secrets.POLKADOTSDK_GHPAGES_APP_ID }}
private-key: ${{ secrets.POLKADOTSDK_GHPAGES_APP_KEY }}

- name: Store benchmark result
uses: benchmark-action/github-action-benchmark@v1
with:
tool: "customSmallerIsBetter"
output-file-path: ${{ github.event.inputs.output-file-path }}
benchmark-data-dir-path: "bench/${{ github.event.inputs.benchmark-data-dir-path }}"
github-token: ${{ steps.app-token.outputs.token }}
auto-push: true
7 changes: 7 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,13 @@ default:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues

.publish-gh-pages-refs:
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "master"

# handle the specific case where benches could store incorrect bench data because of the downstream staging runs
# exclude cargo-check-benches from such runs
.test-refs-check-benches:
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/pipeline/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ build-runtimes-polkavm:
- .run-immediately
# - .collect-artifact
variables:
# this variable gets overriden by "rusty-cachier environment inject", use the value as default
# this variable gets overridden by "rusty-cachier environment inject", use the value as default
CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
before_script:
- mkdir -p ./artifacts/subkey
Expand Down
76 changes: 70 additions & 6 deletions .gitlab/pipeline/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,13 @@

publish-rustdoc:
stage: publish
extends: .kubernetes-env
extends:
- .kubernetes-env
- .publish-gh-pages-refs
variables:
CI_IMAGE: node:18
GIT_DEPTH: 100
RUSTDOCS_DEPLOY_REFS: "master"
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "master"
needs:
- job: build-rustdoc
artifacts: true
Expand Down Expand Up @@ -60,9 +57,76 @@ publish-rustdoc:
- git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
echo "___Nothing to commit___"
- git push origin gh-pages --force
# artificial sleep to publish gh-pages
- sleep 300
after_script:
- rm -rf .git/ ./*

publish-subsystem-benchmarks:
stage: publish
variables:
CI_IMAGE: "paritytech/tools:latest"
extends:
- .kubernetes-env
- .publish-gh-pages-refs
needs:
- job: subsystem-regression-tests
artifacts: true
- job: publish-rustdoc
artifacts: false
script:
# setup ssh
- eval $(ssh-agent)
- ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
- mkdir ~/.ssh && touch ~/.ssh/known_hosts
- ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
# Set git config
- rm -rf .git/config
- git config user.email "devops-team@parity.io"
- git config user.name "${GITHUB_USER}"
- git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git"
- git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
- git fetch origin gh-pages
# Push result to github
- git checkout gh-pages --force
- mkdir -p bench/gitlab/ || echo "Directory exists"
- rm -rf bench/gitlab/*.json || echo "No json files"
- cp -r charts/*.json bench/gitlab/
- git add bench/gitlab/
- git commit -m "Add json files with benchmark results for ${CI_COMMIT_REF_NAME}"
- git push origin gh-pages
# artificial sleep to publish gh-pages
- sleep 300
allow_failure: true
after_script:
- rm -rf .git/ ./*

trigger_workflow:
stage: deploy
extends:
- .kubernetes-env
- .publish-gh-pages-refs
needs:
- job: publish-subsystem-benchmarks
artifacts: false
- job: subsystem-regression-tests
artifacts: true
script:
- echo "Triggering workflow"
- |
for benchmark in $(ls charts/*.json); do
export bencmark_name=$(basename $benchmark)
echo "Benchmark: $bencmark_name"
export benchmark_dir=$(echo $bencmark_name | sed 's/\.json//')
curl -q -X POST \
-H "Accept: application/vnd.github.v3+json" \
-H "Authorization: token $GITHUB_TOKEN" \
https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/actions/workflows/subsystem-benchmarks.yml/dispatches \
-d '{"ref":"refs/heads/master","inputs":{"benchmark-data-dir-path":"'$benchmark_dir'","output-file-path":"'$bencmark_name'"}}'
sleep 300
done
allow_failure: true

# note: images are used not only in zombienet but also in rococo, wococo and versi
.build-push-image:
image: $BUILDAH_IMAGE
Expand Down
12 changes: 9 additions & 3 deletions .gitlab/pipeline/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ test-linux-stable:
# "upgrade_version_checks_should_work" is currently failing
- |
time cargo nextest run \
--filter-expr 'not deps(/polkadot-subsystem-bench/)' \
--workspace \
--locked \
--release \
Expand Down Expand Up @@ -70,7 +69,7 @@ test-linux-stable-runtime-benchmarks:
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
script:
- time cargo nextest run --filter-expr 'not deps(/polkadot-subsystem-bench/)' --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
- time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet

# can be used to run all tests
# test-linux-stable-all:
Expand Down Expand Up @@ -497,12 +496,19 @@ test-syscalls:

subsystem-regression-tests:
stage: test
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: always
expire_in: 1 days
paths:
- charts/
extends:
- .docker-env
- .common-refs
- .run-immediately
script:
- cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks
- cargo bench --profile=testnet -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks
- cargo bench --profile=testnet -p polkadot-availability-distribution --bench availability-distribution-regression-bench --features subsystem-benchmarks
tags:
- benchmark
allow_failure: true
3 changes: 2 additions & 1 deletion .gitlab/pipeline/zombienet.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
.zombienet-refs:
extends: .build-refs
variables:
ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.91"
ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.99"
PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics"

include:
# substrate tests
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/rust-features.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#
# The steps of this script:
# 1. Check that all required dependencies are installed.
# 2. Check that all rules are fullfilled for the whole workspace. If not:
# 2. Check that all rules are fulfilled for the whole workspace. If not:
# 4. Check all crates to find the offending ones.
# 5. Print all offending crates and exit with code 1.
#
Expand Down
Loading

0 comments on commit d5d1fe5

Please sign in to comment.