diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a24c04af12..1a2b90083d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,19 +2,19 @@ name: "CodeQL" on: push: - branches: - - "main" - - "v.?[0-9]+.[0-9]+.[0-9]+" - - "v.?[0-9]+.[0-9]+" - - "v?[0-9]+.[0-9]+.[0-9]+" - - "v?[0-9]+.[0-9]+" + branches: + - "main" + - "v.?[0-9]+.[0-9]+.[0-9]+" + - "v.?[0-9]+.[0-9]+" + - "v?[0-9]+.[0-9]+.[0-9]+" + - "v?[0-9]+.[0-9]+" pull_request: - branches: - - "main" - - "v.?[0-9]+.[0-9]+.[0-9]+" - - "v.?[0-9]+.[0-9]+" - - "v?[0-9]+.[0-9]+.[0-9]+" - - "v?[0-9]+.[0-9]+" + branches: + - "main" + - "v.?[0-9]+.[0-9]+.[0-9]+" + - "v.?[0-9]+.[0-9]+" + - "v?[0-9]+.[0-9]+.[0-9]+" + - "v?[0-9]+.[0-9]+" schedule: - cron: "37 18 * * 6" diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 5179d37527..36b380c3e0 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -37,16 +37,16 @@ jobs: load-engine-matrix: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - + - name: Load the engine matrix id: load-engine-matrix shell: bash run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - + run-tests: needs: load-engine-matrix timeout-minutes: 25 @@ -68,7 +68,7 @@ jobs: # RUNNER: macos-latest, # TARGET: aarch64-apple-darwin # } - + runs-on: ${{ matrix.host.RUNNER }} steps: @@ -80,7 +80,7 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ matrix.dotnet }} - + - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: @@ -112,7 +112,7 @@ jobs: benchmarks/results/* utils/clusters/** - # TODO Add amazonlinux +# TODO Add amazonlinux lint-rust: timeout-minutes: 10 diff --git a/.github/workflows/java-cd.yml b/.github/workflows/java-cd.yml index 40d9de0938..79f8971590 100644 --- a/.github/workflows/java-cd.yml +++ b/.github/workflows/java-cd.yml @@ -1,4 +1,4 @@ -name: Create Workflow +name: Java Prepare Deployment on: push: @@ -24,6 +24,7 @@ permissions: jobs: load-platform-matrix: runs-on: ubuntu-latest + environment: AWS_ACTIONS outputs: PLATFORM_MATRIX: ${{ steps.load-platform-matrix.outputs.PLATFORM_MATRIX }} steps: @@ -273,6 +274,20 @@ jobs: brew services stop redis fi + publish-release-to-maven: + if: ${{ inputs.maven_publish == true || github.event_name == 'push' }} + needs: [publish-to-maven-central-deployment] + runs-on: ubuntu-latest + environment: AWS_ACTIONS + env: + DEPLOYMENT_ID: ${{ needs.publish-to-maven-central-deployment.outputs.DEPLOYMENT_ID }} + steps: + - name: Publish to Maven + run: | + curl --request POST \ + -u "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" \ + "https://central.sonatype.com/api/v1/publisher/deployment/${{ env.DEPLOYMENT_ID }}" + drop-deployment-if-validation-fails: if: ${{ failure() }} needs: [publish-to-maven-central-deployment, test-deployment-on-all-architectures] @@ -283,7 +298,6 @@ jobs: - name: Drop deployment if validation fails run: | curl --request DELETE \ - --verbose \ -u "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" \ "https://central.sonatype.com/api/v1/publisher/deployment/${{ env.DEPLOYMENT_ID }}" diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index dca394b591..25db537ab5 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -4,29 +4,29 @@ name: NPM - Continuous Deployment on: pull_request: - paths: - - .github/workflows/npm-cd.yml - - .github/workflows/build-node-wrapper/action.yml - - .github/workflows/start-self-hosted-runner/action.yml - - .github/workflows/install-rust-and-protoc/action.yml - - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + paths: + - .github/workflows/npm-cd.yml + - .github/workflows/build-node-wrapper/action.yml + - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/install-rust-and-protoc/action.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json push: tags: - "v*.*" workflow_dispatch: - inputs: - version: - description: 'The release version of GLIDE, formatted as *.*.* or *.*.*-rc*' - required: true + inputs: + version: + description: 'The release version of GLIDE, formatted as *.*.* or *.*.*-rc*' + required: true concurrency: group: npm-${{ github.head_ref || github.ref }} cancel-in-progress: true permissions: - id-token: write + id-token: write jobs: start-self-hosted-runner: @@ -34,17 +34,17 @@ jobs: runs-on: ubuntu-latest environment: AWS_ACTIONS steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Start self hosted EC2 runner - uses: ./.github/workflows/start-self-hosted-runner - with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Start self hosted EC2 runner + uses: ./.github/workflows/start-self-hosted-runner + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} load-platform-matrix: runs-on: ubuntu-latest @@ -58,9 +58,9 @@ jobs: id: load-platform-matrix shell: bash run: | - # Get the matrix from the matrix.json file, without the object that has the IMAGE key - export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["npm"])))' < .github/json_matrices/build-matrix.json | jq -c .)" - echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT + # Get the matrix from the matrix.json file, without the object that has the IMAGE key + export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["npm"])))' < .github/json_matrices/build-matrix.json | jq -c .)" + echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT publish-binaries: needs: [start-self-hosted-runner, load-platform-matrix] @@ -73,7 +73,7 @@ jobs: strategy: fail-fast: false matrix: - build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} + build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} steps: - name: Setup self-hosted runner access if: ${{ contains(matrix.build.RUNNER, 'self-hosted') && matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} @@ -85,7 +85,7 @@ jobs: run: | apk update apk add git - + - name: Checkout if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} uses: actions/checkout@v4 @@ -100,7 +100,7 @@ jobs: workspace: $GITHUB_WORKSPACE npm-scope: ${{ vars.NPM_SCOPE }} npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} - arch: ${{ matrix.build.ARCH }} + arch: ${{ matrix.build.ARCH }} - name: Set the release version shell: bash @@ -114,8 +114,8 @@ jobs: fi echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV env: - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} - name: Setup node if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} @@ -127,15 +127,15 @@ jobs: scope: "${{ vars.NPM_SCOPE }}" always-auth: true token: ${{ secrets.NPM_AUTH_TOKEN }} - + - name: Setup node for publishing if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-musl' }} working-directory: ./node run: | - npm config set registry https://registry.npmjs.org/ - npm config set '//registry.npmjs.org/:_authToken' ${{ secrets.NPM_AUTH_TOKEN }} - npm config set scope ${{ vars.NPM_SCOPE }} - + npm config set registry https://registry.npmjs.org/ + npm config set '//registry.npmjs.org/:_authToken' ${{ secrets.NPM_AUTH_TOKEN }} + npm config set scope ${{ vars.NPM_SCOPE }} + - name: Update package version in config.toml uses: ./.github/workflows/update-glide-version with: @@ -153,7 +153,7 @@ jobs: publish: "true" github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - + - name: Check if RC and set a distribution tag for the package shell: bash run: | @@ -170,9 +170,9 @@ jobs: - name: Check that the release version dont have typo init if: ${{ github.event_name != 'pull_request' && contains(env.RELEASE_VERSION, '-') && !contains(env.RELEASE_VERSION, 'rc') }} run: | - echo "The release version "${GITHUB_REF:11}" contains a typo, please fix it" - echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release." - exit 1 + echo "The release version "${GITHUB_REF:11}" contains a typo, please fix it" + echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release." + exit 1 - name: Publish to NPM if: github.event_name != 'pull_request' @@ -183,7 +183,7 @@ jobs: set +e # 2>&1 1>&3- redirects stderr to stdout and then redirects the original stdout to another file descriptor, # effectively separating stderr and stdout. The 3>&1 at the end redirects the original stdout back to the console. - # https://github.com/npm/npm/issues/118#issuecomment-325440 - ignoring notice messages since currentlly they are directed to stderr + # https://github.com/npm/npm/issues/118#issuecomment-325440 - ignoring notice messages since currentlly they are directed to stderr { npm_publish_err=$(npm publish --tag ${{ env.NPM_TAG }} --access public 2>&1 1>&3- | grep -v "notice") ;} 3>&1 if [[ "$npm_publish_err" == *"You cannot publish over the previously published versions"* ]] then @@ -202,8 +202,8 @@ jobs: if: ${{ matrix.build.ARCH == 'arm64' }} shell: bash run: | - git reset --hard - git clean -xdf + git reset --hard + git clean -xdf publish-base-to-npm: if: github.event_name != 'pull_request' @@ -247,7 +247,7 @@ jobs: target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - + - name: Check if RC and set a distribution tag for the package shell: bash run: | @@ -260,7 +260,7 @@ jobs: export npm_tag="latest" fi echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV - + - name: Publish the base package if: github.event_name != 'pull_request' shell: bash @@ -285,7 +285,7 @@ jobs: strategy: fail-fast: false matrix: - build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} + build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} steps: - name: Setup self-hosted runner access if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }} @@ -294,20 +294,20 @@ jobs: - name: install Redis and git for alpine if: ${{ contains(matrix.build.TARGET, 'musl') }} run: | - apk update - apk add redis git - node -v - + apk update + apk add redis git + node -v + - name: install Redis and Python for ubuntu if: ${{ contains(matrix.build.TARGET, 'linux-gnu') }} run: | - sudo apt-get update - sudo apt-get install redis-server python3 + sudo apt-get update + sudo apt-get install redis-server python3 - name: install Redis, Python for macos if: ${{ contains(matrix.build.RUNNER, 'mac') }} run: | - brew install redis python3 + brew install redis python3 - name: Checkout if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl'}} @@ -370,5 +370,5 @@ jobs: if: ${{ contains(matrix.build.RUNNER, 'self-hosted') }} shell: bash run: | - git reset --hard - git clean -xdf + git reset --hard + git clean -xdf diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml index 2fe6a45e31..fcea61ee6b 100644 --- a/.github/workflows/ort.yml +++ b/.github/workflows/ort.yml @@ -3,32 +3,32 @@ name: The OSS Review Toolkit (ORT) on: schedule: - - cron: "0 0 * * *" + - cron: "0 0 * * *" pull_request: - paths: - - .github/workflows/ort.yml - - .github/workflows/run-ort-tools/action.yml - - utils/get_licenses_from_ort.py + paths: + - .github/workflows/ort.yml + - .github/workflows/run-ort-tools/action.yml + - utils/get_licenses_from_ort.py workflow_dispatch: - inputs: - branch: - description: 'The branch to run against the ORT tool' - required: true - version: - description: 'The release version of GLIDE' - required: true + inputs: + branch: + description: 'The branch to run against the ORT tool' + required: true + version: + description: 'The release version of GLIDE' + required: true jobs: run-ort: if: github.repository_owner == 'valkey-io' name: Create attribution files runs-on: ubuntu-latest strategy: - fail-fast: false - env: - PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON" - NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" - RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST" - JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA" + fail-fast: false + env: + PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON" + NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" + RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST" + JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA" steps: - name: Set the release version shell: bash @@ -36,17 +36,17 @@ jobs: export version=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo '255.255.255'; else echo "$INPUT_VERSION"; fi` echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV env: - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} - + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} + - name: Set the base branch run: | - export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi` - echo "Base branch is: ${BASE_BRANCH}" - echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV + export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi` + echo "Base branch is: ${BASE_BRANCH}" + echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV env: - EVENT_NAME: ${{ github.event_name }} - INPUT_BRANCH: ${{ github.event.inputs.branch }} + EVENT_NAME: ${{ github.event_name }} + INPUT_BRANCH: ${{ github.event.inputs.branch }} - name: Checkout uses: actions/checkout@v4 @@ -64,73 +64,73 @@ jobs: uses: actions/cache@v4 id: cache-ort with: - path: | - ./ort - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-ort + path: | + ./ort + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-ort - name: Checkout ORT Repository if: steps.cache-ort.outputs.cache-hit != 'true' uses: actions/checkout@v4 - with: + with: repository: "oss-review-toolkit/ort" path: "./ort" ref: "26.0.0" submodules: recursive - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1.78 + uses: dtolnay/rust-toolchain@1.78 - name: Install ORT if: steps.cache-ort.outputs.cache-hit != 'true' working-directory: ./ort/ run: | - export JAVA_OPTS="$JAVA_OPTS -Xmx8g" - ./gradlew installDist + export JAVA_OPTS="$JAVA_OPTS -Xmx8g" + ./gradlew installDist - name: Create ORT config file run: | - mkdir -p ~/.ort/config - cat << EOF > ~/.ort/config/config.yml - ort: - analyzer: - allowDynamicVersions: true - enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector] - EOF - cat ~/.ort/config/config.yml + mkdir -p ~/.ort/config + cat << EOF > ~/.ort/config/config.yml + ort: + analyzer: + allowDynamicVersions: true + enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector] + EOF + cat ~/.ort/config/config.yml - ### NodeJS ### + ### NodeJS ### - name: Set up Node.js 16.x uses: actions/setup-node@v4 with: node-version: 16.x - - name: Create package.json file for the Node wrapper + - name: Create package.json file for the Node wrapper uses: ./.github/workflows/node-create-package-file with: - release_version: ${{ env.RELEASE_VERSION }} - os: "ubuntu-latest" + release_version: ${{ env.RELEASE_VERSION }} + os: "ubuntu-latest" - name: Fix Node base NPM package.json file for ORT working-directory: ./node/npm/glide run: | - # Remove the glide-rs dependency to avoid duplication - sed -i '/ "glide-rs":/d' ../../package.json - export pkg_name=valkey-glide-base - export package_version="${{ env.RELEASE_VERSION }}" - export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - mv package.json package.json.tmpl - envsubst < package.json.tmpl > "package.json" - cat package.json - + # Remove the glide-rs dependency to avoid duplication + sed -i '/ "glide-rs":/d' ../../package.json + export pkg_name=valkey-glide-base + export package_version="${{ env.RELEASE_VERSION }}" + export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` + mv package.json package.json.tmpl + envsubst < package.json.tmpl > "package.json" + cat package.json + - name: Run ORT tools for Node uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/node" - - ### Python ### + folder_path: "${{ github.workspace }}/node" + + ### Python ### - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -146,14 +146,14 @@ jobs: - name: Run ORT tools for Python uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/python" + folder_path: "${{ github.workspace }}/python" ### Rust ### - name: Run ORT tools for Rust uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/glide-core" + folder_path: "${{ github.workspace }}/glide-core" ### Java ### @@ -163,60 +163,60 @@ jobs: distribution: "temurin" java-version: 11 - - name: Run ORT tools for Java + - name: Run ORT tools for Java uses: ./.github/workflows/run-ort-tools with: - folder_path: "${{ github.workspace }}/java" + folder_path: "${{ github.workspace }}/java" ### Process results ### - name: Check for diff run: | - cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS - cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS - cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS - cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS - GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS` - if [ -n "$GIT_DIFF" ]; then - echo "FOUND_DIFF=true" >> $GITHUB_ENV - else - echo "FOUND_DIFF=false" >> $GITHUB_ENV - fi + cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS + cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS + cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS + cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS + GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS` + if [ -n "$GIT_DIFF" ]; then + echo "FOUND_DIFF=true" >> $GITHUB_ENV + else + echo "FOUND_DIFF=false" >> $GITHUB_ENV + fi - name: Retrieve licenses list working-directory: ./utils run: | - { - echo 'LICENSES_LIST<> "$GITHUB_ENV" + { + echo 'LICENSES_LIST<> "$GITHUB_ENV" ### Create PR ### - name: Create pull request if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }} run: | - export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi` - echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}" - git config --global user.email "valkey-glide@lists.valkey.io" - git config --global user.name "ort-bot" - git checkout -b ${BRANCH_NAME} - git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS - git commit -m "Updated attribution files" -s - git push --set-upstream origin ${BRANCH_NAME} -f - title="Updated attribution files for ${BRANCH_NAME}" - gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}' + export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi` + echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}" + git config --global user.email "valkey-glide@lists.valkey.io" + git config --global user.name "ort-bot" + git checkout -b ${BRANCH_NAME} + git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS + git commit -m "Updated attribution files" -s + git push --set-upstream origin ${BRANCH_NAME} -f + title="Updated attribution files for ${BRANCH_NAME}" + gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}' env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} - name: Get current date id: date run: | - CURR_DATE=$(date +'%Y-%m-%d-%H') - echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT + CURR_DATE=$(date +'%Y-%m-%d-%H') + echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT - name: Upload the final package list continue-on-error: true diff --git a/.github/workflows/pypi-cd.yml b/.github/workflows/pypi-cd.yml index 61052d4537..b30810c233 100644 --- a/.github/workflows/pypi-cd.yml +++ b/.github/workflows/pypi-cd.yml @@ -169,13 +169,13 @@ jobs: fi PB_REL="https://github.com/protocolbuffers/protobuf/releases" ARCH=`uname -p` - if [[ $ARCH == 'x86_64' ]]; then + if [[ $ARCH == 'x86_64' ]]; then PROTOC_ARCH="x86_64" - elif [[ $ARCH == 'aarch64' ]]; then + elif [[ $ARCH == 'aarch64' ]]; then PROTOC_ARCH="aarch_64" - else + else echo "Running on unsupported architecture: $ARCH. Expected one of: ['x86_64', 'aarch64']" - exit 1 + exit 1 fi curl -LO $PB_REL/download/v3.20.3/protoc-3.20.3-linux-${PROTOC_ARCH}.zip unzip protoc-3.20.3-linux-${PROTOC_ARCH}.zip -d $HOME/.local diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 8c39c14091..2511c8c1f4 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -48,16 +48,16 @@ jobs: load-engine-matrix: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - + - name: Load the engine matrix id: load-engine-matrix shell: bash run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - + test: runs-on: ${{ matrix.host.RUNNER }} needs: load-engine-matrix @@ -67,10 +67,10 @@ jobs: matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" + # - "3.8" + # - "3.9" + # - "3.10" + # - "3.11" - "3.12" host: - { @@ -150,17 +150,17 @@ jobs: matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" + # - "3.8" + # - "3.9" + # - "3.10" + # - "3.11" - "3.12" host: - { OS: ubuntu, RUNNER: ubuntu-latest, TARGET: x86_64-unknown-linux-gnu - } + } # - { # OS: macos, # RUNNER: macos-latest, @@ -171,12 +171,12 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - + - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - + - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper with: @@ -184,14 +184,14 @@ jobs: target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} - + - name: Test pubsub with pytest working-directory: ./python run: | source .env/bin/activate cd python/tests/ pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html - + - name: Upload test reports if: always() continue-on-error: true @@ -288,30 +288,30 @@ jobs: name: smoke-test-report-amazon-linux path: | python/python/tests/pytest_report.html - + start-self-hosted-runner: - if: github.event.pull_request.head.repo.owner.login == 'valkey-io' - runs-on: ubuntu-latest - environment: AWS_ACTIONS - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Start self hosted EC2 runner - uses: ./.github/workflows/start-self-hosted-runner - with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + if: github.event.pull_request.head.repo.owner.login == 'valkey-io' + runs-on: ubuntu-latest + environment: AWS_ACTIONS + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start self hosted EC2 runner + uses: ./.github/workflows/start-self-hosted-runner + with: + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} test-modules: - needs: [start-self-hosted-runner, load-engine-matrix] - name: Running Module Tests - runs-on: ${{ matrix.host.RUNNER }} - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: + needs: [start-self-hosted-runner, load-engine-matrix] + name: Running Module Tests + runs-on: ${{ matrix.host.RUNNER }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} python: - "3.12" @@ -321,43 +321,43 @@ jobs: NAMED_OS: "linux", RUNNER: ["self-hosted", "Linux", "ARM64"], TARGET: "aarch64-unknown-linux-gnu", - } - - steps: - - name: Setup self-hosted runner access - if: ${{ contains(matrix.host.RUNNER, 'self-hosted') }} - run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Python for self-hosted Ubuntu runners - run: | + } + + steps: + - name: Setup self-hosted runner access + if: ${{ contains(matrix.host.RUNNER, 'self-hosted') }} + run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide + + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python for self-hosted Ubuntu runners + run: | sudo apt update -y sudo apt upgrade -y sudo apt install python3 python3-venv python3-pip -y - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test with pytest - working-directory: ./python - run: | - source .env/bin/activate - cd python/tests/ - pytest --asyncio-mode=auto --tls --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} -k server_modules --html=pytest_report.html --self-contained-html - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: smoke-test-report-amazon-linux - path: | - python/python/tests/pytest_report.html + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test with pytest + working-directory: ./python + run: | + source .env/bin/activate + cd python/tests/ + pytest --asyncio-mode=auto --tls --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} -k server_modules --html=pytest_report.html --self-contained-html + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: smoke-test-report-amazon-linux + path: | + python/python/tests/pytest_report.html diff --git a/java/benchmarks/build.gradle b/java/benchmarks/build.gradle index 84c1cafc95..9518f874b9 100644 --- a/java/benchmarks/build.gradle +++ b/java/benchmarks/build.gradle @@ -13,7 +13,7 @@ dependencies { def releaseVersion = System.getenv("GLIDE_RELEASE_VERSION"); if (releaseVersion) { - implementation "io.valkey:valkey-glide:"+ releaseVersion + ":${osdetector.classifier}" + implementation "io.valkey:valkey-glide:" + releaseVersion + ":${osdetector.classifier}" } else { implementation project(':client') } @@ -41,4 +41,3 @@ application { } -