From 371cb5f6c73e3ae32c7a1d3ce343d45a9dd174da Mon Sep 17 00:00:00 2001 From: Vinicius Stevam <45455812+vinistevam@users.noreply.github.com> Date: Wed, 28 Feb 2024 08:32:25 -0300 Subject: [PATCH] chore: simplify ccache cache logic (#8542) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## **Description** The mix of the build logic and the fact that every push in the main triggers a pipeline in bitrise prevents the ccache from caching the main build files, and subsequent PRs get the benefit of the native dependencies previously cached in `main`. This PR simplifies the approach by saving the native modules whenever something changes so every branch can benefit from the native dependencies cached. - [Example pipeline presenting the issue](https://app.bitrise.io/app/be69d4368ee7e86d/pipelines/ac5b23e0-3481-4906-93b9-f76f58c12014) ## **Related issues** Fixes: https://github.com/MetaMask/mobile-planning/issues/1514 ## **Manual testing steps** 1. Run smoke and regression pipeline ## **Screenshots/Recordings** ### **Before** ### **After** ## **Pre-merge author checklist** - [x] I’ve followed [MetaMask Coding Standards](https://github.com/MetaMask/metamask-mobile/blob/main/.github/guidelines/CODING_GUIDELINES.md). - [x] I've clearly explained what problem this PR is solving and how it is solved. - [x] I've linked related issues - [x] I've included manual testing steps - [x] I've included screenshots/recordings if applicable - [x] I’ve included tests if applicable - [x] I’ve documented my code using [JSDoc](https://jsdoc.app/) format if applicable - [x] I’ve applied the right labels on the PR (see [labeling guidelines](https://github.com/MetaMask/metamask-mobile/blob/main/.github/guidelines/LABELING_GUIDELINES.md)). Not required for external contributors. - [x] I’ve properly set the pull request status: - [x] In case it's not yet "ready for review", I've set it to "draft". - [x] In case it's "ready for review", I've changed it from "draft" to "non-draft". ## **Pre-merge reviewer checklist** - [ ] I've manually tested the PR (e.g. pull and build branch, run the app, test code being changed). - [ ] I confirm that this PR addresses all acceptance criteria described in the ticket it closes and includes the necessary testing evidence such as recordings and or screenshots. --- bitrise.yml | 44 +++++++++++++++++++++------------ scripts/cache/set-cache-envs.sh | 31 ++++------------------- 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index 3cf96721a89..9de4fa55ced 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -408,6 +408,12 @@ workflows: after_run: - _notify_failure_on_slack steps: + - script@1: + title: Generating ccache key using native folder checksum + inputs: + - content: |- + #!/usr/bin/env bash + ./scripts/cache/set-cache-envs.sh android - restore-gradle-cache@2: {} - install-missing-android-tools@3: inputs: @@ -420,12 +426,6 @@ workflows: - content: |- #!/usr/bin/env bash "${ANDROID_HOME}/tools/bin/sdkmanager" "cmake;3.22.1" - - script@1: - title: Check native folder diff for Android - inputs: - - content: |- - #!/usr/bin/env bash - ./scripts/cache/set-cache-envs.sh android - script@1: title: Install CCache & symlink inputs: @@ -435,16 +435,22 @@ workflows: sudo apt install ccache - restore-cache@2: title: Restore CCache - run_if: '{{not (enveq "SKIP_CCACHE_RESTORE" "true")}}' inputs: - key: '{{ getenv "CCACHE_KEY" }}' + - script@1: + title: Set skip ccache upload + run_if: '{{ enveq "BITRISE_CACHE_HIT" "exact" }}' + inputs: + - content: |- + #!/usr/bin/env bash + envman add --key SKIP_CCACHE_UPLOAD --value "true" - restore-cache@2: title: Restore build from cache inputs: - key: apk-cache-{{ .CommitHash }} - script@1: title: Run detox build - run_if: '{{not (enveq "BITRISE_CACHE_HIT" "exact")}}' + run_if: '{{ not (enveq "BITRISE_CACHE_HIT" "exact") }}' is_always_run: false inputs: - content: |- @@ -571,6 +577,12 @@ workflows: after_run: - _notify_failure_on_slack steps: + - script@1: + title: Generating ccache key using native folder checksum + inputs: + - content: |- + #!/usr/bin/env bash + ./scripts/cache/set-cache-envs.sh ios - certificate-and-profile-installer@1: {} - set-xcode-build-number@1: inputs: @@ -584,12 +596,6 @@ workflows: envman add --key BREW_OPT_APPLESIMUTILS --value "/usr/local/opt/applesimutils" brew tap wix/brew title: Set Env Path for caching deps - - script@1: - title: Check native folder diff for iOS - inputs: - - content: |- - #!/usr/bin/env bash - ./scripts/cache/set-cache-envs.sh ios - script@1: title: Install CCache & symlink inputs: @@ -604,16 +610,22 @@ workflows: ln -s $(which ccache) /usr/local/bin/clang++ - restore-cache@2: title: Restore CCache - run_if: '{{not (enveq "SKIP_CCACHE_RESTORE" "true")}}' inputs: - key: '{{ getenv "CCACHE_KEY" }}' + - script@1: + title: Set skip ccache upload + run_if: '{{ enveq "BITRISE_CACHE_HIT" "exact" }}' + inputs: + - content: |- + #!/usr/bin/env bash + envman add --key SKIP_CCACHE_UPLOAD --value "true" - restore-cache@2: title: Restore build from cache inputs: - key: ipa-cache-{{ .CommitHash }} - script@1: title: Run detox build - run_if: '{{not (enveq "BITRISE_CACHE_HIT" "exact")}}' + run_if: '{{ not (enveq "BITRISE_CACHE_HIT" "exact") }}' is_always_run: false inputs: - content: |- diff --git a/scripts/cache/set-cache-envs.sh b/scripts/cache/set-cache-envs.sh index c23a7be1ec1..7a65d27f497 100755 --- a/scripts/cache/set-cache-envs.sh +++ b/scripts/cache/set-cache-envs.sh @@ -2,34 +2,13 @@ # Check differences for a given folder and save the checksum as CCACHE_KEY set_cache_envs() { local folder="$1" - local current_branch=$(git rev-parse --abbrev-ref HEAD) local prefix_ccache_key="ccache-$folder" - if [ "$current_branch" = "main" ]; then - # If on main branch, skip cache restore but save it for future runs - echo "On main branch, we will not use cache, but will save it for future runs" - envman add --key SKIP_CCACHE_RESTORE --value true - else - # Fetch changes from the main branch - git fetch origin refs/heads/main - - # Get the differences between the current branch and the main branch for the specified folder - local differences=$(git diff origin/main...$current_branch -- $folder) - - # Check if there are no differences - if [ -z "$differences" ]; then - # If no differences, use the main cache and skip cache upload - echo "No differences detected, we will use main cache" - envman add --key SKIP_CCACHE_UPLOAD --value true - envman add --key CCACHE_KEY --value "$prefix_ccache_key-main" - echo "Checksum (CCACHE_KEY) set to $prefix_ccache_key-main" - else - # Generate a checksum for the differences and set it as CCACHE_KEY - local checksum=$(echo "$differences" | sha512sum | awk '{print $1}') - envman add --key CCACHE_KEY --value "$prefix_ccache_key-$checksum" - echo "Checksum (CCACHE_KEY) set to $prefix_ccache_key-$checksum" - fi - fi + # Generate a checksum for the native dependencies and set it as CCACHE_KEY + local checksum=$(find "$folder" -type f -exec sha512sum {} \; | sha512sum | awk '{print $1}') + echo "Checksum: $checksum" + envman add --key CCACHE_KEY --value "$prefix_ccache_key-$checksum" + echo "CCACHE_KEY set to $prefix_ccache_key-$checksum" } # Check if both folder_to_check is provided