From df975c9be0f2fe3244365b79823af5e043b1aeb7 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Tue, 25 Jun 2024 15:59:21 -0400 Subject: [PATCH] Add git-cherry-pick pipeline (#1278) * Add git-cherry-pick pipeline --------- Signed-off-by: Krish Jain --- docs/PIPELINES-GIT.md | 37 ++++++ e2e-tests/git-checkout-build.yaml | 38 ++++++ e2e-tests/test-fixtures/create-git-repo | 17 ++- pkg/build/pipelines/git-checkout.yaml | 151 ++++++++++++++++++++---- 4 files changed, 212 insertions(+), 31 deletions(-) create mode 100644 docs/PIPELINES-GIT.md diff --git a/docs/PIPELINES-GIT.md b/docs/PIPELINES-GIT.md new file mode 100644 index 000000000..920deab8a --- /dev/null +++ b/docs/PIPELINES-GIT.md @@ -0,0 +1,37 @@ +# Built-in git pipeline + +Melange includes a built-in pipeline to checkout git repos. + +To get started quickly, checkout the `git-checkout` pipeline. + + +How to use it? + +``` + - uses: git-checkout + with: + repository: + tag: ${{package.version}} + expected-commit: + +``` + +You have these inputs (defined in https://github.com/chainguard-dev/melange/blob/main/pkg/build/pipelines/git-checkout.yaml): + +How to use the cherry-picking feature? + + +To fix https://nvd.nist.gov/vuln/detail/CVE-2024-4032 for example you can do something nice: + +``` +pipeline: + - uses: git-checkout + with: + expected-commit: 976ea78599d71f22e9c0fefc2dc37c1d9fc835a4 + repository: https://github.com/python/cpython.git + tag: v3.10.14 + cherry-picks: | + 3.10/c62c9e518b784fe44432a3f4fc265fb95b651906: CVE-2024-4032 +``` + +Note the format of cherry-picking: ``[branch/]commit: comment here`` diff --git a/e2e-tests/git-checkout-build.yaml b/e2e-tests/git-checkout-build.yaml index 3cb023f94..3b29b754c 100644 --- a/e2e-tests/git-checkout-build.yaml +++ b/e2e-tests/git-checkout-build.yaml @@ -3,18 +3,22 @@ package: version: 6.14 epoch: 0 description: This package mainly just tests git-checkout pipeline + environment: contents: packages: - busybox - git + vars: workd: /tmp/test-git-checkout-workd giturl: "file:///tmp/test-git-checkout-workd/repos/my-repo" + pipeline: - name: "Create the bogus package content" runs: | echo "package does not do anything" > "${{targets.contextdir}}/README" + - name: "Create a git repo" runs: | rm -Rf ${{vars.workd}} @@ -23,6 +27,7 @@ pipeline: ./create-git-repo "$repo" touch "$repo/git-daemon-export-ok" + - name: "standard tag on branch" uses: git-checkout working-directory: standard @@ -30,6 +35,7 @@ pipeline: repository: ${{vars.giturl}} tag: 2.0 expected-commit: 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 + - name: "check standard tag on branch" working-directory: standard runs: | @@ -37,18 +43,21 @@ pipeline: [ "$hash" = 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 ] cd .. rm -Rf standard + - name: "standard no-working-directory" uses: git-checkout with: repository: ${{vars.giturl}} tag: 2.0 expected-commit: 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 + - name: "check standard no-working-directory" runs: | hash=$(git rev-parse --verify HEAD) [ "$hash" = 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 ] [ -f create-git-repo ] || { echo "create-git-repo did not exist"; exit 1; } + - name: "destination" uses: git-checkout working-directory: destination-base @@ -57,6 +66,7 @@ pipeline: tag: 2.0 expected-commit: 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 destination: dest + - name: "check destination" working-directory: destination-base runs: | @@ -65,6 +75,7 @@ pipeline: [ "$hash" = 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 ] cd ../.. rm -R destination-base + - name: "depth positive 1" uses: git-checkout working-directory: depth-positive-1 @@ -73,6 +84,7 @@ pipeline: repository: ${{vars.giturl}} tag: 2.0 expected-commit: 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 + - name: "check depth positive 1" working-directory: depth-positive-1 runs: | @@ -85,6 +97,7 @@ pipeline: [ "$hash" = 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 ] cd .. rm -R depth-positive-1 + - name: "depth negative 1" working-directory: depth-negative-1 uses: git-checkout @@ -93,6 +106,7 @@ pipeline: repository: ${{vars.giturl}} tag: 2.0 expected-commit: 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 + - name: "check depth -1" working-directory: depth-negative-1 runs: | @@ -105,12 +119,14 @@ pipeline: [ "$hash" = 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 ] cd .. rm -R depth-negative-1 + - name: "branch without expected" working-directory: branch-no-expected uses: git-checkout with: repository: ${{vars.giturl}} branch: 1.x + - name: "check branch without expected" working-directory: branch-no-expected runs: | @@ -118,6 +134,7 @@ pipeline: [ "$hash" = 4b1593d8d8038f8c0ce1e2c608c9dd89066a2a0f ] cd .. rm -R branch-no-expected + # for an annotated tag you can point to either the commit # or the tag object hash object - name: "annotated hash" @@ -127,6 +144,7 @@ pipeline: repository: ${{vars.giturl}} tag: 2.0-annotated expected-commit: 4ce5bdbf45a68a166d931dd1247878829b5c0113 + - name: "check annotated hash" working-directory: annotated-hash runs: | @@ -135,6 +153,7 @@ pipeline: [ "$hash" = 3dfc3dd573b814be48c07f7f8ae3c19a23b69865 ] cd .. rm -R annotated-hash + # special case with clone --branch if there is a tag and a branch # with the same name. - name: "tag and branch same name" @@ -144,6 +163,7 @@ pipeline: repository: ${{vars.giturl}} tag: dev expected-commit: 2b9bb894348794bc840a2ee7553d54a1c80b9278 + - name: "check tag and branch same name" working-directory: tag-and-branch runs: | @@ -151,3 +171,21 @@ pipeline: [ "$hash" = 2b9bb894348794bc840a2ee7553d54a1c80b9278 ] cd .. rm -R tag-and-branch + + - name: "process cherry-picks" + uses: git-checkout + working-directory: cherry-pick-test + with: + repository: ${{vars.giturl}} + branch: 1.x + cherry-picks: | + main/582b4d7d62f1c512568649ce8b6db085a3d85a9f: here comment + + - name: "check cherry-picks" + working-directory: cherry-pick-test + runs: | + hash=$(git rev-parse --verify HEAD) + expected_hash="225e712ae452645acbd8f137b13d6b1ded8a96a1" + [ "$hash" != "$expected_hash" ] + cd .. + rm -R cherry-pick-test diff --git a/e2e-tests/test-fixtures/create-git-repo b/e2e-tests/test-fixtures/create-git-repo index d6644e520..951988fd8 100755 --- a/e2e-tests/test-fixtures/create-git-repo +++ b/e2e-tests/test-fixtures/create-git-repo @@ -33,6 +33,9 @@ TEMP_D=$(mktemp -d) trap cleanup EXIT dest="$1" +if [ -z "$dest" ]; then + fail "must provide dest dir for bare git repo" +fi if [ -e "$dest" ]; then fail "do not give existing dir for dest" fi @@ -86,16 +89,20 @@ v git checkout --quiet main v wfile README "mainline stuff" v gcommit -m "mainline stuff" README -v git checkout --quiet main -v wfile README "mainline stuff 2" -v gcommit -m "mainline stuff 2" README - -v git checkout --quiet -b 2.x main^ +v git checkout --quiet -b 2.x main v wfile README "2.0-release content" v gcommit -m "release 2.0" README v gtag 2.0 v gtag --annotate --message="Release 2.0" 2.0-annotated HEAD +v git checkout --quiet main +v wfile README "mainline stuff 2" +v gcommit -m "mainline stuff 2" README + +v wfile file1.txt "exciting file contents" +v git add file1.txt +v gcommit -m "finally add file1.txt" file1.txt + v git checkout --quiet main v cd "$startd" diff --git a/pkg/build/pipelines/git-checkout.yaml b/pkg/build/pipelines/git-checkout.yaml index e23d50ba8..ee425009c 100644 --- a/pkg/build/pipelines/git-checkout.yaml +++ b/pkg/build/pipelines/git-checkout.yaml @@ -9,35 +9,46 @@ inputs: description: | The repository to check out sources from. required: true - destination: description: | The path to check out the sources to. default: . - depth: description: | The depth to use when cloning. Set to -1 to not specify depth when cloning. default: 1 - branch: description: | The branch to check out, otherwise HEAD is checked out. For reproducibility, tag is generally favored over branch. Branch and tag are mutually exclusive. - tag: description: | The tag to check out. Branch and tag are mutually exclusive. - expected-commit: description: | The expected commit hash - recurse-submodules: description: | Indicates whether --recurse-submodules should be passed to git clone. default: false + cherry-picks: + description: | + List of cherry picks to apply. + New line separated entries. + Lines can be empty. + Any content on a line after `#` is ignored. + After removing comments, each line is of the form: + + [branch/]commit-id: comment explaining cherry-pick + + comment and commit-id are required. branch on origin that + the commit lives should be provided or git is not guaranteed to + have a reference to the commit-id. + + Example: + cherry-picks: | + 3.10/62705d869aca4055e8a96e2ed4f9013e9917c661: pipeline: - runs: | @@ -48,9 +59,80 @@ pipeline: fail() { msg FAIL "$@"; exit 1; } vr() { msg "execute:" "$@"; "$@"; } + process_cherry_picks() { + local cpicksf="$1" oifs="$IFS" count=0 + local fetched_branches="" + local sdate=${SOURCE_DATE_EPOCH:-0} + if [ "$sdate" -lt 315532800 ]; then + msg "Setting commit date to Jan 1, 1980 (SOURCE_DATE_EPOCH found ${SOURCE_DATE_EPOCH})" + sdate=315532800 + fi + if [ -z "$cpicksf" ]; then + return 0 + fi + if [ ! -f "$cpicksf" ]; then + msg "cherry picks input '$cpicksf' is not a file" + return 1 + fi + + local line="" branch="" hash="" comment="" + while IFS= read -r line; do + # Drop anything after # + line=${line%%#*} + [ -z "$line" ] && continue + # Ensure the line contains ':' + if ! echo "$line" | grep -q ':'; then + msg "Invalid format, expected '[branch/]commit: comment'. Found: $line" + return 1 + fi + # Split the line into branch/hash and comment parts + branch=${line%%:*} + comment=${line#*:} + comment=$(set -f; echo $comment) # Strip leading/trailing whitespace + + if [ -z "$comment" ]; then + msg "Empty comment for cherry-pick: $line" + return 1 # or continue to skip this cherry-pick + fi + # Extract commit hash + hash=${branch##*/} + # If branch information exists, strip it off to just leave the branch name + [ "$branch" != "$hash" ] && branch=${branch%/*} || branch="" + + if [ -n "$branch" ]; then + case " $fetched_branches " in + *" $branch "*) ;; + *) vr git fetch origin $branch:$branch || { + msg "failed to fetch branch $branch" + return 1 + } + fetched_branches="$fetched_branches $branch ";; + esac + fi + + vr env \ + GIT_COMMITTER_DATE="@$sdate" \ + git cherry-pick -x "$hash" || { + msg "failed to cherry-pick $hash from branch $branch" + return 1 + } + + msg "Cherry-picked $hash from $branch with comment: $comment" + + count=$((count + 1)) + done < "$cpicksf" + + if [ $count -gt 0 ]; then + msg "applied $count cherry-pick(s). head is now $(git rev-parse HEAD)" + fi + } + + + main() { local repo=$1 dest=${2:-.} depth=${3:-"-1"} branch=$4 local tag=$5 expcommit=$6 recurse=${7:-false} + local cherry_pick="$8" msg "repo='$repo' dest='$dest' depth='$depth' branch='$branch'" \ "tag='$tag' expcommit='$expcommit' recurse='$recurse'" @@ -87,7 +169,10 @@ pipeline: vr git config --global --add safe.directory "$workdir" vr git config --global --add safe.directory "$dest_fullpath" - vr git clone $quiet "--origin=$remote" $flags \ + vr git clone $quiet "--origin=$remote" \ + "--config=user.name=Melange Build" \ + "--config=user.email=melange-build@cgr.dev" \ + $flags \ ${depthflag:+"$depthflag"} "$repo" "$workdir" vr cd "$workdir" @@ -108,6 +193,7 @@ pipeline: " got $foundcommit" fi msg "tip of ${branch:-HEAD} is commit $foundcommit" + process_cherry_picks "$cherry_pick" || fail "failed to apply cherry-pick" return 0 fi @@ -121,32 +207,45 @@ pipeline: foundcommit=$(git rev-parse --verify HEAD) if [ -z "$expcommit" ] || [ "$expcommit" = "$foundcommit" ]; then msg "tag $tag is $foundcommit" - return 0 - fi - - # If it's a tag, then it could be a lightweight or annotated tag. - # Lightweight tags point directly to the commit and do not have - # any messages, signatures, or other data. Annotated tags point - # to its own git object containing the tag data, with a reference - # to the underlying commit. We expect most tags to be using - # annotated tags. - tagobj=$(git rev-parse --verify --end-of-options \ - "refs/$remote/tags/$tag") - if [ "$expcommit" != "$tagobj" ]; then - [ "$tagobj" != "$expcommit" ] && - msg "tag object hash was $tagobj" - fail "Expected commit $expcommit for $tag, found $foundcommit" + else + # If it's a tag, then it could be a lightweight or annotated tag. + # Lightweight tags point directly to the commit and do not have + # any messages, signatures, or other data. Annotated tags point + # to its own git object containing the tag data, with a reference + # to the underlying commit. We expect most tags to be using + # annotated tags. + tagobj=$(git rev-parse --verify --end-of-options \ + "refs/$remote/tags/$tag") + if [ "$expcommit" != "$tagobj" ]; then + [ "$tagobj" != "$expcommit" ] && + msg "tag object hash was $tagobj" + fail "Expected commit $expcommit for $tag, found $foundcommit" + fi + + msg "Warning: The provided expected-commit ($expcommit)" + msg "was the hash of the annotated tag object for $tag." + msg "Update to set expected-commit to $foundcommit" fi - msg "Warning: The provided expected-commit ($expcommit)" - msg "was the hash of the annotated tag object for $tag." - msg "Update to set expected-commit to $foundcommit" + process_cherry_picks "$cherry_pick" || + fail "failed to apply cherry-pick" return 0 } + cpickf=$(mktemp) || { + echo "failed mktemp" + exit 1 + } + + cat >"$cpickf" <<"END_CHERRY_PICKS" + ${{inputs.cherry-picks}} + END_CHERRY_PICKS + main \ "${{inputs.repository}}" "${{inputs.destination}}" \ "${{inputs.depth}}" "${{inputs.branch}}" \ "${{inputs.tag}}" "${{inputs.expected-commit}}" \ - "${{inputs.recurse-submodules}}" + "${{inputs.recurse-submodules}}" "$cpickf" + + rm -f "$cpickf"