diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
index 4a57189b0e..6c7c79ae81 100644
--- a/.github/workflows/version_bump.yml
+++ b/.github/workflows/version_bump.yml
@@ -3,6 +3,10 @@ name: Version bump
 on:
   workflow_dispatch:
     inputs:
+      dry_run:
+        description: "Run the action without creating a PR or release draft"
+        required: true
+        type: boolean
       bump:
         description: "Version bump type"
         required: true
@@ -57,8 +61,16 @@ jobs:
         github_token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}
         extra_requirements: 'git+https://github.com/meltano/commitizen-version-bump@main'
         changelog_increment_filename: _changelog_fragment.md
+        # TODO: Remove this constraint once the issue is fixed
+        # https://github.com/commitizen-tools/commitizen/issues/1024
+        commitizen_version: "3.18.0"
+
+    - name: Add job summary
+      run: |
+        cat _changelog_fragment.md >> $GITHUB_STEP_SUMMARY
 
     - name: Draft Release
+      if: ${{ !github.event.inputs.dry_run }}
       id: draft-release
       uses: softprops/action-gh-release@v2
       with:
@@ -74,6 +86,7 @@ jobs:
         sudo chown -R $USER:$USER .git/objects
 
     - name: Create Pull Request
+      if: ${{ !github.event.inputs.dry_run }}
       uses: peter-evans/create-pull-request@v6
       id: create-pull-request
       with: