fix: binary execution mode compatibility (#406)

* fix: fixing vanity address and init command issues with pyinstaller

* test: adding tests

* chore: refinements

* test: some test refinements

* chore: minor tweaks

* fix: fixing variable names

* fix: fixing based on PR comments

* test: fixing test

* test: add email env

* test: add --no-git to the tests

* chore: adding git identity prior to pytest in ci

* chore: testing ci

* chore: adding portability tests

* chore: testing ci

* chore: testing ci

* chore: testing ci

* chore: test ci

* chore: test

* chore: testing ci

* chore: test ci

* test: fixing test names-PR reviews

* docs: add docs about python_path usage in templates

* docs: change wordings

* chore: replacing init test to run against dummy template

* chore: addressing pr comments

* chore: testing ci

* chore: test

* chore: test

* chore: test

* chore: testing ci

* chore: testing ci

* chore: test

* chore: testing ci

* chore: minor refinements

* chore: minor tweaks

* fix: addressing pr comments

* test: fix a typo in test

* chore: fixing generator test for windows

* fix: PR comments

* docs: updating tutorial docs

---------

Co-authored-by: Altynbek Orumbayev <altynbek.orumbayev@makerx.com.au>

.github/actions/build-binaries/action.yaml

@@ -1,10 +1,10 @@
-name: "Setup, Build, and Test"
-description: "Set up Python with Poetry, build and test binaries"
+name: "Setup, Build, and Test Pyinstaller Binaries"
+description: "Build, test and upload binaries to GitHub releases and/or artifacts (ensure poetry dependencies are installed before running this action)"
 inputs:
   package_name:
     description: "The name of the package to build and test"
     required: true
-  upload_binaries:
+  production_release:
     description: "Flag to determine if this is a production release"
     required: true
   operating_system:
@@ -24,41 +24,60 @@ runs:
       run: ${{ inputs.build_command }}
       shell: bash
 
-    - name: Test Executable
+    - name: Set TEST_CLI_PATH
       run: |
-        ls -l dist
-        ./dist/${{ inputs.package_name }}/${{ inputs.package_name }}${{ inputs.operating_system == 'windows-latest' && '.exe' || '' }} --help
+        TEST_CLI_PATH="${{ github.workspace }}/dist/${{ inputs.package_name }}/${{ inputs.package_name }}"
+        if [ "${{ inputs.operating_system }}" == "Windows" ]; then
+          TEST_CLI_PATH="${{ github.workspace }}\dist\${{ inputs.package_name }}\${{ inputs.package_name }}.exe"
+        fi
+        echo "TEST_CLI_PATH=$TEST_CLI_PATH" >> $GITHUB_ENV
       shell: bash
 
+    - name: Run portability tests
+      run: |
+        echo "Executing: ${{ env.TEST_CLI_PATH }}"
+        git config --global user.email "actions@github.com" && git config --global user.name "github-actions" 
+        poetry run pytest tests/ -m pyinstaller_binary_tests --cli_path "${{ env.TEST_CLI_PATH }}" --log-cli-level=INFO
+      shell: ${{ inputs.operating_system == 'Windows' && 'cmd' || 'bash' }}
+
     - name: Set release version
       shell: bash
       continue-on-error: true
-      if: ${{ inputs.upload_binaries == 'true' }}
+      if: ${{ inputs.production_release == 'true' }}
       run: |
         echo "RELEASE_VERSION=$(git describe --tags $(git rev-list --tags --max-count=1))" >> $GITHUB_ENV
         git describe --tags $(git rev-list --tags --max-count=1)
 
     - name: Zip binaries
       shell: bash
-      continue-on-error: true
-      if: ${{ inputs.upload_binaries == 'true' }}
       run: |
         ls -l dist
         cd dist/algokit/
-        tar -zcvf ../../algokit-${{ env.RELEASE_VERSION }}-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz *
+        if [ "${{ inputs.production_release }}" == "true" ]; then
+          tar -zcvf ../../algokit-${{ env.RELEASE_VERSION }}-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz *
+        else
+          tar -zcvf ../../algokit-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz *
+        fi
         cd ../..
         ls -l
 
-    - name: Upload binary as artifact
-      if: ${{ inputs.upload_binaries == 'true' }}
+    - name: Upload binary as artifact (release)
+      if: ${{ inputs.production_release == 'true' }}
       uses: actions/upload-artifact@v4
       with:
-        name: algokit-cli-${{ inputs.operating_system }}-py${{ inputs.python_version }}
+        name: algokit-${{ inputs.operating_system }}-py${{ inputs.python_version }}
         path: algokit-${{ env.RELEASE_VERSION }}-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz
 
+    - name: Upload binary as artifact (dev)
+      if: ${{ inputs.production_release != 'true' }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: algokit-${{ inputs.operating_system }}-py${{ inputs.python_version }}
+        path: algokit-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz
+
     - name: Append binary to release
       continue-on-error: true
-      if: ${{ inputs.upload_binaries == 'true' }}
+      if: ${{ inputs.production_release == 'true' }}
       uses: softprops/action-gh-release@v1
       with:
         files: |

.github/workflows/build-binaries.yaml

@@ -3,133 +3,54 @@ name: Build, Test and Publish Pyinstaller Binaries
 on:
   workflow_call:
     inputs:
-      upload_binaries:
+      production_release:
         required: true
         type: string
       python_version:
         required: true
         type: string
 
 jobs:
-  build-binaries-ubuntu:
-    runs-on: ubuntu-latest
+  build-binaries:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        include:
+          - os: ubuntu-latest
+            build_command: "poetry run poe package_unix"
+          - os: windows-latest
+            build_command: "poetry run poe package_windows"
+          - os: macos-latest
+            build_command: "poetry run poe package_unix"
     steps:
-      - name: Checkout source code (for release)
+      - name: Checkout source code
         uses: actions/checkout@v4
-        if: ${{ inputs.upload_binaries == 'true' }}
         with:
-          fetch-depth: 0
-
-      - name: Checkout source code (for build)
-        uses: actions/checkout@v4
-        if: ${{ inputs.upload_binaries != 'true' }}
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ inputs.python_version }}
-
-      - name: Set up Poetry
-        uses: ./.github/actions/setup-poetry
-
-      - uses: actions/cache@v4
-        name: Setup poetry cache
-        with:
-          path: ./.venv
-          key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-${{ inputs.python_version }}
-
-      - name: Install dependencies
-        run: poetry install --no-interaction
-
-      - name: Build linux binary
-        uses: ./.github/actions/build-binaries
-        with:
-          python_version: ${{ inputs.python_version }}
-          package_name: "algokit"
-          upload_binaries: ${{ inputs.upload_binaries }}
-          operating_system: ${{ runner.os }}
-          build_command: "poetry run poe package_unix"
-
-  build-binaries-windows:
-    runs-on: windows-latest
-    steps:
-      - name: Checkout source code (for release)
-        uses: actions/checkout@v4
-        if: ${{ inputs.upload_binaries == 'true' }}
-        with:
-          fetch-depth: 0
-
-      - name: Checkout source code (for build)
-        uses: actions/checkout@v4
-        if: ${{ inputs.upload_binaries != 'true' }}
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ inputs.python_version }}
-
-      - name: Set up Poetry
-        uses: ./.github/actions/setup-poetry
-
-      - uses: actions/cache@v4
-        name: Setup poetry cache
-        with:
-          path: ./.venv
-          key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-${{ inputs.python_version }}
-
-      - name: Install dependencies
-        run: poetry install --no-interaction
-
-      - name: Build windows binary
-        uses: ./.github/actions/build-binaries
-        with:
-          python_version: ${{ inputs.python_version }}
-          package_name: "algokit"
-          upload_binaries: ${{ inputs.upload_binaries }}
-          operating_system: ${{ runner.os }}
-          build_command: "poetry run poe package_windows"
-
-  build-binaries-macos:
-    runs-on: macos-latest
-    steps:
-      - name: Checkout source code (for release)
-        uses: actions/checkout@v4
-        if: ${{ inputs.upload_binaries == 'true' }}
-        with:
-          fetch-depth: 0
-
-      - name: Checkout source code (for build)
-        uses: actions/checkout@v4
-        if: ${{ inputs.upload_binaries != 'true' }}
-        with:
-          fetch-depth: 1
+          fetch-depth: ${{ inputs.production_release == 'true' && '0' || '1' }}
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python_version }}
 
       - name: Set up Poetry
         uses: ./.github/actions/setup-poetry
 
-      - uses: actions/cache@v4
-        name: Setup poetry cache
+      - name: Setup poetry cache
+        uses: actions/cache@v4
         with:
           path: ./.venv
           key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-${{ inputs.python_version }}
 
       - name: Install dependencies
         run: poetry install --no-interaction
 
-      - name: Build macos binary
+      - name: Build & test binary
         uses: ./.github/actions/build-binaries
         with:
           python_version: ${{ inputs.python_version }}
           package_name: "algokit"
-          upload_binaries: ${{ inputs.upload_binaries }}
+          production_release: ${{ inputs.production_release }}
           operating_system: ${{ runner.os }}
-          build_command: "poetry run poe package_unix"
+          build_command: ${{ matrix.build_command }}

.github/workflows/build-python.yaml

@@ -15,7 +15,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -35,8 +35,10 @@ jobs:
 
       - name: pytest + coverage
         shell: bash
+        # git config is needed due to several tests relying on e2e copier invocation and copier relies on git during `copy` command
         run: |
           set -o pipefail
+          git config --global user.email "actions@github.com" && git config --global user.name "github-actions"
           poetry run pytest -n auto --junitxml=pytest-junit.xml --cov-report=term-missing:skip-covered --cov=src | tee pytest-coverage.txt
         id: pytest
 

.github/workflows/cd.yaml

@@ -39,7 +39,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
@@ -123,7 +123,7 @@ jobs:
     uses: ./.github/workflows/build-binaries.yaml
     needs: release
     with:
-      upload_binaries: "true"
+      production_release: "true"
       python_version: "3.12"
 
   cd-publish-release-packages:

.github/workflows/check-python.yaml

@@ -11,7 +11,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.12"
 

.github/workflows/pr.yaml

@@ -12,14 +12,14 @@ jobs:
     uses: ./.github/workflows/check-python.yaml
 
   pr-build:
-    name: Build and Test Python
+    name: Build & Test Python
     needs: pr-check
     uses: ./.github/workflows/build-python.yaml
 
   pr-binaries-build:
-    name: Build Binaries
+    name: Build & Test Binaries
     needs: pr-check
     uses: ./.github/workflows/build-binaries.yaml
     with:
-      upload_binaries: "false"
+      production_release: "false"
       python_version: "3.12"

docs/tutorials/algokit-template.md

@@ -143,6 +143,35 @@ When creating an AlgoKit template, there are a few default behaviors that you ca
 
 By combining predefined Copier answers with these default behaviors, you can create a smooth, efficient, and intuitive initialization experience for the users of your template.
 
+### Executing Python Tasks in Templates
+
+If you need to use Python scripts as tasks within your Copier templates, ensure that you have Python installed on the host machine.
+By convention, AlgoKit automatically detects the Python installation on your machine and fills in the `python_path` variable accordingly.
+This process ensures that any Python scripts included as tasks within your Copier templates will execute using the system's Python interpreter.
+It's important to note that the use of `_copier_python` is not recommended. Here's an example of specifying a Python script execution in your `copier.yaml` without needing to explicitly use `_copier_python`:
+
+```yaml
+- "{{ python_path }} your_python_script.py"
+```
+
+If you'd like your template to be backwards compatible with versions of `algokit-cli` older than `v1.11.3` when executing custom python scripts via `copier` tasks, you can use a conditional statement to determine the Python path:
+
+```yaml
+- "{{ python_path if python_path else _copier_python }} your_python_script.py"
+# _copier_python above is used for backwards compatibility with versions < v1.11.3 of the algokit cli
+```
+
+And to define `python_path` in your Copier questions:
+
+```yaml
+# Auto determined by algokit-cli from v1.11.3 to allow execution of python script
+# in binary mode.
+python_path:
+  type: str
+  help: Path to the sys.executable.
+  when: false
+```
+
 ### Working with Generators
 
 After mastering the use of `copier` and building your templates based on the official AlgoKit template repositories, you can enhance your proficiency by learning to define `custom generators`. Essentially, generators are smaller-scope `copier` templates designed to provide additional functionality after a project has been initialized from the template.