Merge pull request #11 from smaht-dac/pyinstaller-experiment-20240611

Pyinstaller experiment 20240611

.github/workflows/main-binaries-release.yml

@@ -0,0 +1,174 @@
+name: BINARY RELEASE
+
+permissions:
+  contents: write
+
+on:
+  push:
+    # Only run this workflow to create binaries when a non-beta tag is created.
+    tags:
+      - 'v[0-9]+\.[0-9]+\.[0-9]+'
+
+#   tags:
+#     - 'v*'
+
+jobs:
+
+  build-macos-x86:
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+      - name: Build MacOS Binary (x86)
+        run: |
+          echo "GitHub runner info:"
+          uname -a
+          echo "Building MacOS binary (x86)."
+          make exe-macos
+          mv -f binaries/submitr-macos binaries/submitr-macos-x86
+          echo "Done building MacOS binary (x86). Info below:"
+          ls -l binaries/submitr-macos-x86
+          file binaries/submitr-macos-x86
+          echo smaht-submitr version: `binaries/submitr-macos-x86 version`
+      - name: Upload MacOS Binary (x86) Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: submitr-macos-x86
+          path: binaries/submitr-macos-x86
+
+  build-macos-arm:
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+      - name: Build MacOS Binary (arm)
+        run: |
+          echo "GitHub runner info:"
+          uname -a
+          echo "Building MacOS binary (arm)."
+          make exe-macos
+          mv -f binaries/submitr-macos binaries/submitr-macos-arm
+          echo "Done building MacOS binary (arm). Info below:"
+          ls -l binaries/submitr-macos-arm
+          file binaries/submitr-macos-arm
+          echo smaht-submitr version: `binaries/submitr-macos-arm version`
+      - name: Upload MacOS Binary (arm) Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: submitr-macos-arm
+          path: binaries/submitr-macos-arm
+
+  build-linux-x86:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Linux Binary (x86)
+        run: |
+          echo "GitHub runner info:"
+          uname -a
+          echo "Building Linux binary (x86)."
+          make exe-linux-x86
+          echo "Done building Linux binary (x86). Info below:"
+          ls -l binaries/submitr-linux-x86
+          file binaries/submitr-linux-x86
+          echo smaht-submitr version: `binaries/submitr-linux-x86 version`
+      - name: Upload Linux Binary (x86) Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: submitr-linux-x86
+          path: binaries/submitr-linux-x86
+
+  build-linux-arm:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+      - uses: docker/setup-buildx-action@v2
+      - name: Build Linux Binary (arm)
+        run: |
+          echo "GitHub runner info:"
+          uname -a
+          echo "Building Linux binary (arm)."
+          make exe-linux-arm
+          echo "Done building Linux binary (arm). Info below:"
+          ls -l binaries/submitr-linux-arm
+          file binaries/submitr-linux-arm
+          echo smaht-submitr version: `binaries/submitr-linux-arm version`
+      - name: Upload Linux Binary (arm) Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: submitr-linux-arm
+          path: binaries/submitr-linux-arm
+
+  create-release:
+    runs-on: ubuntu-latest
+    needs: [build-macos-x86, build-macos-arm, build-linux-x86, build-linux-arm]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Download MacOS Binary (x86) Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: submitr-macos-x86
+
+      - name: Download MacOS Binary (arm) Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: submitr-macos-arm
+
+      - name: Download Linux Binary (x86) Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: submitr-linux-x86
+
+      - name: Download Linux Binary (arm) Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: submitr-linux-arm
+
+      - name: Create GitHub Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref_name }}  # Adjust the tag name as needed
+          release_name: Release ${{ github.ref_name }}
+          draft: false
+          prerelease: false
+
+      - name: Upload MacOS Binary (x86) to GitHub Release
+        run: |
+          gh release upload ${{ github.ref_name }} submitr-macos-x86 --repo ${{ github.repository }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload MacOS Binary (arm) to GitHub Release
+        run: |
+          gh release upload ${{ github.ref_name }} submitr-macos-arm --repo ${{ github.repository }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload Linux Binary (x86) to GitHub Release
+        run: |
+          gh release upload ${{ github.ref_name }} submitr-linux-x86 --repo ${{ github.repository }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload Linux Binary (arm) to GitHub Release
+        run: |
+          gh release upload ${{ github.ref_name }} submitr-linux-arm --repo ${{ github.repository }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/main-binaries-test.yml

@@ -0,0 +1,103 @@
+name: BINARY RELEASE TEST
+
+permissions:
+  contents: write
+
+on:
+  push:
+    tags:
+      - 'v*'
+# push:
+#   # Only run this workflow to create binaries when a non-beta tag is created.
+#   tags:
+#     - 'v[0-9]+\.[0-9]+\.[0-9]+'
+
+jobs:
+
+  test-macos-x86:
+    runs-on: macos-13
+    steps:
+      - name: Test MacOS Binary (x86)
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "GitHub runner info:" `uname -a`
+          echo "Smoke testing MacOS binary (x86)"
+          echo "Installing MacOS binary (x86):" ${{ github.event.release.tag_name }}
+          # This GITHUB_TOKEN authorization header for curl and argument to install.sh is
+          # ONLY to make this work consistently from within this workflow; without we would
+          # intermittently get an error like: API rate limit exceeded for 13.105.117.6
+          curl -H "Authorization: token ${GITHUB_TOKEN}" https://raw.githubusercontent.com/smaht-dac/submitr/pyinstaller-experiment-20240611/install.sh | /bin/bash -s -- "$GITHUB_TOKEN"
+          echo "Running smaht-submitr version. Should be the same as what was mentioned above."
+          # TODO: More actual testing besides just getting the version.
+          actual_version=`./submitr version`
+          echo "smaht-submitr version:" ${actual_version}
+          file ./submitr | grep x86_64
+          echo yes | ./submitr rcloner info foobar --noprogress
+          ls -l ~/Library/Application\ Support/edu.harvard.hms/smaht-submitr/rclone
+          echo "Done smoke testing MacOS binary (x86)"
+
+  test-macos-arm:
+    runs-on: macos-14
+    steps:
+      - name: Test MacOS Binary (arm)
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "GitHub runner info:" `uname -a`
+          expected_version=${{ github.event.release.tag_name }}
+          echo "Smoke testing MacOS binary (arm)"
+          # This GITHUB_TOKEN authorization header for curl and argument to install.sh is
+          # ONLY to make this work consistently from within this workflow; without we would
+          # intermittently get an error like: API rate limit exceeded for 13.105.117.6
+          echo "Installing MacOS binary (arm):" ${{ github.event.release.tag_name }}
+          curl -H "Authorization: token ${GITHUB_TOKEN}" https://raw.githubusercontent.com/smaht-dac/submitr/pyinstaller-experiment-20240611/install.sh | /bin/bash -s -- "$GITHUB_TOKEN"
+          echo "Running smaht-submitr version. Should be the same as what was mentioned above."
+          actual_version=`./submitr version`
+          # TODO: More actual testing besides just getting the version.
+          echo "smaht-submitr version:" ${actual_version}
+          file ./submitr | grep arm64
+          echo yes | ./submitr rcloner info foobar --noprogress
+          ls -l ~/Library/Application\ Support/edu.harvard.hms/smaht-submitr/rclone
+          echo "Done smoke testing MacOS binary (arm)"
+
+  test-linux-x86:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Test Linux Binary (x86)
+        run: |
+          echo "GitHub runner info:" `uname -a`
+          expected_version=${{ github.event.release.tag_name }}
+          echo "Smoke testing Linux binary (x86)"
+          echo "Installing Linux binary (x86):" ${{ github.event.release.tag_name }}
+          curl https://raw.githubusercontent.com/smaht-dac/submitr/pyinstaller-experiment-20240611/install.sh | /bin/bash
+          echo "Running smaht-submitr version. Should be the same as what was mentioned above."
+          actual_version=`./submitr version`
+          # TODO: More actual testing besides just getting the version.
+          echo "smaht-submitr version:" ${actual_version}
+          file ./submitr | grep x86-64
+          echo yes | ./submitr rcloner info foobar --noprogress
+          ls -l ~/.local/share/edu.harvard.hms/smaht-submitr/rclone
+          echo "Done smoke testing Linux binary (x86)"
+
+  test-linux-arm:
+    # TODO: This is wrong; need to run within docker as there is no GitHub runner for Linux ARM.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+      - uses: docker/setup-buildx-action@v2
+      - name: Build Linux Binary (arm)
+        run: |
+          echo "GitHub runner info:" `uname -a`
+          expected_version=${{ github.event.release.tag_name }}
+          echo "Smoke testing Linux binary (arm)"
+          echo "Creating a Linux ARM docker container for this."
+          echo "Installing Linux binary (arm):" ${{ github.event.release.tag_name }}
+          docker run --platform linux/arm64 --rm arm64v8/centos \
+              sh -c "curl https://raw.githubusercontent.com/smaht-dac/submitr/pyinstaller-experiment-20240611/install.sh | \
+                     /bin/bash ; ./submitr version ; \
+                     echo yes | ./submitr rcloner info foobar --noprogress ; \
+                     ls -l ~/.local/share/edu.harvard.hms/smaht-submitr/rclone"
+          echo "Done smoke testing Linux binary (arm)"

.github/workflows/main.yml

@@ -21,6 +21,8 @@ jobs:
 
     # The type of runner that the job will run on
     runs-on: ubuntu-22.04
+    ## runs-on: macos-13
+    # experimental end
     strategy:
       matrix:
         python_version: ['3.9', '3.10', '3.11']
@@ -41,6 +43,51 @@ jobs:
         run: |
           poetry install
           poetry run flake8 submitr
+
+          # The following actually seems to works on GitHub Actions runner (runs-on) macos-13 (but not macos-14);
+          # but takes a long long time, mostly on Python install within the docker container; could build my own
+          # docker contained with Python/penv preinstalled.
+          #
+          # echo "Installing docker."
+          # HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask docker
+          # echo "Installing colima."
+          # brew install colima
+          # echo "Starting colima."
+          # colima start
+          # echo "Waiting for docker get up/running. This can take over 2 minutes ..."
+          # while (! docker stats --no-stream ); do
+          #   echo "Waiting for docker to start ..."
+          #   sleep 5
+          # done
+          # echo "Docker now seems to be up/running."
+
+          ## echo "Installing docker."
+          ## HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask docker
+
+          ## echo "Installing colima."
+          ## brew install colima
+
+          ## echo "Starting colima."
+          ## colima start
+
+          ## echo "Waiting for docker get up/running. This can take over 2 minutes ..."
+          ## while (! docker stats --no-stream ); do
+          ## echo "Waiting for docker to start ..."
+          ## sleep 3
+          ## done
+          ## echo "Docker now seems to be up/running."
+
+          # echo "Preparing docker."
+          # sudo /Applications/Docker.app/Contents/MacOS/Docker\ Desktop.app/Contents/MacOS/Docker\ Desktop --unattended --install-privileged-components
+          # echo "Starting docker."
+          # open -a /Applications/Docker.app --args --unattended --accept-license
+          # echo "Waiting for docker get up/running. This can take over 2 minutes ..."
+          # while ! /Applications/Docker.app/Contents/Resources/bin/docker info &>/dev/null; do sleep 1; done
+          # echo "Docker now seems to be up/running."
+
+          ## echo "Building Linux binaries."
+          ## make exe-linux
+          ## echo "Done building Linux binaries."
       
       - name: CI
         run: |

.gitignore

@@ -12,7 +12,6 @@ build/
 docs/html/
 develop-eggs/
 dist/
-downloads/
 eggs/
 .eggs/
 lib/

CHANGELOG.rst

@@ -6,6 +6,15 @@ smaht-submitr
 Change Log
 ----------
 
+
+
+1.0.0
+=====
+* Using pyinstaller to create a single independent executable (per command)
+  so commands can be run without having Python/pyenv/etc installed.
+* Some changes related to starting work on integration tests with the portal.
+
+
 0.8.3
 =====
 

Dockerfile-for-pyinstaller

@@ -0,0 +1,28 @@
+# Dockerfile to build single self-contained executable for smaht-submitr using pyinstaller.
+# Tested with IMAGE values: centos, arm64v8/centos - for x86_64 and arm64 architectures respectively.
+# FYI: It turns out that binaries built on RedHat (CentOS) work on Debian (Ubuntu); but not vice versa.
+ARG IMAGE=
+FROM ${IMAGE}
+
+RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+RUN yum install sudo -y
+RUN sudo yum install git python3 gcc make bzip2 zlib-devel readline-devel sqlite-devel openssl-devel tk-devel libffi-devel wget xz-devel -y
+
+WORKDIR /app
+COPY . .
+
+RUN curl https://pyenv.run | /bin/bash
+RUN export PYENV_ROOT="$HOME/.pyenv" ; \
+    export PATH="$PYENV_ROOT/bin:$PATH" ; \
+    eval "$(pyenv init -)" ; \
+    eval "$(pyenv virtualenv-init -)" ; \
+    pyenv install 3.11.8 ; \
+    pyenv virtualenv 3.11.8 submitr-3.11 ; \
+    pyenv activate submitr-3.11 ; \
+    make build ; \
+    pip install pyinstaller ; \
+    pyinstaller --onefile submitr/scripts/submitr.py
+
+RUN chmod a+x dist/submitr
+CMD ["./dist/submitr"]