From 4d1ebc9654b20c2fdfb3d9178b0d0b5879792025 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 13:11:35 -0700 Subject: [PATCH 01/15] Add sanity check --- .github/workflows/build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60c1b5a..6467c2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -157,6 +157,13 @@ jobs: source /opt/lib/nuitka/bin/activate pip install --upgrade pip make -C /src executable + - name: Sanity check + uses: Eeems-Org/run-in-remarkable-action@v1 + with: + fw_version: 2.15.1 + run: | + rmufuse-remarkable --help + path: dist - uses: actions/upload-artifact@v3 with: name: rmufuse-remarkable From de4c2765e2d1328b3f30a54425f942f69175af84 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 14:13:19 -0700 Subject: [PATCH 02/15] Fix command name --- .github/workflows/build.yml | 4 ++-- .gitignore | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6467c2d..66ad4a8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -156,13 +156,13 @@ jobs: cp -a /opt/lib/nuitka .venv source /opt/lib/nuitka/bin/activate pip install --upgrade pip - make -C /src executable + make -C /src portable - name: Sanity check uses: Eeems-Org/run-in-remarkable-action@v1 with: fw_version: 2.15.1 run: | - rmufuse-remarkable --help + rmufuse --help path: dist - uses: actions/upload-artifact@v3 with: diff --git a/.gitignore b/.gitignore index 228ca04..aacf6dd 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,4 @@ cython_debug/ #.idea/ *.orig .nuitka/ +nuitka-crash-report.xml From 6d5e4fc3b5cbf60a23a07971a738a3271ff8d51f Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 14:20:06 -0700 Subject: [PATCH 03/15] Fix target --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66ad4a8..716307c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,6 +149,7 @@ jobs: env: src_path: ${{ github.workspace }} script: | + set -e apt update apt install -y \ protobuf-compiler \ @@ -156,14 +157,14 @@ jobs: cp -a /opt/lib/nuitka .venv source /opt/lib/nuitka/bin/activate pip install --upgrade pip - make -C /src portable + make -C /src executable - name: Sanity check uses: Eeems-Org/run-in-remarkable-action@v1 with: fw_version: 2.15.1 run: | rmufuse --help - path: dist + path: ${{ github.workspace }}/dist - uses: actions/upload-artifact@v3 with: name: rmufuse-remarkable From 3aa930b8d981e6bc6d6970626566f1d7ee8f51bb Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 14:54:37 -0700 Subject: [PATCH 04/15] portable build for remarkable --- .github/workflows/build.yml | 10 +++++++--- Makefile | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 716307c..20414a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -157,14 +157,14 @@ jobs: cp -a /opt/lib/nuitka .venv source /opt/lib/nuitka/bin/activate pip install --upgrade pip - make -C /src executable + make -C /src portable - name: Sanity check uses: Eeems-Org/run-in-remarkable-action@v1 with: fw_version: 2.15.1 run: | - rmufuse --help - path: ${{ github.workspace }}/dist + rmufuse-portable --help + path: dist - uses: actions/upload-artifact@v3 with: name: rmufuse-remarkable @@ -281,6 +281,10 @@ jobs: name="rmufuse-${{ matrix.artifact }}" mv rmufuse "$name" gh release upload "$TAG" "$name" --clobber + elif [ -f rmufuse-portable ]; then + name="rmufuse-${{ matrix.artifact }}" + mv rmufuse-portable "$name" + gh release upload "$TAG" "$name" --clobber else find . -type f | xargs -rI {} gh release upload "$TAG" {} --clobber fi diff --git a/Makefile b/Makefile index 6b97fed..e30f0f8 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ FW_DATA := wVbHkgKisg- PROTO_SOURCE := $(shell find protobuf -type f -name '*.proto') PROTO_OBJ := $(addprefix $(PACKAGE),$(PROTO_SOURCE:%.proto=%_pb2.py)) +FUSE_FOLDER := $(shell pkgconf --variable=libdir fuse) + OBJ := $(shell find ${PACKAGE} -type f) OBJ += requirements.txt OBJ += pyproject.toml @@ -85,7 +87,10 @@ dist/${PACKAGE}-${VERSION}-${ABI}-${ABI}-${PLATFORM}.whl: dist $(OBJ) dist/rmufuse: dist .venv/bin/activate $(OBJ) . .venv/bin/activate; \ - python -m pip install --extra-index-url=https://wheels.eeems.codes/ wheel nuitka; \ + python -m pip install \ + --extra-index-url=https://wheels.eeems.codes/ \ + wheel \ + nuitka; \ NUITKA_CACHE_DIR="$(realpath .)/.nuitka" \ nuitka3 \ --enable-plugin=pylint-warnings \ @@ -95,11 +100,33 @@ dist/rmufuse: dist .venv/bin/activate $(OBJ) --lto=yes \ --assume-yes-for-downloads \ --python-flag=-m \ - --remove-output \ --output-dir=dist \ + --remove-output \ --output-filename=rmufuse \ remarkable_update_fuse +dist/rmufuse-portable: dist .venv/bin/activate $(OBJ) + . .venv/bin/activate; \ + python -m pip install \ + --extra-index-url=https://wheels.eeems.codes/ \ + wheel \ + nuitka; \ + NUITKA_CACHE_DIR="$(realpath .)/.nuitka" \ + nuitka3 \ + --enable-plugin=pylint-warnings \ + --enable-plugin=upx \ + --warn-implicit-exceptions \ + --onefile \ + --lto=yes \ + --assume-yes-for-downloads \ + --python-flag=-m \ + --output-dir=dist \ + --remove-output \ + --output-filename=rmufuse-portable \ + '--include-data-files=${FUSE_FOLDER}/libfuse.so.2.*=libfuse.so.2' \ + remarkable_update_fuse + patchelf dist/rmufuse + .venv/bin/activate: requirements.txt @echo "Setting up development virtual env in .venv" python -m venv .venv @@ -145,9 +172,11 @@ test: .venv/bin/activate .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed $(OBJ . .venv/bin/activate; \ python test.py -executable: .venv/bin/activate dist/rmufuse +executable: dist/rmufuse dist/rmufuse --help +portable: dist/rmufuse-portable + all: release .PHONY: \ @@ -156,6 +185,7 @@ all: release clean \ dev \ executable \ + portable \ install \ release \ sdist \ From 21ce64e5af9fc609b660c469dee65859d840a048 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 15:07:09 -0700 Subject: [PATCH 05/15] Add missing dependencies --- .github/workflows/build.yml | 4 +++- Makefile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20414a4..7f9f7d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,6 +129,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build with nuitka run: | + set -e docker run \ --rm \ --platform=linux/arm/v7 \ @@ -153,7 +154,8 @@ jobs: apt update apt install -y \ protobuf-compiler \ - libfuse-dev + libfuse-dev \ + pkgconf cp -a /opt/lib/nuitka .venv source /opt/lib/nuitka/bin/activate pip install --upgrade pip diff --git a/Makefile b/Makefile index e30f0f8..caa9d90 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,7 @@ dist/rmufuse-portable: dist .venv/bin/activate $(OBJ) --output-dir=dist \ --remove-output \ --output-filename=rmufuse-portable \ - '--include-data-files=${FUSE_FOLDER}/libfuse.so.2.*=libfuse.so.2' \ + '--include-data-files=${FUSE_FOLDER}/libfuse.so=libfuse.so.2' \ remarkable_update_fuse patchelf dist/rmufuse From 188ed36a7eddca2a324e90774a1bdaa179205f58 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 15:34:07 -0700 Subject: [PATCH 06/15] Add make_for_remarkable.sh for easier local testing --- .github/workflows/build.yml | 27 +++++---------------------- make_for_remarkable.sh | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 22 deletions(-) create mode 100755 make_for_remarkable.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f9f7d9..dc59c9b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -130,36 +130,19 @@ jobs: - name: Build with nuitka run: | set -e - docker run \ - --rm \ - --platform=linux/arm/v7 \ - -v "${src_path}":/src \ - eeems/nuitka-arm-builder:bullseye-3.11 \ - bash -ec "$script" \ + ./make_for_remarkable.sh portable \ | while read -r line; do - if [[ "$line" == 'Nuitka'*':ERROR:'* ]]; then + if [[ "$line" == 'Nuitka'*':ERROR:'* ]] || [[ "$line" == "ERROR:"* ]]; then echo "::error file=${{ steps.args.outputs.main_file }},title=Nuitka Error::$line" - elif [[ "$line" == 'Nuitka'*':WARNING:'* ]]; then + elif [[ "$line" == 'Nuitka'*':WARNING:'* ]] || [[ "$line" == "WARNING:"* ]]; then echo "::warning file=${{ steps.args.outputs.main_file }},title=Nuitka Warning::$line" - elif [[ "$line" == 'Nuitka:INFO:'* ]]; then + elif [[ "$line" == 'Nuitka:INFO:'* ]] || [[ "$line" == "INFO:"* ]]; then echo "$line" else echo "::debug::$line" fi done - env: - src_path: ${{ github.workspace }} - script: | - set -e - apt update - apt install -y \ - protobuf-compiler \ - libfuse-dev \ - pkgconf - cp -a /opt/lib/nuitka .venv - source /opt/lib/nuitka/bin/activate - pip install --upgrade pip - make -C /src portable + working-directory: ${{ github.workspace }} - name: Sanity check uses: Eeems-Org/run-in-remarkable-action@v1 with: diff --git a/make_for_remarkable.sh b/make_for_remarkable.sh new file mode 100755 index 0000000..5359c52 --- /dev/null +++ b/make_for_remarkable.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +qemu_arm_flags=$(grep flags: /proc/sys/fs/binfmt_misc/qemu-arm | cut -d' ' -f2) +if ! echo "$qemu_arm_flags" | grep -q F; then + echo "INFO: Enabling linux/arm/v7 docker builds with qmeu" + docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7 +fi +script=" +apt update +apt install -y \\ + protobuf-compiler \\ + libfuse-dev \\ + pkgconf +cp -a /opt/lib/nuitka .venv +source /opt/lib/nuitka/bin/activate +pip install --upgrade pip build +make -C /src \"\$@\" +" +docker run \ + --rm \ + --platform=linux/arm/v7 \ + -v "$(pwd)":/src \ + eeems/nuitka-arm-builder:bullseye-3.11 \ + bash -ec "$script" "$@" From a0e28d12bfba5773fbcdf2fb9f97a762480fe1bc Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 15:36:13 -0700 Subject: [PATCH 07/15] Update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 90e29c3..75be5bb 100644 --- a/README.md +++ b/README.md @@ -51,4 +51,10 @@ make wheel # Build wheel package in dist/ make sdist # Build sdist package in dist/ make dev # Test mounting 2.15.1.1189 to .venv/mnt make install # Build wheel and install it with pipx or pip install --user +make executable # Build a standalone executable +make portable # Build a standalone executable with some extra dependencies embedded ``` + +### Building for the reMarkable (Or really any linux/arm/v7 device) + +The same as above, but use `./make_for_remarkable.sh` instead of `make`. This requires docker to be installed. From 5a75edd3499f884c88ea80fb2e632e872805e2b2 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 16:01:09 -0700 Subject: [PATCH 08/15] Run local file --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc59c9b..7017595 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -147,8 +147,7 @@ jobs: uses: Eeems-Org/run-in-remarkable-action@v1 with: fw_version: 2.15.1 - run: | - rmufuse-portable --help + run: ./rmufuse-portable --help path: dist - uses: actions/upload-artifact@v3 with: From cf873ffb4277757c13cb76fdd79eba89c9e5e961 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 20:52:30 -0700 Subject: [PATCH 09/15] Fix argument pass --- make_for_remarkable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make_for_remarkable.sh b/make_for_remarkable.sh index 5359c52..e3bedee 100755 --- a/make_for_remarkable.sh +++ b/make_for_remarkable.sh @@ -21,4 +21,4 @@ docker run \ --platform=linux/arm/v7 \ -v "$(pwd)":/src \ eeems/nuitka-arm-builder:bullseye-3.11 \ - bash -ec "$script" "$@" + bash -ec "$script" -- "$@" From 5c5cb8b4b3d4470acf9a5d817c52fce6a9379ed8 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 20:55:04 -0700 Subject: [PATCH 10/15] Use my token for installing protoc --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7017595..d2638ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,7 @@ jobs: uses: arduino/setup-protoc@v2 with: version: "25.x" + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout the Git repository uses: actions/checkout@v4 - uses: actions/setup-python@v4 @@ -55,6 +56,7 @@ jobs: uses: arduino/setup-protoc@v2 with: version: "25.x" + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout the Git repository uses: actions/checkout@v4 - uses: actions/setup-python@v4 @@ -174,6 +176,7 @@ jobs: uses: arduino/setup-protoc@v2 with: version: "25.x" + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install build tool run: pip install build - name: Building package @@ -199,6 +202,7 @@ jobs: uses: arduino/setup-protoc@v2 with: version: "25.x" + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install build tool run: pip install build - name: Building package From 8d3e931e1b1fc11ad1a52f3ded494b22a23f9100 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Fri, 19 Jan 2024 21:35:00 -0700 Subject: [PATCH 11/15] Add more dependencies --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index caa9d90..cb6747d 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,6 @@ FW_DATA := wVbHkgKisg- PROTO_SOURCE := $(shell find protobuf -type f -name '*.proto') PROTO_OBJ := $(addprefix $(PACKAGE),$(PROTO_SOURCE:%.proto=%_pb2.py)) -FUSE_FOLDER := $(shell pkgconf --variable=libdir fuse) - OBJ := $(shell find ${PACKAGE} -type f) OBJ += requirements.txt OBJ += pyproject.toml @@ -123,7 +121,9 @@ dist/rmufuse-portable: dist .venv/bin/activate $(OBJ) --output-dir=dist \ --remove-output \ --output-filename=rmufuse-portable \ - '--include-data-files=${FUSE_FOLDER}/libfuse.so=libfuse.so.2' \ + '--include-data-files=$(shell pkgconf --variable=libdir fuse)/libfuse.so=libfuse.so.2' \ + '--include-data-files=$(shell pkgconf --variable=libdir libssl)/libssl.so=libssl.so.1' \ + '--include-data-files=$(shell pkgconf --variable=libdir libcrypto)/libcrypto.so=libcrypto.so.3' \ remarkable_update_fuse patchelf dist/rmufuse From e63ad8615329dc17e4cfece6fb80e63d249a8c4a Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Thu, 4 Apr 2024 19:45:46 -0600 Subject: [PATCH 12/15] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb6747d..c61d2d8 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ dist/rmufuse-portable: dist .venv/bin/activate $(OBJ) '--include-data-files=$(shell pkgconf --variable=libdir libssl)/libssl.so=libssl.so.1' \ '--include-data-files=$(shell pkgconf --variable=libdir libcrypto)/libcrypto.so=libcrypto.so.3' \ remarkable_update_fuse - patchelf dist/rmufuse + patchelf --print-needed dist/rmufuse-portable .venv/bin/activate: requirements.txt @echo "Setting up development virtual env in .venv" From f401480a7490ca03607ee648da5575ae9ab32022 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Sun, 8 Sep 2024 13:24:25 -0600 Subject: [PATCH 13/15] Fix .venv --- make_for_remarkable.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/make_for_remarkable.sh b/make_for_remarkable.sh index e3bedee..535b067 100755 --- a/make_for_remarkable.sh +++ b/make_for_remarkable.sh @@ -11,8 +11,8 @@ apt install -y \\ protobuf-compiler \\ libfuse-dev \\ pkgconf -cp -a /opt/lib/nuitka .venv -source /opt/lib/nuitka/bin/activate +cp -a /opt/lib/nuitka /src/.venv +source /src/.venv/bin/activate pip install --upgrade pip build make -C /src \"\$@\" " From e9570307ede6b079cdd2820a2ee1befb1dbe1159 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Sun, 8 Sep 2024 14:12:32 -0600 Subject: [PATCH 14/15] Fix source line --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 739610b..15df403 100644 --- a/Makefile +++ b/Makefile @@ -105,7 +105,7 @@ dist/rmufuse: dist $(VENV_BIN_ACTIVATE) $(OBJ) remarkable_update_fuse dist/rmufuse-portable: dist $(VENV_BIN_ACTIVATE) $(OBJ) - $(VENV_BIN_ACTIVATE); \ + . $(VENV_BIN_ACTIVATE); \ python -m pip install \ --extra-index-url=https://wheels.eeems.codes/ \ wheel \ From 595aa0df5c99b5a8db027eb10e665c5de17de620 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Sun, 8 Sep 2024 14:59:30 -0600 Subject: [PATCH 15/15] Upload even if sanity check fails --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1eff9c6..037d7bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -155,17 +155,17 @@ jobs: fi done working-directory: ${{ github.workspace }} + - uses: actions/upload-artifact@v4 + with: + name: rmufuse-remarkable + path: dist + if-no-files-found: error - name: Sanity check uses: Eeems-Org/run-in-remarkable-action@v1 with: fw_version: 2.15.1 run: ./rmufuse-portable --help path: dist - - uses: actions/upload-artifact@v4 - with: - name: rmufuse-remarkable - path: dist - if-no-files-found: error build-wheel: name: Build wheel with python ${{ matrix.python }} needs: [test]