From 373d3db35ffada7776268e69adb597fa7e8d5f7e Mon Sep 17 00:00:00 2001
From: exquo <62397152+exquo@users.noreply.github.com>
Date: Thu, 23 Mar 2023 07:12:16 +0000
Subject: [PATCH] Simplify matrix

---
 .github/workflows/libclient-build.yaml |  72 +++++++-----
 .gitignore                             |   1 +
 filename_for_matrix_item.py            |  59 +++-------
 generate_matrix.py                     | 154 +++++++++----------------
 util.py                                |  15 +++
 util.sh                                |   7 ++
 6 files changed, 137 insertions(+), 171 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 util.py

diff --git a/.github/workflows/libclient-build.yaml b/.github/workflows/libclient-build.yaml
index eb7e9ef..d80dca8 100644
--- a/.github/workflows/libclient-build.yaml
+++ b/.github/workflows/libclient-build.yaml
@@ -9,6 +9,7 @@ on:
   schedule:
     - cron: '33 3 * * *'
 
+
 defaults:
   run:
     shell: bash   # Explicit for windows
@@ -45,10 +46,9 @@ jobs:
           [ -n "$upstream_ver" ] || upstream_ver=$(bash util.sh get_latest_release_name "$LIB_REPO")
           echo "upstream_ver=$upstream_ver" | tee -a $GITHUB_OUTPUT
           release_name=$(bash util.sh release_name "$LIB_REPO" "$upstream_ver")
-          echo "release_name = $release_name"
           this_repo_release=$(bash util.sh get_release_data "$release_name")
           [ -z "$this_repo_release" ] || release_name=''
-          echo "release_name=$release_name" >> $GITHUB_OUTPUT
+          echo "release_name=$release_name" | tee -a $GITHUB_OUTPUT
 
 
   new_release:
@@ -79,6 +79,7 @@ jobs:
 
 
   matrix_setup:
+
     needs:
       - new_release
       - compare_releases
@@ -110,12 +111,13 @@ jobs:
       matrix: ${{ fromJSON(needs.matrix_setup.outputs.matrix) }}
       fail-fast: false  # do not abort all if some of the builds fail
 
-    name: build_${{ matrix.cross.target || matrix.host.triple }}
+    name: build_${{ matrix.build-env.target || matrix.build-env.triple }}
 
-    runs-on: ${{ matrix.host.runner }}
+    runs-on: ${{ matrix.build-env.runner }}
+    container: ${{ matrix.build-env.container }}
 
     env:
-      TARGET: ${{ matrix.cross.target }}
+      TARGET: ${{ matrix.build-env.target }}
       THIS_REPO_DIR: _this_repo_checkout
 
     # Ref:
@@ -134,9 +136,19 @@ jobs:
       - name: Checkout this repo sources
         uses: actions/checkout@v3
 
+      - name: Install required packages
+        if: ${{ matrix.build-env.req-pkg }}
+        env:
+          INSTALL_CMD: ${{ matrix.build-env.install-cmd }}
+          PKGS: ${{ matrix.build-env.req-pkg }}
+        run: |
+          bash -c "$INSTALL_CMD $PKGS"
+
       - name: Generate file names for the current matrix item
         id: filenames
-        run: python3 filename_for_matrix_item.py '${{ needs.matrix_setup.outputs.matrix }}' '${{ matrix.lib.name }}' '${{ matrix.host.runner }}' '${{ matrix.cross.target }}'
+        env:
+          MATRIX_RUN: ${{ toJSON(matrix) }}
+        run: python3 filename_for_matrix_item.py "$MATRIX_RUN"
 
       - name: Clone upstream source
         uses: actions/checkout@v3
@@ -146,33 +158,26 @@ jobs:
 
       - name: Install Rust toolchain
         run: rustup toolchain install nightly --profile minimal
-        # Why nightly: https://github.com/signalapp/libsignal/issues/141#issuecomment-1211192153
+          # Why nightly: https://github.com/signalapp/libsignal/issues/141#issuecomment-1211192153
 
       - name: Add target to rust toolchain
-        if: ${{ matrix.cross.target }}
+        if: ${{ matrix.build-env.target }}
         run: rustup target add $TARGET
 
-      - name: Install required packages
-        if: ${{ matrix.host.req-pkg || matrix.cross.req-pkg }}
-        run: |
-          bash -c "$INSTALL_CMD $PKGS"
-        env:
-          INSTALL_CMD: ${{ matrix.host.install-cmd }}
-          PKGS: ${{ matrix.host.req-pkg }} ${{ matrix.cross.req-pkg }}
-
       - name: Add linker to cargo config
         # Alternatively can add to rust flags
-          #RUSTFLAGS: -C linker=${{ matrix.cross.linker }}
-        if: ${{ matrix.cross.linker }}
+          #RUSTFLAGS: -C linker=${{ matrix.build-env.linker }}
+        if: ${{ matrix.build-env.linker }}
+        env:
+          LINKER: ${{ matrix.build-env.linker }}
         run: |
-          cat >>~/.cargo/config <<EOF
+          cat >> ~/.cargo/config <<EOF
           [target.$TARGET]
           linker = "$LINKER"
           EOF
-        env:
-          LINKER: ${{ matrix.cross.linker }}
 
       - name: Setup rust cache
+        if: false   # Not using cache
         # On macos-latest cache does not work properly:
           # https://github.com/actions/cache/issues/403
           # https://github.com/rust-lang/cargo/issues/8603
@@ -184,40 +189,43 @@ jobs:
           path: |
             ~/.cargo
             ./target
-          key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cross.target }}-${{ matrix.lib.name }}-${{ hashFiles('Cargo.lock') }}
+          key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.build-env.target }}-${{ matrix.lib.name }}-${{ hashFiles('Cargo.lock') }}
           restore-keys: |
-            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cross.target }}-${{ matrix.lib.name }}-
-            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.cross.target }}-
+            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.build-env.target }}-${{ matrix.lib.name }}-
+            ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.build-env.target }}-
 
       - name: Cargo build
+        env:
+          RUSTFLAGS: -C link-arg=-s  ${{ matrix.build-env.rust-flags }}
+          CARGO_FLAGS: ${{ matrix.lib.cargo-flags }}
+          BUILD_ENV_VARS: ${{ matrix.build-env.build-env-vars }}
         run: |
           if [[ -n $TARGET ]]; then
             export CARGO_BUILD_TARGET=$TARGET
           fi
           env $BUILD_ENV_VARS cargo build --release --verbose  $CARGO_FLAGS
-        env:
-          RUSTFLAGS: -C link-arg=-s  ${{ matrix.host.rust-flags }} ${{ matrix.cross.rust-flags }}
-          CARGO_FLAGS: ${{ matrix.lib.cargo-flags }}
-          BUILD_ENV_VARS: ${{ matrix.host.build-env-vars }} ${{ matrix.cross.build-env-vars }}
 
       - name: Inspect built file
         env:
           FILENAME: ${{ steps.filenames.outputs.lib_filename }}
-        working-directory: target/${{ matrix.cross.target }}/release
+        working-directory: target/${{ matrix.build-env.target }}/release
         run: |
           file $FILENAME
-          ldd $FILENAME || :
+          readelf --arch-specific $FILENAME  || :
+          ldd -v $FILENAME || :
           objdump -T $FILENAME | grep LIBC  || :
+          objdump -T $FILENAME  | grep LIBC | cut -d'_' -f2 | sort -V | tail -n1 | cut -d ' ' -f1 || :
           openssl sha256 $FILENAME
 
       - name: Create archive
         env:
-          DIR: target/${{ matrix.cross.target }}/release
+          DIR: target/${{ matrix.build-env.target }}/release
           FILENAME: ${{ steps.filenames.outputs.lib_filename }}
           ARCHIVE_NAME: ${{ steps.filenames.outputs.archive_name }}
         run: tar -czvf ${ARCHIVE_NAME}.tar.gz --directory ${DIR}  ${FILENAME}
 
-      - name: Checkout this repo (for gh cli)
+      - name: Checkout this repo
+          # (for gh cli)
         uses: actions/checkout@v3
         with:
           path: ${{ env.THIS_REPO_DIR }}
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..bee8a64
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+__pycache__
diff --git a/filename_for_matrix_item.py b/filename_for_matrix_item.py
index 4a8715e..6d49f19 100644
--- a/filename_for_matrix_item.py
+++ b/filename_for_matrix_item.py
@@ -1,52 +1,31 @@
 #!/usr/bin/env python3
 
 import json
-import os
 import sys
 
-matrix = json.loads(sys.argv[1])
-lib_name = sys.argv[2]
-host_runner = sys.argv[3]
-try:
-    cross_target = sys.argv[4]
-except IndexError:
-    cross_target = None
-
-def get_matrix_item(matrix, matrix_key, list_key, list_key_val):
-    matrix_key_list = matrix.get(matrix_key) or []
-    for item in matrix_key_list:
-        if item and item[list_key] == list_key_val:
-            return item
-    for item in matrix["include"]:
-        include_item = item[matrix_key]
-        if include_item[list_key] == list_key_val:
-            return include_item
-    raise ValueError
-
-lib_dict = get_matrix_item(matrix, "lib", "name", lib_name)
-host_dict = get_matrix_item(matrix, "host", "runner", host_runner)
-cross_dict = get_matrix_item(matrix, "cross", "target", cross_target) if cross_target else None
-
-lib_filename = "".join([
-    host_dict.get("lib-prefix") or "",
-    lib_dict["filename"],
-    host_dict["lib-suffix"]
-    ])
+import util
+
+
+matrix_run = json.loads(sys.argv[1])
+
+lib = matrix_run["lib"]
+build_env = matrix_run["build-env"]
+
+lib_filename = "".join((
+    build_env.get("lib-prefix") or "",
+    lib["filename"],
+    build_env["lib-suffix"]
+    ))
 print(lib_filename)
 
-archive_name = "".join([
+archive_name = "".join((
     lib_filename,
     "-",
-    lib_dict["ref"],
+    lib["ref"],
     "-",
-    cross_dict["target"] if cross_dict else host_dict["triple"]
-    ])
+    build_env.get("target") or build_env["triple"]
+    ))
 print(archive_name)
 
-gh_output_file = os.environ["GITHUB_OUTPUT"]
-with open(gh_output_file, 'a') as gh_out:
-    gh_out.write('\n'.join((
-        f"lib_filename={lib_filename}",
-        f"archive_name={archive_name}",
-        ''
-        )))
+util.gha_set_output_param("lib_filename", lib_filename)
+util.gha_set_output_param("archive_name", archive_name)
diff --git a/generate_matrix.py b/generate_matrix.py
index 49181bd..92a0915 100644
--- a/generate_matrix.py
+++ b/generate_matrix.py
@@ -2,32 +2,31 @@
 
 import json
 import pprint
-import os
 import sys
 
-print(sys.argv)
-libs_ver = {}
-libs_ver["libsignal"] = sys.argv[1]
+import util
+
 
+print(sys.argv)
+libs_ver= {"libsignal": sys.argv[1]}
 
 libs = {
-        #"zkgroup": {
-            # zkgroup is included in libsignal-client v0.10.0, and is no longer a dependency in libsignal-service-java
-            #"repo": "signalapp/zkgroup",
-            #"jar_name": "zkgroup-java",
-            #"filename": "zkgroup",
-            #},
         "libsignal": {
             "repo": "signalapp/libsignal",
-            "jar_name": "signal-client-java",
             "filename": "signal_jni",
             "cargo-flags": "-p libsignal-jni",
             },
+        #"zkgroup": {
+            ### UPD: zkgroup is included in libsignal-client v0.10.0, and is no longer a dependency in libsignal-service-java
+            #"repo": "signalapp/zkgroup",
+            #"jar_name": "zkgroup-java",
+            #"filename": "zkgroup",
+            #},
         }
 
 hosts = {
         "linux": {
-                "runner": "ubuntu-18.04",
+                "runner": "ubuntu-20.04",
                 "lib-prefix": "lib",
                 "lib-suffix": ".so",
                 "triple": "x86_64-unknown-linux-gnu",
@@ -39,107 +38,64 @@
                 "lib-suffix": ".dylib",
                 "triple": "x86_64-apple-darwin",
             },
-        #"macos-aarch64": {
-                #"runner": "macos-11",
-                #"lib-prefix": "lib",
-                #"lib-suffix": ".dylib",
-                #"triple": "aarch64-apple-darwin",
-                    ## macos-11 runner uses x86-64 processers.
-                    ## Using a cross target aarch64-apple (below)
-            #},
         "windows": {
                 "runner": "windows-latest",
                 "lib-suffix": ".dll",
-                "triple": "x86_64-pc-windows",  # no "-msvc" because static lib, see next line
+                "triple": "x86_64-pc-windows",
                 "install-cmd": "choco install",
-                "req-pkg": "nasm",  # req for boringssl
+                "req-pkg": "nasm",  # for boringssl
                 "rust-flags": "-C target-feature=+crt-static",
-                        # Static linking to remove MSVC dependendency.
-                        # zkgroup/ffi/node/Makefile
-                        # libsignal-client/node/build_node_bridge.py
-            },
+                    # Static linking to remove MSVC dependendency. See:
+                    # zkgroup/ffi/node/Makefile
+                    # libsignal-client/node/build_node_bridge.py
+                },
         }
 
-cross_targets = [
-        {
-            "target": "aarch64-unknown-linux-gnu",
-            "req-pkg": "gcc-aarch64-linux-gnu g++-aarch64-linux-gnu",
-            "linker": "aarch64-linux-gnu-gcc",
-            "build-env-vars": "CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ CPATH=/usr/aarch64-linux-gnu/include",
-                #https://github.com/signalapp/libsignal/issues/482#issuecomment-1220896664
-        },
-        {
-            "target": "armv7-unknown-linux-gnueabihf",
-            "req-pkg": "gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf",
-            "linker": "arm-linux-gnueabihf-gcc",
-            "build-env-vars": "CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ CPATH=/usr/arm-linux-gnueabihf/include RUST_BACKTRACE=1",
-        },
-        {
-            "target": "i686-unknown-linux-gnu",
-            "req-pkg": "gcc-i686-linux-gnu g++-i686-linux-gnu",
-            "linker": "i686-linux-gnu-gcc",
-            "build-env-vars": "CC=i686-linux-gnu-gcc CXX=i686-linux-gnu-g++ CPATH=/usr/i686-linux-gnu/include",
-        },
-        {
-            "target": "aarch64-apple-darwin",
-            "host": {
-                "runner": "macos-11",
-                "lib-prefix": "lib",
-                "lib-suffix": ".dylib",
+def cross_template(arch, subarch="", env="gnu", vendor="unknown", sys_os="linux", compilers=None):
+    compilers = compilers or {"C": "gcc", "C++": "g++"}
+    host_dict = hosts.get(sys_os, {})
+    cross_dict = {
+            "target": f"{arch}{subarch}-{vendor}-{sys_os}-{env}",
+            "req-pkg": " ".join((
+                f"{compiler}-{arch}-{sys_os}-{env}" for compiler in compilers.values()
+                )),
+            "linker": f"{arch}-{sys_os}-{env}-{compilers['C']}",
+            "build-env-vars": " ".join((
+                f"CC={arch}-{sys_os}-{env}-{compilers['C']}",
+                f"CXX={arch}-{sys_os}-{env}-{compilers['C++']}",
+                f"CPATH=/usr/{arch}-{sys_os}-{env}/include",
+                )),
+            }
+    return host_dict | cross_dict
+
+build_envs = [
+        hosts["linux"] | {
+            ### Rust container on Debian 10
+            "container": "rust:buster",
+            "install-cmd": "bash ./util.sh add_gh_ppa && apt-get update && apt-get install -y",
+            "req-pkg": "python3 clang libclang-dev cmake make gh",
             },
-        },
-    ]
+        hosts["macos"],
+        hosts["windows"],
+        ### Cross-compiling ###
+        cross_template("aarch64"),
+        cross_template("arm", "v7", "gnueabihf"),
+        cross_template("i686"),
+        hosts["macos"] | {"target": "aarch64-apple-darwin"},
+        ]
+
 
+for lib_name, lib_ver in libs_ver.items():
+    libs[lib_name]["ref"] = lib_ver
 
 matrix = {
-        "lib": [],
-        "host": [],
-        "cross": [None],
+        "lib": list(libs.values()),
+        "build-env": build_envs,
         "include": [],
         }
 
-
-for lib_name, lib_dict in libs.items():
-    if not libs_ver[lib_name]:
-        continue
-    lib_dict["name"] = lib_name
-    lib_dict["ref"] = libs_ver[lib_name]
-    matrix["lib"].append(lib_dict)
-
-matrix["host"] = list(hosts.values())
-
-target_to_host = {"linux": "linux", "apple": "macos"}
-for cross in cross_targets:
-    try:
-        host = cross.pop("host")
-    except KeyError:
-        for target_partial_name, host_name in target_to_host.items():
-            if target_partial_name in cross["target"]:
-                host = hosts[host_name]
-                break
-    for lib in matrix["lib"]:
-        matrix["include"].append(
-                {
-                    "lib": lib,
-                    "host": host,
-                    "cross": cross,
-                }
-            )
-
-if not matrix.get("host"):
-    matrix = {"include": matrix["include"]}
-    jobs_total = len(matrix["include"])
-else:
-    jobs_total = 1
-    for jobs_num in (len(key_array) for key, key_array in matrix.items() if key != "include"):
-        jobs_total *= jobs_num
-    jobs_total += len(matrix["include"])
+jobs_total = len(libs) * len(build_envs) + len(matrix.get("include", []))
 print("Total num of jobs:", jobs_total)
 
 pprint.pprint(matrix)
-
-j = json.dumps(matrix)
-gh_output_file = os.environ["GITHUB_OUTPUT"]
-with open(gh_output_file, 'a') as gh_out:
-    gh_out.write(f"matrix={j}\n")
-
+util.gha_set_output_param("matrix", json.dumps(matrix))
diff --git a/util.py b/util.py
new file mode 100644
index 0000000..f33cb4e
--- /dev/null
+++ b/util.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+
+def gha_set_output_param(name, value):
+    gh_output_file = os.environ["GITHUB_OUTPUT"]
+    with open(gh_output_file, 'a') as gh_out:
+        gh_out.write(f"{name}={value}\n")
+
+
+if __name__ == "__main__":
+    fn = globals()[sys.argv[1]]
+    fn(*sys.argv[2:])
diff --git a/util.sh b/util.sh
index 521773e..e75420f 100755
--- a/util.sh
+++ b/util.sh
@@ -128,5 +128,12 @@ gradle_deps_vers () {
 		done
 }
 
+add_gh_ppa() {
+	# From https://github.com/cli/cli/blob/trunk/docs/install_linux.md, sans `sudo` (for docker)
+	type -p curl >/dev/null || apt install curl -y
+	curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+	&& chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+}
 
 "$@"