diff --git a/.github/workflows/stable-cargo-quickinstall.yml b/.github/workflows/stable-cargo-quickinstall.yml new file mode 100644 index 00000000..a6cfcf08 --- /dev/null +++ b/.github/workflows/stable-cargo-quickinstall.yml @@ -0,0 +1,143 @@ +name: cargo-quickinstall stable build + +on: + push: + branches: [ trigger/stable ] + workflow_dispatch: + +env: + index: https://github.com/cargo-prebuilt/index/releases/download/stable-index/ + crate: cargo-quickinstall + version: 0.2.10 + dl: https://static.crates.io/crates/cargo-quickinstall/cargo-quickinstall-0.2.10.crate + checksum: 714f691dbeaf4b11d53c86dac4d5fb4477eb89da4d7a30d53762517b3ace4748 + git: https://github.com/cargo-bins/cargo-quickinstall + bins: cargo-quickinstall + file: ./crates/cargo-quickinstall.toml + CARGO_TERM_COLOR: always + python-version: "3.12" + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Cache + uses: actions/cache@v4 + id: cache + with: + path: | + build + key: ${{ env.crate }}-${{ env.version }}-stable-crate + enableCrossOsArchive: true + - name: Create Folder + if: ${{ !steps.cache.outputs.cache-hit }} + run: mkdir -p ./build + - name: Download crate and check hash + if: ${{ !steps.cache.outputs.cache-hit }} + run: | + wget ${{ env.dl }} + echo "${{ env.checksum }} ${{ env.crate }}-${{ env.version }}.crate" | sha256sum -c + tar -xf ${{ env.crate }}-${{ env.version }}.crate + mv ${{ env.crate }}-${{ env.version }}/* ./build + - name: Update Rust + if: ${{ !steps.cache.outputs.cache-hit }} + run: | + rustup update + rustc --version + - name: Generated lockfile if needed + if: ${{ !steps.cache.outputs.cache-hit }} + working-directory: ./build + run: test -f Cargo.lock || cargo +stable generate-lockfile --verbose + + t1-aarch64-apple-darwin: + strategy: + fail-fast: false + matrix: + target: [ x86_64-apple-darwin, aarch64-apple-darwin ] + runs-on: macos-14 + needs: [ setup ] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.python-version }} + - name: Get crate from cache + uses: actions/cache@v4 + with: + path: | + build + key: ${{ env.crate }}-${{ env.version }}-stable-crate + enableCrossOsArchive: true + fail-on-cache-miss: true +# - uses: Swatinem/rust-cache@v2 +# if: ${{ !false }} +# with: +# workspaces: "./build -> target" +# prefix-key: "v0-rust-${{ matrix.target }}-${{ env.crate }}-${{ env.version }}" + - name: Update Rust and Add Target + run: | + rustup update + rustc --version + rustup target add ${{ matrix.target }} + - uses: cargo-prebuilt/cargo-prebuilt-action@v3 + with: + pkgs: cargo-auditable + - name: Build crate + working-directory: ./build + run: cargo +stable auditable build --verbose --release --locked --target ${{ matrix.target }} + - name: Collect + run: python ./scripts/collect.py ${{ matrix.target }} ./build/target/${{ matrix.target }}/release ${{ env.bins }} + - name: Artifact + uses: actions/upload-artifact@v4 + with: + name: target-${{ matrix.target }}-macos14 + path: | + ${{ matrix.target }}.tar.gz + ${{ matrix.target }}.hashes.json + +# t1-apple-darwin: +# strategy: +# fail-fast: false +# matrix: +# target: [ x86_64-apple-darwin, aarch64-apple-darwin ] +# runs-on: macos-latest +# needs: [ setup ] +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-python@v5 +# with: +# python-version: ${{ env.python-version }} +# - name: Get crate from cache +# uses: actions/cache@v4 +# with: +# path: | +# build +# key: ${{ env.crate }}-${{ env.version }}-stable-crate +# enableCrossOsArchive: true +# fail-on-cache-miss: true +## - uses: Swatinem/rust-cache@v2 +## if: ${{ !false }} +## with: +## workspaces: "./build -> target" +## prefix-key: "v0-rust-${{ matrix.target }}-${{ env.crate }}-${{ env.version }}" +# - name: Update Rust and Add Target +# run: | +# rustup update +# rustc --version +# rustup target add ${{ matrix.target }} +# - uses: cargo-prebuilt/cargo-prebuilt-action@v3 +# with: +# pkgs: cargo-auditable +# - name: Build crate +# working-directory: ./build +# run: cargo +stable auditable build --verbose --release --locked --target ${{ matrix.target }} +# - name: Collect +# run: python ./scripts/collect.py ${{ matrix.target }} ./build/target/${{ matrix.target }}/release ${{ env.bins }} +# - name: Artifact +# uses: actions/upload-artifact@v4 +# with: +# name: target-${{ matrix.target }} +# path: | +# ${{ matrix.target }}.tar.gz +# ${{ matrix.target }}.hashes.json diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..7658bf3b --- /dev/null +++ b/Cross.toml @@ -0,0 +1,11 @@ +[target.x86_64-unknown-freebsd] +image = "ghcr.io/cargo-prebuilt/cross-openssl:x86_64-unknown-freebsd" + +[target.x86_64-unknown-netbsd] +image = "ghcr.io/cargo-prebuilt/cross-openssl:x86_64-unknown-netbsd" + +[target.x86_64-unknown-illumos] +image = "ghcr.io/cargo-prebuilt/cross-openssl:x86_64-unknown-illumos" + +[target.powerpc64-unknown-linux-gnu] +image = "ghcr.io/cargo-prebuilt/cross-openssl:powerpc64-unknown-linux-gnu" diff --git a/crates/cargo-quickinstall.toml b/crates/cargo-quickinstall.toml new file mode 100644 index 00000000..65132edb --- /dev/null +++ b/crates/cargo-quickinstall.toml @@ -0,0 +1,5 @@ +[info] +id = "cargo-quickinstall" +git = "https://github.com/cargo-bins/cargo-quickinstall" +unsupported = [] +bins = ["cargo-quickinstall"] diff --git a/keys/cargo-prebuilt-index.pub b/keys/cargo-prebuilt-index.pub new file mode 100644 index 00000000..b3bb3eb1 --- /dev/null +++ b/keys/cargo-prebuilt-index.pub @@ -0,0 +1,2 @@ +untrusted comment: minisign public key 51FF575479E09402 +RWQClOB5VFf/UXuhG+697EOSWlSyIPWjyehJpepjgQ7qsLnZxGQzDnqA \ No newline at end of file diff --git a/keys/cargo-prebuilt-index.pub.base64 b/keys/cargo-prebuilt-index.pub.base64 new file mode 100644 index 00000000..a48eec5c --- /dev/null +++ b/keys/cargo-prebuilt-index.pub.base64 @@ -0,0 +1 @@ +RWQClOB5VFf/UXuhG+697EOSWlSyIPWjyehJpepjgQ7qsLnZxGQzDnqA \ No newline at end of file diff --git a/scripts/check.py b/scripts/check.py new file mode 100644 index 00000000..fbedf474 --- /dev/null +++ b/scripts/check.py @@ -0,0 +1,140 @@ +import concurrent.futures +import glob +import json +import sys +import tomllib +import urllib.request +from typing import Any, Optional + + +stable_index: str = "/releases/download/stable-index/" +banned_index: str = "/releases/download/banned-index/" +crates_io_index: str = "https://index.crates.io/" +crates_io_cdn: str = "https://static.crates.io/crates/{CRATE}/{CRATE}-{VERSION}.crate" + + +def get_index_url(crate: str) -> str: + crate: str = crate.lower() + length: int = len(crate) + url: str = crates_io_index + if 1 <= length <= 2: + url += f"{length}/{crate}" + elif length == 3: + url += f"3/{crate[0]}/{crate}" + else: + url += f"{crate[0:2]}/{crate[2:4]}/{crate}" + return url + + +def get_newest_crate(versions: list[Any]) -> Any: + latest: Optional[Any] = None + store = (-1, -1, -1) + for v in versions: + if "-" in v["vers"]: + pass + elif not v["yanked"]: + semver = v["vers"].split(".") + semver = (int(semver[0]), int(semver[1]), int(semver[2])) + if semver[0] > store[0]: + store = semver + latest = v + elif semver[0] == store[0]: + if semver[1] > store[1]: + store = semver + latest = v + elif semver[1] == store[1]: + if semver[2] > store[2]: + store = semver + latest = v + return latest + + +def process(filename: str, pull_request: bool, allow: list[str], server_url: str, repo: str): + with open(filename, "rb") as file: + crate_toml = tomllib.load(file) + crate: str = crate_toml["info"]["id"] + + if (not pull_request) or (len(allow) == 0 or crate in allow): + if not pull_request: + try: + res = urllib.request.urlopen(f"{server_url}/{repo}{banned_index}{crate}") + if res.status == 200: + return None + except urllib.error.HTTPError: + pass + + version: str = "" + try: + res = urllib.request.urlopen(f"{server_url}/{repo}{stable_index}{crate}") + version = (res.read().decode("utf-8").strip()) + except urllib.error.HTTPError: + pass + + # Get from index.crates.io + req = urllib.request.Request( + get_index_url(crate), + data=None, + headers={ + "User-Agent": f"cargo-prebuilt_bot ({server_url}/{repo})" + } + ) + res = urllib.request.urlopen(req) + crate_infos_raw: str = res.read().decode("utf-8") if res and res.status == 200 else sys.exit(3) + crate_infos_raw: list[str] = crate_infos_raw.strip().split("\n") + + crate_infos: list[Any] = [] + for c in crate_infos_raw: + crate_infos.append(json.loads(c)) + + latest_crate: Any = get_newest_crate(crate_infos) + + if pull_request or version != latest_crate["vers"]: + return { + "crate": crate, + "version": latest_crate["vers"], + "dl": crates_io_cdn.replace("{CRATE}", crate).replace("{VERSION}", latest_crate["vers"]), + "checksum": latest_crate["cksum"], + "file": filename, + } + + return None + + +def main(pull_request: str, duplicate: str, server_url: str, repo: str): + pull_request: bool = True if pull_request.lower() == "true" else False + duplicate: bool = True if duplicate.lower() == "true" else False + + if not pull_request and duplicate: + print("{}") + return + + if pull_request: + with open("./pr/_allowlist", "r") as file: + allow: str = file.readline() + else: + allow: str = "" + allow: list[str] = allow.split(",") + + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + to_update_raw = executor.map(lambda f: process(f, pull_request, allow, server_url, repo), glob.glob("./crates/*.toml")) + + to_update = [] + for i in to_update_raw: + if i is not None: + to_update.append(i) + + x = { + "include": [] + } + for c in to_update: + x["include"].append(c) + + if len(x["include"]) == 0: + print("{}") + else: + print(json.dumps(x)) + + +if __name__ == "__main__": + argv = sys.argv + main(argv[1], argv[2], argv[3], argv[4]) diff --git a/scripts/collect.py b/scripts/collect.py new file mode 100644 index 00000000..a5173d88 --- /dev/null +++ b/scripts/collect.py @@ -0,0 +1,63 @@ +import hashlib +import json +import os +import stat +import sys +import tarfile + + +def main(target, build_path, bins): + bins = bins.split(",") + + hash_obj = { + "bins": [], + "archive": [], + } + + ending = "" + if "windows" in target: + ending = ".exe" + + with tarfile.open(target + ".tar.gz", "w:gz") as archive: + for b in bins: + basename = b + ending + path = build_path + "/" + basename + + # Permission Fix + if "windows" not in target: + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC) + + # Hashes + with open(path, "rb") as file: + file = file.read() + h = hashlib.sha256(file).hexdigest() + hash_obj["bins"].append({"bin": basename, "hash": h, "type": "sha256"}) + h = hashlib.sha512(file).hexdigest() + hash_obj["bins"].append({"bin": basename, "hash": h, "type": "sha512"}) + h = hashlib.sha3_256(file).hexdigest() + hash_obj["bins"].append({"bin": basename, "hash": h, "type": "sha3_256"}) + h = hashlib.sha3_512(file).hexdigest() + hash_obj["bins"].append({"bin": basename, "hash": h, "type": "sha3_512"}) + + # Add to archive + archive.add(path, basename) + + with open(target + ".tar.gz", "rb") as file: + file = file.read() + h = hashlib.sha256(file).hexdigest() + hash_obj["archive"].append({"hash": h, "type": "sha256"}) + h = hashlib.sha512(file).hexdigest() + hash_obj["archive"].append({"hash": h, "type": "sha512"}) + h = hashlib.sha3_256(file).hexdigest() + hash_obj["archive"].append({"hash": h, "type": "sha3_256"}) + h = hashlib.sha3_512(file).hexdigest() + hash_obj["archive"].append({"hash": h, "type": "sha3_512"}) + + with open(target + ".hashes.json", "w") as file: + file.write(json.dumps(hash_obj)) + + +if __name__ == "__main__": + argv = sys.argv + main(argv[1], argv[2], argv[3]) diff --git a/scripts/crate-info.py b/scripts/crate-info.py new file mode 100644 index 00000000..207f7c00 --- /dev/null +++ b/scripts/crate-info.py @@ -0,0 +1,18 @@ +import tomllib +import sys + + +def main(item: str): + with open("Cargo.toml", "rb") as file: + cargo_toml = tomllib.load(file) + package = cargo_toml["package"] + + if item in package: + print(package[item].replace("'", "%%SINGLE_QUOTE%%")) + else: + print("") + + +if __name__ == "__main__": + argv = sys.argv + main(argv[1]) diff --git a/scripts/docker/entrypoint.sh b/scripts/docker/entrypoint.sh new file mode 100755 index 00000000..0de71f93 --- /dev/null +++ b/scripts/docker/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euxo pipefail + +rustup update + +exec cargo +stable "$@" diff --git a/scripts/gen.py b/scripts/gen.py new file mode 100644 index 00000000..06254752 --- /dev/null +++ b/scripts/gen.py @@ -0,0 +1,130 @@ +import tomllib +import sys +import misc + +t2_targets: list[str] = [ + "riscv64gc-unknown-linux-gnu", # Optional Support (64-bit) + "s390x-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + + "armv7-unknown-linux-gnueabihf", # Optional Support (32-bit) + "armv7-unknown-linux-musleabihf", +] + +win_targets: list[str] = [ + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc" +] + +t3_targets: list[str] = [ + "x86_64-unknown-freebsd", + "x86_64-unknown-netbsd", + "x86_64-unknown-illumos", + "powerpc64-unknown-linux-gnu", +] + + +def main(pull_request: str, index: str, crate: str, version: str, dl: str, checksum: str, filename: str): + pull_request: bool = True if pull_request == "true" else False + + with open(filename, "rb") as file: + crate_toml = tomllib.load(file) + unsupported: str = crate_toml["info"]["unsupported"] + git_url: str = crate_toml["info"]["git"] + bins: str = ",".join(crate_toml["info"]["bins"]) + + with open("./stable.template.yml", "r") as file: + action_template: str = file.read() + + action = action_template.replace("%%INDEX%%", index) + action = action.replace("%%CRATE%%", crate) + action = action.replace("%%VERSION%%", version) + action = action.replace("%%DOWNLOAD%%", dl) + action = action.replace("%%CHECKSUM%%", checksum) + action = action.replace("%%GIT%%", git_url) + action = action.replace("%%BINS%%", bins) + action = action.replace("%%FILE%%", filename) + action = action.replace("%%IF%%", str(not pull_request).lower()) + + # Flags + flags = misc.gen_flags(crate_toml) + + apple_flags = flags["apple"] + final_apple_flags = "" + if apple_flags[0] is not None: + final_apple_flags += f"--features '{apple_flags[0]}' " + if apple_flags[1]: + final_apple_flags += "--no-default-features " + final_apple_flags += apple_flags[2] + + linux_flags = flags["linux"] + final_linux_flags = "" + if linux_flags[0] is not None: + final_linux_flags += f"--features '{linux_flags[0]}' " + if linux_flags[1]: + final_linux_flags += "--no-default-features " + final_linux_flags += linux_flags[2] + + windows_flags = flags["windows"] + final_windows_flags = "" + if windows_flags[0] is not None: + final_windows_flags += f"--features '{windows_flags[0]}' " + if windows_flags[1]: + final_windows_flags += "--no-default-features " + final_windows_flags += windows_flags[2] + + # Write Flags + action = action.replace("%%APPLE_FLAGS%%", final_apple_flags) + action = action.replace("%%LINUX_FLAGS%%", final_linux_flags) + action = action.replace("%%WINDOWS_FLAGS%%", final_windows_flags) + + # T2 + # Cross + targets = "" + for possible in t2_targets: + if possible not in unsupported: + if len(targets) != 0: + targets += "," + targets += possible + if len(targets) != 0: + action = action.replace("%%T2_CROSS_HAS_TARGETS%%", "true") + action = action.replace("%%T2_CROSS_TARGETS%%", targets) + else: + action = action.replace("%%T2_CROSS_HAS_TARGETS%%", "false") + action = action.replace("%%T2_CROSS_TARGETS%%", "err_no_targets") + # Windows + targets = "" + for possible in win_targets: + if possible not in unsupported: + if len(targets) != 0: + targets += "," + targets += possible + if len(targets) != 0: + action = action.replace("%%T2_WIN_HAS_TARGETS%%", "true") + action = action.replace("%%T2_WIN_TARGETS%%", targets) + else: + action = action.replace("%%T2_WIN_HAS_TARGETS%%", "false") + action = action.replace("%%T2_WIN_TARGETS%%", "err_no_targets") + + # T3 + # Cross + targets = "" + for possible in t3_targets: + if possible not in unsupported: + if len(targets) != 0: + targets += "," + targets += possible + if len(targets) != 0: + action = action.replace("%%T3_CROSS_HAS_TARGETS%%", "true") + action = action.replace("%%T3_CROSS_TARGETS%%", targets) + else: + action = action.replace("%%T3_CROSS_HAS_TARGETS%%", "false") + action = action.replace("%%T3_CROSS_TARGETS%%", "err_no_targets") + + with open("./.github/workflows/stable-" + crate + ".yml", "w") as file: + file.write(action) + + +if __name__ == "__main__": + argv = sys.argv + main(argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]) diff --git a/scripts/info.py b/scripts/info.py new file mode 100644 index 00000000..5925372b --- /dev/null +++ b/scripts/info.py @@ -0,0 +1,87 @@ +import glob +import json +import sys +import tomllib +import datetime +import misc + + +def main(filename: str, version: str, license_spdx: str, description: str, rustc_version_guess: str): + with open(filename, "rb") as file: + crate_toml = tomllib.load(file) + + description = description.replace("%%SINGLE_QUOTE%%", "'") + + features = misc.gen_flags(crate_toml) + + targets = [] + for t in glob.glob("target-*"): + targets.append(t[7:]) + + info = { # info.json + "info_version": "1", + "id": crate_toml["info"]["id"], + "version": version, + "license": license_spdx, + "git": crate_toml["info"]["git"], + "description": description, + "bins": crate_toml["info"]["bins"], + "info": { + "rustc_version_guess": rustc_version_guess[6:], + "index_publish_date": datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d"), + "features_apple": str(features["apple"][0]), + "features_linux": str(features["linux"][0]), + "features_windows": str(features["windows"][0]), + "no_default_features_apple": str(features["apple"][1]), + "no_default_features_linux": str(features["linux"][1]), + "no_default_features_windows": str(features["windows"][1]), + }, + "archive": { + "compression": "gz", + "ext": "tar.gz" + }, + "files": { + "hash": "hashes.json", + "license": "license.report", + "deps": "deps.report", + "audit": "audit.report", + "sig_info": "info.json.minisig", + "sig_hash": "hashes.json.minisig", + }, + "targets": targets, + } + + with open("./info.json", "w") as file: + file.write(json.dumps(info)) + + hashes = { # hashes.json + "hashes_version": "1", + "hashes": {} + } + + # Fill hashes + for t in targets: + with open(f"./target-{t}/{t}.hashes.json", "r") as file: + blob = { + "archive": {}, + "bins": {} + } + hash_file = json.loads(file.read()) + + for h in hash_file["archive"]: + blob["archive"][h["type"]] = h["hash"] + + for b in hash_file["bins"]: + if b["bin"] not in blob["bins"]: + blob["bins"][b["bin"]] = {} + blob["bins"][b["bin"]][b["type"]] = b["hash"] + + hashes["hashes"][t] = blob + + with open("./hashes.json", "w") as file: + file.write(json.dumps(hashes)) + + +if __name__ == "__main__": + argv = sys.argv + main(argv[1], argv[2], argv[3], argv[4], argv[5]) diff --git a/scripts/misc.py b/scripts/misc.py new file mode 100644 index 00000000..48971c75 --- /dev/null +++ b/scripts/misc.py @@ -0,0 +1,55 @@ +# Misc code to reduce duplicate blocks + +def gen_flags(crate_toml): + apple_flags = [None, False, ""] # FEATURES(0), NO_DEFAULT_FEATURES(1), EXTRA_FLAGS(2) + linux_flags = [None, False, ""] + windows_flags = [None, False, ""] + + if "target" in crate_toml: + targets = crate_toml["target"] + if "all" in targets: + if "features" in targets["all"]: + f = targets["all"]["features"] + apple_flags[0] = f + linux_flags[0] = f + windows_flags[0] = f + if "no-default-features" in targets["all"]: + f = targets["all"]["no-default-features"] + apple_flags[1] = f + linux_flags[1] = f + windows_flags[1] = f + if "flags" in targets["all"]: + f = targets["all"]["flags"] + apple_flags[2] = f + linux_flags[2] = f + windows_flags[2] = f + + if "apple" in targets: + if "features" in targets["apple"]: + apple_flags[0] = targets["apple"]["features"] + if "no-default-features" in targets["apple"]: + apple_flags[1] = targets["apple"]["no-default-features"] + if "flags" in targets["apple"]: + apple_flags[2] = targets["apple"]["flags"] + + if "linux" in targets: + if "features" in targets["linux"]: + linux_flags[0] = targets["linux"]["features"] + if "no-default-features" in targets["linux"]: + linux_flags[1] = targets["linux"]["no-default-features"] + if "flags" in targets["linux"]: + linux_flags[2] = targets["linux"]["flags"] + + if "windows" in targets: + if "features" in targets["windows"]: + windows_flags[0] = targets["windows"]["features"] + if "no-default-features" in targets["windows"]: + windows_flags[1] = targets["windows"]["no-default-features"] + if "flags" in targets["windows"]: + windows_flags[2] = targets["windows"]["flags"] + + return { + "apple": apple_flags, + "linux": linux_flags, + "windows": windows_flags, + }