From c9d5e6de404c2d5d8f6e59a12a4a0de0b82a85bb Mon Sep 17 00:00:00 2001 From: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> Date: Sat, 9 Nov 2024 16:19:35 +1000 Subject: [PATCH] feature: faster self-install without rate limit issue (#1963) * Add new hidden option `--self-install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix typo Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Optimize: Only call `LazyJobserverClient::new` when necessary Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * `--self-install` should include a path to a binary On windows, we cannot just copy an executable with process running from it, so better to copy it to a temporary location. Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Impl manifest update and basic API of `self_install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Add dep atomic-file-install to cargo-binstall Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Impl `self_install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Make `--self-install` a boolean flag Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Accept no duration in `MainExit::new` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Impl self-install mode in main_impl.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use `--self-install` mode in install-from-binstall-release.sh Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use `--self-install` in install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix import in mod entry Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix call of `self_install` in `main_impl` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix `--self-install` clap doc Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix `entry::self_install` Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix parsing semver number in entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix compilation in entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * fix entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix fmt in bin_util.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix fmt in entry.rs Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use --self-install if supported in unix install script Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Use --self-install if available in powershell install script Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fi install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix install-from-binstall-release.ps1 Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Create self-install.sh Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Add e2e-test-self-install to justfile Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix args::parse() for self-install mode Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> * Fix args parsing: Do no require positional arg if --self-install is present Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> --------- Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com> --- crates/bin/Cargo.toml | 1 + crates/bin/src/args.rs | 9 +++++- crates/bin/src/bin_util.rs | 13 ++++----- crates/bin/src/entry.rs | 46 +++++++++++++++++++++++++++++++ crates/bin/src/main_impl.rs | 9 +++--- e2e-tests/self-install.sh | 14 ++++++++++ install-from-binstall-release.ps1 | 7 ++++- install-from-binstall-release.sh | 2 +- justfile | 3 +- 9 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 e2e-tests/self-install.sh diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index d37b036fa..6cba3dae2 100644 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -22,6 +22,7 @@ pkg-fmt = "zip" pkg-fmt = "zip" [dependencies] +atomic-file-install = { version = "1.0.5", path = "../atomic-file-install" } binstalk = { path = "../binstalk", version = "0.28.11", default-features = false } binstalk-manifests = { path = "../binstalk-manifests", version = "0.15.8" } clap = { version = "4.5.3", features = ["derive", "env"] } diff --git a/crates/bin/src/args.rs b/crates/bin/src/args.rs index dc4480b4a..8cf81ac7a 100644 --- a/crates/bin/src/args.rs +++ b/crates/bin/src/args.rs @@ -52,7 +52,7 @@ pub struct Args { #[clap( help_heading = "Package selection", value_name = "crate[@version]", - required_unless_present_any = ["version", "help"], + required_unless_present_any = ["version", "self_install", "help"], )] pub(crate) crate_names: Vec, @@ -404,6 +404,9 @@ pub struct Args { /// This would override the `log_level`. #[clap(help_heading = "Meta", short, long, conflicts_with("verbose"))] pub(crate) quiet: bool, + + #[clap(long, hide(true))] + pub(crate) self_install: bool, } #[derive(Debug, Clone)] @@ -515,6 +518,10 @@ pub fn parse() -> (Args, PkgOverride) { // Load options let mut opts = Args::parse_from(args); + if opts.self_install { + return (opts, Default::default()); + } + if opts.log_level.is_none() { if let Some(log) = env::var("BINSTALL_LOG_LEVEL") .ok() diff --git a/crates/bin/src/bin_util.rs b/crates/bin/src/bin_util.rs index df49c3f1e..9761526f5 100644 --- a/crates/bin/src/bin_util.rs +++ b/crates/bin/src/bin_util.rs @@ -36,13 +36,12 @@ impl Termination for MainExit { } impl MainExit { - pub fn new(res: Result<()>, done: Duration) -> Self { - res.map(|()| MainExit::Success(Some(done))) - .unwrap_or_else(|err| { - err.downcast::() - .map(MainExit::Error) - .unwrap_or_else(MainExit::Report) - }) + pub fn new(res: Result<()>, done: Option) -> Self { + res.map(|()| MainExit::Success(done)).unwrap_or_else(|err| { + err.downcast::() + .map(MainExit::Error) + .unwrap_or_else(MainExit::Report) + }) } } diff --git a/crates/bin/src/entry.rs b/crates/bin/src/entry.rs index 5fbf6e840..43feaa3a9 100644 --- a/crates/bin/src/entry.rs +++ b/crates/bin/src/entry.rs @@ -5,6 +5,7 @@ use std::{ time::Duration, }; +use atomic_file_install::atomic_install; use binstalk::{ errors::{BinstallError, CrateContextError}, fetchers::{Fetcher, GhCrateMeta, QuickInstall, SignaturePolicy}, @@ -20,16 +21,20 @@ use binstalk::{ resolve::{CrateName, Resolution, ResolutionFetch, VersionReqExt}, CargoTomlFetchOverride, Options, Resolver, }, + TARGET, }; use binstalk_manifests::{ cargo_config::Config, cargo_toml_binstall::{PkgOverride, Strategy}, + crate_info::{CrateInfo, CrateSource}, crates_manifests::Manifests, }; +use compact_str::CompactString; use file_format::FileFormat; use home::cargo_home; use log::LevelFilter; use miette::{miette, Report, Result, WrapErr}; +use semver::Version; use tokio::task::block_in_place; use tracing::{debug, error, info, warn}; @@ -582,3 +587,44 @@ fn do_install_fetches_continue_on_failure( Ok(()) }) } + +pub fn self_install(args: Args) -> Result<()> { + // Load .cargo/config.toml + let cargo_home = cargo_home().map_err(BinstallError::from)?; + let mut config = Config::load_from_path(cargo_home.join("config.toml"))?; + + // Compute paths + let cargo_root = args.root; + let (install_path, manifests, _) = compute_paths_and_load_manifests( + cargo_root.clone(), + args.install_path, + args.no_track, + cargo_home, + &mut config, + )?; + + let mut dest = install_path.join("cargo-binstall"); + if cfg!(windows) { + assert!(dest.set_extension("exe")); + } + + atomic_install(&env::current_exe().map_err(BinstallError::from)?, &dest) + .map_err(BinstallError::from)?; + + if let Some(manifests) = manifests { + manifests.update(vec![CrateInfo { + name: CompactString::const_new("cargo-binstall"), + version_req: CompactString::const_new("*"), + current_version: Version::new( + env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(), + env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(), + env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(), + ), + source: CrateSource::cratesio_registry(), + target: CompactString::const_new(TARGET), + bins: vec![CompactString::const_new("cargo-binstall")], + }])?; + } + + Ok(()) +} diff --git a/crates/bin/src/main_impl.rs b/crates/bin/src/main_impl.rs index 2c78960b4..51d29ccc3 100644 --- a/crates/bin/src/main_impl.rs +++ b/crates/bin/src/main_impl.rs @@ -12,9 +12,6 @@ use crate::{ }; pub fn do_main() -> impl Termination { - // This must be the very first thing to happen - let jobserver_client = LazyJobserverClient::new(); - let (args, cli_overrides) = args::parse(); if args.version { @@ -46,6 +43,8 @@ rustc-llvm-version: {rustc_llvm_version}"# println!("{cargo_binstall_version}"); } MainExit::Success(None) + } else if args.self_install { + MainExit::new(entry::self_install(args), None) } else { logging( args.log_level.unwrap_or(LevelFilter::Info), @@ -54,12 +53,14 @@ rustc-llvm-version: {rustc_llvm_version}"# let start = Instant::now(); + let jobserver_client = LazyJobserverClient::new(); + let result = run_tokio_main(|| entry::install_crates(args, cli_overrides, jobserver_client)); let done = start.elapsed(); debug!("run time: {done:?}"); - MainExit::new(result, done) + MainExit::new(result, Some(done)) } } diff --git a/e2e-tests/self-install.sh b/e2e-tests/self-install.sh new file mode 100644 index 000000000..e00f35382 --- /dev/null +++ b/e2e-tests/self-install.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euxo pipefail + +unset CARGO_INSTALL_ROOT + +CARGO_HOME=$(mktemp -d 2>/dev/null || mktemp -d -t 'cargo-home') +export CARGO_HOME +export PATH="$CARGO_HOME/bin:$PATH" + +"./$1" --self-install + +cargo binstall --help +cargo install --list diff --git a/install-from-binstall-release.ps1 b/install-from-binstall-release.ps1 index 0e0a39c24..4a31f878b 100644 --- a/install-from-binstall-release.ps1 +++ b/install-from-binstall-release.ps1 @@ -19,7 +19,12 @@ $url = "$base_url$arch-pc-windows-msvc.zip" Invoke-WebRequest $url -OutFile $tmpdir\cargo-binstall.zip Expand-Archive -Force $tmpdir\cargo-binstall.zip $tmpdir\cargo-binstall Write-Host "" -Invoke-Expression "$tmpdir\cargo-binstall\cargo-binstall.exe -y --force cargo-binstall" + +$ps = Start-Process -PassThru -Wait "$tmpdir\cargo-binstall\cargo-binstall.exe" "--self-install" +if ($ps.ExitCode -ne 0) { + Invoke-Expression "$tmpdir\cargo-binstall\cargo-binstall.exe -y --force cargo-binstall" +} + Remove-Item -Force $tmpdir\cargo-binstall.zip Remove-Item -Recurse -Force $tmpdir\cargo-binstall $cargo_home = if ($Env:CARGO_HOME -ne $null) { $Env:CARGO_HOME } else { "$HOME\.cargo" } diff --git a/install-from-binstall-release.sh b/install-from-binstall-release.sh index fa4f82223..97cf0cca5 100755 --- a/install-from-binstall-release.sh +++ b/install-from-binstall-release.sh @@ -36,7 +36,7 @@ else exit 1 fi -./cargo-binstall -y --force cargo-binstall +./cargo-binstall --self-install || ./cargo-binstall -y --force cargo-binstall CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" diff --git a/justfile b/justfile index bb7ab5564..291ad42a8 100644 --- a/justfile +++ b/justfile @@ -244,6 +244,7 @@ e2e-test-registries: (e2e-test "registries") e2e-test-signing: (e2e-test "signing") e2e-test-continue-on-failure: (e2e-test "continue-on-failure") e2e-test-private-github-repo: (e2e-test "private-github-repo") +e2e-test-self-install: (e2e-test "self-install") # WinTLS (Windows in CI) does not have TLS 1.3 support [windows] @@ -252,7 +253,7 @@ e2e-test-tls: (e2e-test "tls" "1.2") [macos] e2e-test-tls: (e2e-test "tls" "1.2") (e2e-test "tls" "1.3") -e2e-tests: e2e-test-live e2e-test-manifest-path e2e-test-git e2e-test-other-repos e2e-test-strategies e2e-test-version-syntax e2e-test-upgrade e2e-test-tls e2e-test-self-upgrade-no-symlink e2e-test-uninstall e2e-test-subcrate e2e-test-no-track e2e-test-registries e2e-test-signing e2e-test-continue-on-failure e2e-test-private-github-repo +e2e-tests: e2e-test-live e2e-test-manifest-path e2e-test-git e2e-test-other-repos e2e-test-strategies e2e-test-version-syntax e2e-test-upgrade e2e-test-tls e2e-test-self-upgrade-no-symlink e2e-test-uninstall e2e-test-subcrate e2e-test-no-track e2e-test-registries e2e-test-signing e2e-test-continue-on-failure e2e-test-private-github-repo e2e-test-self-install unit-tests: print-env cargo test --no-run --target {{target}}