From 6a256fa3c673a2b655b9f28d7980cb8636525883 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Sat, 20 Apr 2024 12:41:31 -0400 Subject: [PATCH] Add support for UV --- Cargo.toml | 3 + build.rs | 57 ++++++++-- docs/config.md | 14 +++ docs/runtime.md | 18 +++- src/app.rs | 34 ++++++ src/compression.rs | 25 ++--- src/distribution.rs | 252 ++++++++++++++++++++++++++++++-------------- 7 files changed, 301 insertions(+), 102 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0294881..289139b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,4 +74,7 @@ passthrough = [ "PYAPP_SELF_COMMAND", "PYAPP_SKIP_INSTALL", "PYAPP_UPGRADE_VIRTUALENV", + "PYAPP_UV_ENABLED", + "PYAPP_UV_ONLY_BOOTSTRAP", + "PYAPP_UV_VERSION", ] diff --git a/build.rs b/build.rs index 44a895b..5960da2 100644 --- a/build.rs +++ b/build.rs @@ -309,13 +309,12 @@ fn get_distribution_source() -> String { let selected_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); let selected_variant = { let mut variant = env::var("PYAPP_DISTRIBUTION_VARIANT").unwrap_or_default(); - if variant.is_empty() { - if selected_platform == "linux" - && selected_arch == "x86_64" - && selected_python_version != "3.7" - { - variant = "v3".to_string(); - } + if variant.is_empty() + && selected_platform == "linux" + && selected_arch == "x86_64" + && selected_python_version != "3.7" + { + variant = "v3".to_string(); }; variant }; @@ -852,6 +851,47 @@ fn set_pip_allow_config() { } } +fn set_uv_enabled() { + let variable = "PYAPP_UV_ENABLED"; + if is_enabled(variable) { + set_runtime_variable(variable, "1"); + } else { + set_runtime_variable(variable, "0"); + } +} + +fn set_uv_only_bootstrap() { + let variable = "PYAPP_UV_ONLY_BOOTSTRAP"; + if is_enabled(variable) { + set_runtime_variable(variable, "1"); + } else { + set_runtime_variable(variable, "0"); + } +} + +fn set_uv_version() { + let variable = "PYAPP_UV_VERSION"; + let version = env::var(variable).unwrap_or("any".to_string()); + set_runtime_variable(variable, version); + + let artifact_name = if !is_enabled("PYAPP_UV_ENABLED") { + "".to_string() + } else if env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows" { + // Force MinGW-w64 to use msvc + if env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default() == "gnu" { + format!( + "uv-{}-pc-windows-msvc.zip", + env::var("CARGO_CFG_TARGET_ARCH").unwrap() + ) + } else { + format!("uv-{}.zip", env::var("TARGET").unwrap()) + } + } else { + format!("uv-{}.tar.gz", env::var("TARGET").unwrap()) + }; + set_runtime_variable("PYAPP__UV_ARTIFACT_NAME", artifact_name); +} + fn set_skip_install() { let variable = "PYAPP_SKIP_INSTALL"; if is_enabled(variable) { @@ -942,6 +982,9 @@ fn main() { set_pip_project_features(); set_pip_extra_args(); set_pip_allow_config(); + set_uv_enabled(); + set_uv_only_bootstrap(); + set_uv_version(); set_skip_install(); set_indicator(); set_self_command(); diff --git a/docs/config.md b/docs/config.md index d24bcc0..103e874 100644 --- a/docs/config.md +++ b/docs/config.md @@ -134,6 +134,20 @@ You may set the `PYAPP_DISTRIBUTION_EMBED` option to `true` or `1` to embed the You can set the `PYAPP_DISTRIBUTION_PATH` option to use a local path rather than fetching the source, which implicitly enables embedding. The local archive should be similar to the [default distributions](#python-distribution) in that there should be a Python interpreter ready for use. +## UV + +You may set the `PYAPP_UV_ENABLED` option to `true` or `1` to use [UV](https://github.com/astral-sh/uv) for virtual environment creation and project installation. + +### Version ### {: #uv-version } + +You may use a specific `X.Y.Z` version by setting the `PYAPP_UV_VERSION` option. + +By default, a version of UV that has already been downloaded by a PyApp application is used. If UV has not yet been downloaded then the latest version is used. + +### Only bootstrap + +You may set the `PYAPP_UV_ONLY_BOOTSTRAP` option to `true` or `1` to only use UV for virtual environment creation and continue using pip for project installation. + ## pip These options have no effect when the project installation is [disabled](#skipping-project-installation). diff --git a/docs/runtime.md b/docs/runtime.md index 02f1d2f..fd49359 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -21,8 +21,17 @@ flowchart TD DISTEMBEDDED -- Yes --> DISTEXTRACT[[Cache from embedded data]] DISTSOURCE --> FULLISOLATION DISTEXTRACT --> FULLISOLATION - FULLISOLATION -- No --> VENV[[Create virtual environment]] + FULLISOLATION -- No --> UVENABLED([UV enabled]) + UVENABLED -- No --> VENV[[Create virtual environment]] + UVENABLED -- Yes --> UVCACHED([UV cached]) + UVCACHED -- No --> DOWNLOADUV[[Download UV]] + UVCACHED -- Yes --> VENV + DOWNLOADUV --> VENV FULLISOLATION -- Yes --> UNPACK[[Unpack distribution directly]] + UNPACK --> UVENABLEDUNPACK([UV enabled]) + UVENABLEDUNPACK -- No --> EXTERNALPIP[[External pip]] + UVENABLEDUNPACK -- Yes --> UVCACHEDUNPACK([UV cached]) + UVCACHEDUNPACK -- No --> DOWNLOADUVUNPACK[[Download UV]] EXTERNALPIP([External pip]) -- No --> PROJEMBEDDED([Project embedded]) EXTERNALPIP -- Yes --> PIPCACHED([pip cached]) PIPCACHED -- No --> DOWNLOADPIP[[Download pip]] @@ -32,17 +41,20 @@ flowchart TD PROJEMBEDDED -- Yes --> PROJEMBED[[Install from embedded data]] DEPFILE -- No --> SINGLEPROJECT[[Install single project]] DEPFILE -- Yes --> DEPFILEINSTALL[[Install from dependency file]] + UVCACHEDUNPACK -- Yes --> PROJEMBEDDED + DOWNLOADUVUNPACK --> PROJEMBEDDED + VENV --> EXTERNALPIP SINGLEPROJECT --> MNG DEPFILEINSTALL --> MNG PROJEMBED --> MNG - VENV --> EXTERNALPIP - UNPACK --> EXTERNALPIP MNG -- No --> EXECUTE[[Execute project]] MNG -- Yes --> MNGCMD([Command invoked]) MNGCMD -- No --> EXECUTE MNGCMD -- Yes --> MANAGE[[Run management command]] click DISTEMBEDDED href "../config/#distribution-embedding" click FULLISOLATION href "../config/#full-isolation" + click UVENABLED href "../config/#uv" + click UVENABLEDUNPACK href "../config/#uv" click EXTERNALPIP href "../config/#externally-managed" click PROJEMBEDDED href "../config/#project-embedding" click DEPFILE href "../config/#dependency-file" diff --git a/src/app.rs b/src/app.rs index d53e3be..2fb4707 100644 --- a/src/app.rs +++ b/src/app.rs @@ -186,6 +186,26 @@ pub fn pip_external() -> bool { env!("PYAPP_PIP_EXTERNAL") == "1" } +pub fn uv_enabled() -> bool { + env!("PYAPP_UV_ENABLED") == "1" +} + +pub fn uv_only_bootstrap() -> bool { + env!("PYAPP_UV_ONLY_BOOTSTRAP") == "1" +} + +pub fn uv_version() -> String { + env!("PYAPP_UV_VERSION").into() +} + +pub fn uv_artifact_name() -> String { + env!("PYAPP__UV_ARTIFACT_NAME").into() +} + +pub fn uv_as_installer() -> bool { + uv_enabled() && !uv_only_bootstrap() +} + pub fn is_gui() -> bool { env!("PYAPP_IS_GUI") == "1" } @@ -235,6 +255,10 @@ pub fn external_pip_cache() -> PathBuf { cache_dir().join("pip") } +pub fn managed_uv_cache() -> PathBuf { + cache_dir().join("uv").join(uv_version()) +} + pub fn external_pip_zipapp() -> PathBuf { let pip_version = pip_version(); let filename = if pip_version == "latest" { @@ -244,3 +268,13 @@ pub fn external_pip_zipapp() -> PathBuf { }; external_pip_cache().join(filename) } + +pub fn managed_uv() -> PathBuf { + let uv_artifact_name = uv_artifact_name(); + let filename = if uv_artifact_name.ends_with(".zip") { + "uv.exe".to_string() + } else { + "uv".to_string() + }; + managed_uv_cache().join(filename) +} diff --git a/src/compression.rs b/src/compression.rs index 066e396..a12f51a 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -6,22 +6,23 @@ use anyhow::{bail, Result}; use crate::terminal; pub fn unpack(format: String, archive: &PathBuf, destination: &PathBuf) -> Result<()> { + let wait_message = format!("Unpacking distribution ({})", format); match format.as_ref() { - "tar|bzip2" => unpack_tar_bzip2(archive, destination)?, - "tar|gzip" => unpack_tar_gzip(archive, destination)?, - "tar|zstd" => unpack_tar_zstd(archive, destination)?, - "zip" => unpack_zip(archive, destination)?, + "tar|bzip2" => unpack_tar_bzip2(archive, destination, wait_message)?, + "tar|gzip" => unpack_tar_gzip(archive, destination, wait_message)?, + "tar|zstd" => unpack_tar_zstd(archive, destination, wait_message)?, + "zip" => unpack_zip(archive, destination, wait_message)?, _ => bail!("unsupported distribution format: {}", format), } Ok(()) } -fn unpack_tar_bzip2(path: &PathBuf, destination: &PathBuf) -> Result<()> { +fn unpack_tar_bzip2(path: &PathBuf, destination: &PathBuf, wait_message: String) -> Result<()> { let bz = bzip2::read::BzDecoder::new(File::open(path)?); let mut archive = tar::Archive::new(bz); - let spinner = terminal::spinner("Unpacking distribution (tar|bzip2)".to_string()); + let spinner = terminal::spinner(wait_message); let result = archive.unpack(destination); spinner.finish_and_clear(); result?; @@ -29,11 +30,11 @@ fn unpack_tar_bzip2(path: &PathBuf, destination: &PathBuf) -> Result<()> { Ok(()) } -fn unpack_tar_gzip(path: &PathBuf, destination: &PathBuf) -> Result<()> { +pub fn unpack_tar_gzip(path: &PathBuf, destination: &PathBuf, wait_message: String) -> Result<()> { let gz = flate2::read::GzDecoder::new(File::open(path)?); let mut archive = tar::Archive::new(gz); - let spinner = terminal::spinner("Unpacking distribution (tar|gzip)".to_string()); + let spinner = terminal::spinner(wait_message); let result = archive.unpack(destination); spinner.finish_and_clear(); result?; @@ -41,11 +42,11 @@ fn unpack_tar_gzip(path: &PathBuf, destination: &PathBuf) -> Result<()> { Ok(()) } -fn unpack_tar_zstd(path: &PathBuf, destination: &PathBuf) -> Result<()> { +fn unpack_tar_zstd(path: &PathBuf, destination: &PathBuf, wait_message: String) -> Result<()> { let zst = zstd::stream::read::Decoder::new(File::open(path)?)?; let mut archive = tar::Archive::new(zst); - let spinner = terminal::spinner("Unpacking distribution (tar|zstd)".to_string()); + let spinner = terminal::spinner(wait_message); let result = archive.unpack(destination); spinner.finish_and_clear(); result?; @@ -53,10 +54,10 @@ fn unpack_tar_zstd(path: &PathBuf, destination: &PathBuf) -> Result<()> { Ok(()) } -fn unpack_zip(path: &PathBuf, destination: &PathBuf) -> Result<()> { +pub fn unpack_zip(path: &PathBuf, destination: &PathBuf, wait_message: String) -> Result<()> { let mut archive = zip::ZipArchive::new(File::open(path)?)?; - let spinner = terminal::spinner("Unpacking distribution (zip)".to_string()); + let spinner = terminal::spinner(wait_message); let result = archive.extract(destination); spinner.finish_and_clear(); result?; diff --git a/src/distribution.rs b/src/distribution.rs index 41437a0..940422d 100644 --- a/src/distribution.rs +++ b/src/distribution.rs @@ -9,13 +9,39 @@ use tempfile::tempdir; use crate::{app, compression, fs_utils, network, process}; +fn apply_env_vars(command: &mut Command) { + if !app::full_isolation() { + command.env("VIRTUAL_ENV", app::python_path().parent().unwrap()); + } + + if !app::pass_location() { + command.env("PYAPP", "1"); + } else if let Ok(exe_path) = env::current_exe() { + command.env("PYAPP", exe_path); + } else { + command.env("PYAPP", ""); + } + + if !app::exposed_command().is_empty() { + command.env("PYAPP_COMMAND_NAME", app::exposed_command()); + } +} + pub fn python_command(python: &PathBuf) -> Command { let mut command = Command::new(python); + apply_env_vars(&mut command); command.arg(app::python_isolation_flag()); command } +fn uv_command() -> Command { + let mut command = Command::new(app::managed_uv()); + apply_env_vars(&mut command); + + command +} + pub fn run_project() -> Result<()> { let mut command = python_command(&app::python_path()); @@ -26,6 +52,7 @@ pub fn run_project() -> Result<()> { } } + apply_env_vars(&mut command); if !app::exec_code().is_empty() { command.args(["-c", app::exec_code().as_str()]); } else if !app::exec_module().is_empty() { @@ -69,18 +96,6 @@ pub fn run_project() -> Result<()> { } command.args(env::args().skip(1)); - if !app::pass_location() { - command.env("PYAPP", "1"); - } else if let Ok(exe_path) = env::current_exe() { - command.env("PYAPP", exe_path); - } else { - command.env("PYAPP", ""); - } - - if !app::exposed_command().is_empty() { - command.env("PYAPP_COMMAND_NAME", app::exposed_command()); - } - process::exec(command) .with_context(|| "project execution failed, consider restoring from scratch") } @@ -98,27 +113,45 @@ pub fn ensure_ready() -> Result<()> { } pub fn pip_base_command() -> Command { - let mut command = python_command(&app::python_path()); - if app::pip_external() { - let external_pip = app::external_pip_zipapp(); - command.arg(external_pip.to_string_lossy().as_ref()); + if app::uv_as_installer() { + let mut command = uv_command(); + command.arg("pip"); + command } else { - command.args(["-m", "pip"]); + let mut command = python_command(&app::python_path()); + if app::pip_external() { + let external_pip = app::external_pip_zipapp(); + command.arg(external_pip.to_string_lossy().as_ref()); + } else { + command.args(["-m", "pip"]); + } + command } - - command } pub fn pip_install_command() -> Command { let mut command = pip_base_command(); - command.args([ - "install", - "--disable-pip-version-check", - "--no-warn-script-location", - ]); - if !app::pip_allow_config() { - command.arg("--isolated"); + if app::uv_as_installer() { + command.arg("install"); + if app::full_isolation() { + command.args([ + "--python", + app::install_dir() + .join(app::distribution_python_path()) + .to_string_lossy() + .as_ref(), + ]); + } + } else { + command.args([ + "install", + "--disable-pip-version-check", + "--no-warn-script-location", + ]); + if !app::pip_allow_config() { + command.arg("--isolated"); + } } command.args( app::pip_extra_args() @@ -203,38 +236,50 @@ pub fn materialize() -> Result<()> { })?; } - let mut command = - python_command(&unpacked_distribution.join(app::distribution_python_path())); - - if app::upgrade_virtualenv() { - ensure_base_pip(&unpacked_distribution)?; - - let mut upgrade_command = - python_command(&unpacked_distribution.join(app::distribution_python_path())); - upgrade_command.args([ - "-m", - "pip", - "install", - "--upgrade", - "--isolated", - "--disable-pip-version-check", - "--no-warn-script-location", - "virtualenv", - ]); - let (status, output) = - run_setup_command(upgrade_command, "Upgrading virtualenv".to_string())?; - check_setup_status(status, output)?; - - command.args(["-m", "virtualenv"]); - if app::pip_external() { - command.arg("--no-pip"); + let python_path = unpacked_distribution.join(app::distribution_python_path()); + let mut command = if app::uv_enabled() { + ensure_uv_available()?; + let mut command = uv_command(); + command.args(["venv", "--python", &python_path.to_string_lossy().as_ref()]); + if app::uv_only_bootstrap() { + command.arg("--seed"); } + + command } else { - command.args(["-m", "venv"]); - if app::pip_external() { - command.arg("--without-pip"); + let mut command = python_command(&python_path); + if app::upgrade_virtualenv() { + ensure_base_pip(&unpacked_distribution)?; + + let mut upgrade_command = + python_command(&unpacked_distribution.join(app::distribution_python_path())); + upgrade_command.args([ + "-m", + "pip", + "install", + "--upgrade", + "--isolated", + "--disable-pip-version-check", + "--no-warn-script-location", + "virtualenv", + ]); + let (status, output) = + run_setup_command(upgrade_command, "Upgrading virtualenv".to_string())?; + check_setup_status(status, output)?; + + command.args(["-m", "virtualenv"]); + if app::pip_external() { + command.arg("--no-pip"); + } + } else { + command.args(["-m", "venv"]); + if app::pip_external() { + command.arg("--without-pip"); + } } - } + + command + }; command.arg(app::install_dir().to_string_lossy().as_ref()); let (status, output) = @@ -301,7 +346,7 @@ fn install_project() -> Result<()> { } pub fn pip_install(command: Command, wait_message: String) -> Result<(ExitStatus, String)> { - ensure_external_pip()?; + ensure_installer_available()?; run_setup_command(command, wait_message) } @@ -325,12 +370,12 @@ pub fn pip_install_dependency_file( command.args(["-r", temp_path.to_string_lossy().as_ref()]); - ensure_external_pip()?; + ensure_installer_available()?; run_setup_command(command, wait_message) } fn ensure_base_pip(distribution_directory: &Path) -> Result<()> { - if app::distribution_pip_available() { + if app::distribution_pip_available() || app::uv_enabled() { return Ok(()); } @@ -341,54 +386,101 @@ fn ensure_base_pip(distribution_directory: &Path) -> Result<()> { Ok(()) } -fn ensure_external_pip() -> Result<()> { - if !app::pip_external() { - return Ok(()); +fn ensure_installer_available() -> Result<()> { + if app::uv_as_installer() { + ensure_uv_available()?; + } else if app::pip_external() { + let external_pip = app::external_pip_zipapp(); + if external_pip.is_file() { + return Ok(()); + } + + let external_pip_cache = app::external_pip_cache(); + fs::create_dir_all(&external_pip_cache).with_context(|| { + format!( + "unable to create distribution cache {}", + &external_pip_cache.display() + ) + })?; + + let dir = tempdir().with_context(|| "unable to create temporary directory")?; + let temp_path = dir.path().join("pip.pyz"); + + let mut f = fs::File::create(&temp_path).with_context(|| { + format!("unable to create temporary file: {}", &temp_path.display()) + })?; + + let pip_version = app::pip_version(); + let url = if pip_version == "latest" { + "https://bootstrap.pypa.io/pip/pip.pyz".to_string() + } else { + format!( + "https://bootstrap.pypa.io/pip/pip.pyz#/pip-{}.pyz", + app::pip_version() + ) + }; + + network::download( + &url, + &mut f, + external_pip.file_name().unwrap().to_str().unwrap(), + )?; + + fs_utils::move_temp_file(&temp_path, &external_pip)?; } - let external_pip = app::external_pip_zipapp(); - if external_pip.is_file() { + Ok(()) +} + +fn ensure_uv_available() -> Result<()> { + let managed_uv = app::managed_uv(); + if managed_uv.is_file() { return Ok(()); } - let external_pip_cache = app::external_pip_cache(); - fs::create_dir_all(&external_pip_cache).with_context(|| { - format!( - "unable to create distribution cache {}", - &external_pip_cache.display() - ) - })?; + let managed_uv_cache = app::managed_uv_cache(); + fs::create_dir_all(&managed_uv_cache) + .with_context(|| format!("unable to create UV cache {}", &managed_uv_cache.display()))?; let dir = tempdir().with_context(|| "unable to create temporary directory")?; - let temp_path = dir.path().join("pip.pyz"); + let artifact_name = app::uv_artifact_name(); + let temp_path = dir.path().join(&artifact_name); let mut f = fs::File::create(&temp_path) .with_context(|| format!("unable to create temporary file: {}", &temp_path.display()))?; - let pip_version = app::pip_version(); - let url = if pip_version == "latest" { - "https://bootstrap.pypa.io/pip/pip.pyz".to_string() + let uv_version = app::uv_version(); + let url = if uv_version == "any" { + format!( + "https://github.com/astral-sh/uv/releases/latest/download/{}", + &artifact_name, + ) } else { format!( - "https://bootstrap.pypa.io/pip/pip.pyz#/pip-{}.pyz", - app::pip_version() + "https://github.com/astral-sh/uv/releases/download/{}/{}", + &uv_version, &artifact_name, ) }; network::download( &url, &mut f, - external_pip.file_name().unwrap().to_str().unwrap(), + managed_uv.file_name().unwrap().to_str().unwrap(), )?; - fs_utils::move_temp_file(&temp_path, &external_pip) + if artifact_name.ends_with(".zip") { + compression::unpack_zip(&temp_path, &managed_uv_cache, "Unpacking UV".to_string()) + } else { + compression::unpack_tar_gzip(&temp_path, &managed_uv_cache, "Unpacking UV".to_string()) + } } fn run_setup_command(command: Command, message: String) -> Result<(ExitStatus, String)> { + let program = command.get_program().to_string_lossy().to_string(); let (status, output) = process::wait_for(command, message).with_context(|| { format!( - "could not run Python, verify distribution build metadata options: {}", - app::python_path().display() + "could not run program, verify distribution build metadata options: {}", + &program ) })?;