diff --git a/src/env_vars.rs b/src/env_vars.rs index c1e1f0c4..ce8927e3 100644 --- a/src/env_vars.rs +++ b/src/env_vars.rs @@ -7,9 +7,14 @@ use rattler_conda_types::Platform; use crate::linux; use crate::macos; use crate::metadata::Output; -use crate::unix; use crate::windows; +macro_rules! insert { + ($map:expr, $key:expr, $value:expr) => { + $map.insert($key.to_string(), Some($value.to_string())); + }; +} + fn get_stdlib_dir(prefix: &Path, platform: &Platform, py_ver: &str) -> PathBuf { if platform.is_windows() { prefix.join("Lib") @@ -26,22 +31,22 @@ fn get_sitepackages_dir(prefix: &Path, platform: &Platform, py_ver: &str) -> Pat /// Returns a map of environment variables for Python that are used in the build process. /// /// Variables: -/// - PYTHON: path to Python executable -/// - PY3K: 1 if Python 3, 0 if Python 2 -/// - PY_VER: Python version (major.minor), e.g. 3.8 -/// - STDLIB_DIR: Python standard library directory -/// - SP_DIR: Python site-packages directory -/// - NPY_VER: Numpy version (major.minor), e.g. 1.19 -/// - NPY_DISTUTILS_APPEND_FLAGS: 1 (https://github.com/conda/conda-build/pull/3015) -pub fn python_vars(output: &Output) -> HashMap { - let mut result = HashMap::::new(); +/// - `PYTHON`: path to Python executable +/// - `PY3K`: 1 if Python 3, 0 if Python 2 +/// - `PY_VER`: Python version (major.minor), e.g. 3.8 +/// - `STDLIB_DIR`: Python standard library directory +/// - `SP_DIR`: Python site-packages directory +/// - `NPY_VER`: NumPy version (major.minor), e.g. 1.19 +/// - `NPY_DISTUTILS_APPEND_FLAGS`: 1 (https://github.com/conda/conda-build/pull/3015) +pub fn python_vars(output: &Output) -> HashMap> { + let mut result = HashMap::new(); if output.host_platform().is_windows() { let python = output.prefix().join("python.exe"); - result.insert("PYTHON".to_string(), python.to_string_lossy().to_string()); + insert!(result, "PYTHON", python.to_string_lossy()); } else { let python = output.prefix().join("bin/python"); - result.insert("PYTHON".to_string(), python.to_string_lossy().to_string()); + insert!(result, "PYTHON", python.to_string_lossy()); } // find python in the host dependencies @@ -60,32 +65,19 @@ pub fn python_vars(output: &Output) -> HashMap { let stdlib_dir = get_stdlib_dir(output.prefix(), output.host_platform(), &py_ver_str); let site_packages_dir = get_sitepackages_dir(output.prefix(), output.host_platform(), &py_ver_str); - result.insert( - "PY3K".to_string(), - if py_ver[0] == "3" { - "1".to_string() - } else { - "0".to_string() - }, - ); - result.insert("PY_VER".to_string(), py_ver_str); - result.insert( - "STDLIB_DIR".to_string(), - stdlib_dir.to_string_lossy().to_string(), - ); - result.insert( - "SP_DIR".to_string(), - site_packages_dir.to_string_lossy().to_string(), - ); + let py3k = if py_ver[0] == "3" { "1" } else { "0" }; + insert!(result, "PY3K", py3k); + insert!(result, "PY_VER", py_ver_str); + insert!(result, "STDLIB_DIR", stdlib_dir.to_string_lossy()); + insert!(result, "SP_DIR", site_packages_dir.to_string_lossy()); } if let Some(npy_version) = output.variant().get("numpy") { let np_ver = npy_version.split('.').collect::>(); let np_ver = format!("{}.{}", np_ver[0], np_ver[1]); - - result.insert("NPY_VER".to_string(), np_ver); + insert!(result, "NPY_VER", np_ver); + insert!(result, "NPY_DISTUTILS_APPEND_FLAGS", "1"); } - result.insert("NPY_DISTUTILS_APPEND_FLAGS".to_string(), "1".to_string()); result } @@ -97,11 +89,11 @@ pub fn python_vars(output: &Output) -> HashMap { /// - R: Path to R executable /// - R_USER: Path to R user directory /// -pub fn r_vars(output: &Output) -> HashMap { - let mut result = HashMap::::new(); +pub fn r_vars(output: &Output) -> HashMap> { + let mut result = HashMap::new(); if let Some(r_ver) = output.variant().get("r-base") { - result.insert("R_VER".to_string(), r_ver.clone()); + insert!(result, "R_VER", r_ver); let r_bin = if output.host_platform().is_windows() { output.prefix().join("Scripts/R.exe") @@ -111,15 +103,15 @@ pub fn r_vars(output: &Output) -> HashMap { let r_user = output.prefix().join("Libs/R"); - result.insert("R".to_string(), r_bin.to_string_lossy().to_string()); - result.insert("R_USER".to_string(), r_user.to_string_lossy().to_string()); + insert!(result, "R", r_bin.to_string_lossy()); + insert!(result, "R_USER", r_user.to_string_lossy()); } result } -pub fn language_vars(output: &Output) -> HashMap { - let mut result = HashMap::::new(); +pub fn language_vars(output: &Output) -> HashMap> { + let mut result = HashMap::new(); result.extend(python_vars(output)); result.extend(r_vars(output)); @@ -139,8 +131,8 @@ pub fn language_vars(output: &Output) -> HashMap { /// - LANG: Language (e.g. en_US.UTF-8) /// - LC_ALL: Language (e.g. en_US.UTF-8) /// - MAKEFLAGS: Make flags (e.g. -j4) -pub fn os_vars(prefix: &Path, platform: &Platform) -> HashMap { - let mut vars = HashMap::::new(); +pub fn os_vars(prefix: &Path, platform: &Platform) -> HashMap> { + let mut vars = HashMap::new(); let path_var = if platform.is_windows() { "Path" @@ -148,39 +140,31 @@ pub fn os_vars(prefix: &Path, platform: &Platform) -> HashMap { "PATH" }; - vars.insert( - "CPU_COUNT".to_string(), - env::var("CPU_COUNT").unwrap_or_else(|_| num_cpus::get().to_string()), - ); - vars.insert("LANG".to_string(), env::var("LANG").unwrap_or_default()); - vars.insert("LC_ALL".to_string(), env::var("LC_ALL").unwrap_or_default()); - vars.insert( - "MAKEFLAGS".to_string(), - env::var("MAKEFLAGS").unwrap_or_default(), + insert!( + vars, + "CPU_COUNT", + env::var("CPU_COUNT").unwrap_or_else(|_| num_cpus::get().to_string()) ); + vars.insert("LANG".to_string(), env::var("LANG").ok()); + vars.insert("LC_ALL".to_string(), env::var("LC_ALL").ok()); + vars.insert("MAKEFLAGS".to_string(), env::var("MAKEFLAGS").ok()); let shlib_ext = if platform.is_windows() { - ".dll".to_string() + ".dll" } else if platform.is_osx() { - ".dylib".to_string() + ".dylib" } else if platform.is_linux() { - ".so".to_string() + ".so" } else { - ".not_implemented".to_string() + ".not_implemented" }; - vars.insert("SHLIB_EXT".to_string(), shlib_ext); - if let Ok(path) = env::var(path_var) { - vars.insert(path_var.to_string(), path); - } + insert!(vars, "SHLIB_EXT", shlib_ext); + vars.insert(path_var.to_string(), env::var(path_var).ok()); - if cfg!(target_family = "windows") { + if platform.is_windows() { vars.extend(windows::env::default_env_vars(prefix, platform)); - } else if cfg!(target_family = "unix") { - vars.extend(unix::env::default_env_vars(prefix)); - } - - if platform.is_osx() { + } else if platform.is_osx() { vars.extend(macos::env::default_env_vars(prefix, platform)); } else if platform.is_linux() { vars.extend(linux::env::default_env_vars(prefix, platform)); @@ -189,15 +173,9 @@ pub fn os_vars(prefix: &Path, platform: &Platform) -> HashMap { vars } -macro_rules! insert { - ($map:expr, $key:expr, $value:expr) => { - $map.insert($key.to_string(), $value.to_string()); - }; -} - /// Set environment variables that help to force color output. -fn force_color_vars() -> HashMap { - let mut vars = HashMap::::new(); +fn force_color_vars() -> HashMap> { + let mut vars = HashMap::new(); insert!(vars, "CLICOLOR_FORCE", "1"); insert!(vars, "FORCE_COLOR", "1"); @@ -216,8 +194,8 @@ fn force_color_vars() -> HashMap { /// Return all variables that should be set during the build process, including /// operating system specific environment variables. -pub fn vars(output: &Output, build_state: &str) -> HashMap { - let mut vars = HashMap::::new(); +pub fn vars(output: &Output, build_state: &str) -> HashMap> { + let mut vars = HashMap::new(); insert!(vars, "CONDA_BUILD", "1"); insert!(vars, "PYTHONNOUSERSITE", "1"); @@ -324,10 +302,7 @@ pub fn vars(output: &Output, build_state: &str) -> HashMap { // for reproducibility purposes, set the SOURCE_DATE_EPOCH to the configured timestamp // this value will be taken from the previous package for rebuild purposes let timestamp_epoch_secs = output.build_configuration.timestamp.timestamp(); - vars.insert( - "SOURCE_DATE_EPOCH".to_string(), - timestamp_epoch_secs.to_string(), - ); + insert!(vars, "SOURCE_DATE_EPOCH", timestamp_epoch_secs); vars } diff --git a/src/linux/env.rs b/src/linux/env.rs index d19939eb..6d403f69 100644 --- a/src/linux/env.rs +++ b/src/linux/env.rs @@ -7,7 +7,10 @@ use rattler_conda_types::Platform; use crate::unix; /// Get default env vars for Linux -pub fn default_env_vars(prefix: &Path, target_platform: &Platform) -> HashMap { +pub fn default_env_vars( + prefix: &Path, + target_platform: &Platform, +) -> HashMap> { let mut vars = unix::env::default_env_vars(prefix); let build_distro = match target_platform { @@ -21,42 +24,25 @@ pub fn default_env_vars(prefix: &Path, target_platform: &Platform) -> HashMap HashMap { - let mut vars = HashMap::new(); +pub fn default_env_vars( + prefix: &Path, + target_platform: &Platform, +) -> HashMap> { + let mut vars = unix::env::default_env_vars(prefix); let t_string = target_platform.to_string(); let arch = t_string.split('-').collect::>()[1]; let (osx_arch, deployment_target, build) = match arch { @@ -13,11 +18,11 @@ pub fn default_env_vars(_prefix: &Path, target_platform: &Platform) -> HashMap ("x86_64", "10.9", "x86_64-apple-darwin13.4.0"), }; - vars.insert("OSX_ARCH".to_string(), osx_arch.to_string()); + vars.insert("OSX_ARCH".to_string(), Some(osx_arch.to_string())); vars.insert( "MACOSX_DEPLOYMENT_TARGET".to_string(), - deployment_target.to_string(), + Some(deployment_target.to_string()), ); - vars.insert("BUILD".to_string(), build.to_string()); + vars.insert("BUILD".to_string(), Some(build.to_string())); vars } diff --git a/src/package_test/run_test.rs b/src/package_test/run_test.rs index 699833e8..06eef75e 100644 --- a/src/package_test/run_test.rs +++ b/src/package_test/run_test.rs @@ -101,10 +101,10 @@ impl Tests { let platform = Platform::current(); let mut env_vars = env_vars::os_vars(environment, &platform); env_vars.retain(|key, _| key != ShellEnum::default().path_var(&platform)); - env_vars.extend(pkg_vars.clone()); + env_vars.extend(pkg_vars.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); env_vars.insert( "PREFIX".to_string(), - environment.to_string_lossy().to_string(), + Some(environment.to_string_lossy().to_string()), ); let tmp_dir = tempfile::tempdir()?; @@ -580,8 +580,11 @@ impl CommandsTest { let platform = Platform::current(); let mut env_vars = env_vars::os_vars(prefix, &platform); env_vars.retain(|key, _| key != ShellEnum::default().path_var(&platform)); - env_vars.extend(pkg_vars.clone()); - env_vars.insert("PREFIX".to_string(), run_env.to_string_lossy().to_string()); + env_vars.extend(pkg_vars.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + env_vars.insert( + "PREFIX".to_string(), + Some(run_env.to_string_lossy().to_string()), + ); // copy all test files to a temporary directory and set it as the working directory let tmp_dir = tempfile::tempdir()?; diff --git a/src/script.rs b/src/script.rs index f00f6a53..ceae54bf 100644 --- a/src/script.rs +++ b/src/script.rs @@ -9,6 +9,7 @@ use rattler_shell::{ activation::{ActivationError, ActivationVariables, Activator}, shell::{self, Shell}, }; +use std::collections::HashSet; use std::ffi::OsStr; use std::io::Error; use std::{ @@ -565,7 +566,7 @@ impl Script { pub async fn run_script( &self, - env_vars: HashMap, + env_vars: HashMap>, work_dir: &Path, recipe_dir: &Path, run_prefix: &Path, @@ -605,6 +606,7 @@ impl Script { let env_vars = env_vars .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) .chain(self.env().clone().into_iter()) .collect::>(); @@ -690,6 +692,22 @@ impl Script { } impl Output { + /// Add environment variables from the variant to the environment variables. + fn env_vars_from_variant(&self) -> HashMap> { + let languages: HashSet<&str> = HashSet::from(["PERL", "LUA", "R", "NUMPY", "PYTHON"]); + self.variant() + .iter() + .filter_map(|(k, v)| { + let key_upper = k.to_uppercase(); + if !languages.contains(key_upper.as_str()) { + Some((k.clone(), Some(v.to_string()))) + } else { + None + } + }) + .collect() + } + pub async fn run_build_script(&self) -> Result<(), std::io::Error> { let span = tracing::info_span!("Running build script"); let _enter = span.enter(); @@ -698,6 +716,7 @@ impl Output { let target_platform = self.build_configuration.target_platform; let mut env_vars = env_vars::vars(self, "BUILD"); env_vars.extend(env_vars::os_vars(&host_prefix, &target_platform)); + env_vars.extend(self.env_vars_from_variant()); let selector_config = self.build_configuration.selector_config(); let mut jinja = Jinja::new(selector_config.clone()); diff --git a/src/unix/env.rs b/src/unix/env.rs index 5d7341c4..ec599aa6 100644 --- a/src/unix/env.rs +++ b/src/unix/env.rs @@ -1,19 +1,22 @@ use std::{collections::HashMap, path::Path}; -pub fn default_env_vars(prefix: &Path) -> HashMap { +pub fn default_env_vars(prefix: &Path) -> HashMap> { let mut vars = HashMap::new(); vars.insert( "HOME".to_string(), - std::env::var("HOME").unwrap_or_else(|_| "UNKNOWN".to_string()), + Some(std::env::var("HOME").unwrap_or_else(|_| "UNKNOWN".to_string())), ); vars.insert( "PKG_CONFIG_PATH".to_string(), - prefix.join("lib/pkgconfig").to_string_lossy().to_string(), + Some(prefix.join("lib/pkgconfig").to_string_lossy().to_string()), + ); + vars.insert( + "CMAKE_GENERATOR".to_string(), + Some("Unix Makefiles".to_string()), ); - vars.insert("CMAKE_GENERATOR".to_string(), "Unix Makefiles".to_string()); vars.insert( "SSL_CERT_FILE".to_string(), - std::env::var("SSL_CERT_FILE").unwrap_or_default(), + std::env::var("SSL_CERT_FILE").ok(), ); vars } diff --git a/src/variant_config.rs b/src/variant_config.rs index 20ed0eaf..1df99074 100644 --- a/src/variant_config.rs +++ b/src/variant_config.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ - _partialerror, + _partialerror, env_vars, hash::HashInfo, recipe::{ custom_yaml::{HasSpan, Node, RenderedMappingNode, RenderedNode, TryConvertNode}, @@ -437,6 +437,10 @@ impl VariantConfig { let use_keys = &parsed_recipe.build().variant().use_keys; used_vars.extend(use_keys.iter().cloned()); + // Environment variables can be overwritten by the variant configuration + let env_vars = env_vars::os_vars(&PathBuf::new(), &selector_config.host_platform); + used_vars.extend(env_vars.keys().cloned()); + let target_platform = if noarch_type.is_none() { selector_config.target_platform } else { diff --git a/src/windows/env.rs b/src/windows/env.rs index 5c150347..4636b0e4 100644 --- a/src/windows/env.rs +++ b/src/windows/env.rs @@ -39,41 +39,31 @@ fn to_cygdrive(path: &Path) -> String { } } -pub fn default_env_vars(prefix: &Path, target_platform: &Platform) -> HashMap { - let win_arch = match target_platform { - Platform::Win32 => "i386", - Platform::Win64 => "amd64", - // TODO: Is this correct? - Platform::WinArm64 => "arm64", - Platform::NoArch => "noarch", - _ => panic!("Non windows platform passed to windows env vars"), - }; - - let win_msvc = "19.0.0"; - - // let (drive, tail) = prefix.split(":"); - +pub fn default_env_vars( + prefix: &Path, + target_platform: &Platform, +) -> HashMap> { let library_prefix = prefix.join("Library"); - let mut vars = HashMap::::new(); + let mut vars = HashMap::>::new(); vars.insert( "SCRIPTS".to_string(), - prefix.join("Scripts").to_string_lossy().to_string(), + Some(prefix.join("Scripts").to_string_lossy().to_string()), ); vars.insert( "LIBRARY_PREFIX".to_string(), - library_prefix.to_string_lossy().to_string(), + Some(library_prefix.to_string_lossy().to_string()), ); vars.insert( "LIBRARY_BIN".to_string(), - library_prefix.join("bin").to_string_lossy().to_string(), + Some(library_prefix.join("bin").to_string_lossy().to_string()), ); vars.insert( "LIBRARY_INC".to_string(), - library_prefix.join("include").to_string_lossy().to_string(), + Some(library_prefix.join("include").to_string_lossy().to_string()), ); vars.insert( "LIBRARY_LIB".to_string(), - library_prefix.join("lib").to_string_lossy().to_string(), + Some(library_prefix.join("lib").to_string_lossy().to_string()), ); let default_vars = vec![ @@ -112,24 +102,36 @@ pub fn default_env_vars(prefix: &Path, target_platform: &Platform) -> HashMap "i386", + Platform::Win64 => "amd64", + Platform::WinArm64 => "arm64", + Platform::NoArch => "noarch", + _ => panic!("Non windows platform passed to windows env vars"), + }; + vars.insert( "BUILD".to_string(), - std::env::var("BUILD").unwrap_or_else(|_| format!("{}-pc-windows-{}", win_arch, win_msvc)), + Some( + std::env::var("BUILD") + .unwrap_or_else(|_| format!("{}-pc-windows-{}", win_arch, win_msvc)), + ), ); - vars.insert("CYGWIN_PREFIX".to_string(), to_cygdrive(prefix)); + vars.insert("CYGWIN_PREFIX".to_string(), Some(to_cygdrive(prefix))); let re_vs_comntools = Regex::new(r"^VS[0-9]{2,3}COMNTOOLS$").unwrap(); let re_vs_installdir = Regex::new(r"^VS[0-9]{4}INSTALLDIR$").unwrap(); for (key, val) in std::env::vars() { if re_vs_comntools.is_match(&key) || re_vs_installdir.is_match(&key) { - vars.insert(key, val); + vars.insert(key, Some(val)); } } diff --git a/test-data/recipes/env_vars/recipe.yaml b/test-data/recipes/env_vars/recipe.yaml new file mode 100644 index 00000000..ff531e44 --- /dev/null +++ b/test-data/recipes/env_vars/recipe.yaml @@ -0,0 +1,11 @@ +package: + name: env_var_test + version: 0.1.0 + +build: + script: + - if: unix + then: + - echo $MAKEFLAGS > $PREFIX/makeflags.txt + else: + - echo %MAKEFLAGS% > %PREFIX%\makeflags.txt diff --git a/test-data/recipes/env_vars/variants.yaml b/test-data/recipes/env_vars/variants.yaml new file mode 100644 index 00000000..ffe1ee26 --- /dev/null +++ b/test-data/recipes/env_vars/variants.yaml @@ -0,0 +1,2 @@ +MAKEFLAGS: + - OVERRIDDEN_MAKEFLAGS \ No newline at end of file diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 27165739..e316bceb 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -969,3 +969,19 @@ def test_cache_install( pkg2 = get_extracted_package(tmp_path, "check-2") assert (pkg1 / "info/index.json").exists() assert (pkg2 / "info/index.json").exists() + + +def test_env_vars_override(rattler_build: RattlerBuild, recipes: Path, tmp_path: Path): + rattler_build.build( + recipes / "env_vars", + tmp_path, + ) + + pkg = get_extracted_package(tmp_path, "env_var_test") + + # assert (pkg / "info/paths.json").exists() + text = (pkg / "makeflags.txt").read_text() + assert text.strip() == "OVERRIDDEN_MAKEFLAGS" + + variant_config = json.loads((pkg / "info/hash_input.json").read_text()) + assert variant_config["MAKEFLAGS"] == "OVERRIDDEN_MAKEFLAGS"