Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: improve env var handling #1088

Merged
merged 5 commits into from
Oct 7, 2024
Merged

fix: improve env var handling #1088

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
55b132d
put default values into option
wolfv Oct 7, 2024
08e2cc2
move env vars to optional strings
wolfv Oct 7, 2024
da98847
fix fmt
wolfv Oct 7, 2024
87edb49
mark env vars as used and propagate them into build environment
wolfv Oct 7, 2024
f633efd
test overriding env vars with variant config
wolfv Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 53 additions & 78 deletions src/env_vars.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,14 @@ use rattler_conda_types::Platform;
use crate::linux;
use crate::macos;
use crate::metadata::Output;
use crate::unix;
use crate::windows;

macro_rules! insert {
($map:expr, $key:expr, $value:expr) => {
$map.insert($key.to_string(), Some($value.to_string()));
};
}

fn get_stdlib_dir(prefix: &Path, platform: &Platform, py_ver: &str) -> PathBuf {
if platform.is_windows() {
prefix.join("Lib")
Expand All @@ -26,22 +31,22 @@ fn get_sitepackages_dir(prefix: &Path, platform: &Platform, py_ver: &str) -> Pat
/// Returns a map of environment variables for Python that are used in the build process.
///
/// Variables:
/// - PYTHON: path to Python executable
/// - PY3K: 1 if Python 3, 0 if Python 2
/// - PY_VER: Python version (major.minor), e.g. 3.8
/// - STDLIB_DIR: Python standard library directory
/// - SP_DIR: Python site-packages directory
/// - NPY_VER: Numpy version (major.minor), e.g. 1.19
/// - NPY_DISTUTILS_APPEND_FLAGS: 1 (https://github.com/conda/conda-build/pull/3015)
pub fn python_vars(output: &Output) -> HashMap<String, String> {
let mut result = HashMap::<String, String>::new();
/// - `PYTHON`: path to Python executable
/// - `PY3K`: 1 if Python 3, 0 if Python 2
/// - `PY_VER`: Python version (major.minor), e.g. 3.8
/// - `STDLIB_DIR`: Python standard library directory
/// - `SP_DIR`: Python site-packages directory
/// - `NPY_VER`: NumPy version (major.minor), e.g. 1.19
/// - `NPY_DISTUTILS_APPEND_FLAGS`: 1 (https://github.com/conda/conda-build/pull/3015)
pub fn python_vars(output: &Output) -> HashMap<String, Option<String>> {
let mut result = HashMap::new();

if output.host_platform().is_windows() {
let python = output.prefix().join("python.exe");
result.insert("PYTHON".to_string(), python.to_string_lossy().to_string());
insert!(result, "PYTHON", python.to_string_lossy());
} else {
let python = output.prefix().join("bin/python");
result.insert("PYTHON".to_string(), python.to_string_lossy().to_string());
insert!(result, "PYTHON", python.to_string_lossy());
}

// find python in the host dependencies
Expand All @@ -60,32 +65,19 @@ pub fn python_vars(output: &Output) -> HashMap<String, String> {
let stdlib_dir = get_stdlib_dir(output.prefix(), output.host_platform(), &py_ver_str);
let site_packages_dir =
get_sitepackages_dir(output.prefix(), output.host_platform(), &py_ver_str);
result.insert(
"PY3K".to_string(),
if py_ver[0] == "3" {
"1".to_string()
} else {
"0".to_string()
},
);
result.insert("PY_VER".to_string(), py_ver_str);
result.insert(
"STDLIB_DIR".to_string(),
stdlib_dir.to_string_lossy().to_string(),
);
result.insert(
"SP_DIR".to_string(),
site_packages_dir.to_string_lossy().to_string(),
);
let py3k = if py_ver[0] == "3" { "1" } else { "0" };
insert!(result, "PY3K", py3k);
insert!(result, "PY_VER", py_ver_str);
insert!(result, "STDLIB_DIR", stdlib_dir.to_string_lossy());
insert!(result, "SP_DIR", site_packages_dir.to_string_lossy());
}

if let Some(npy_version) = output.variant().get("numpy") {
let np_ver = npy_version.split('.').collect::<Vec<_>>();
let np_ver = format!("{}.{}", np_ver[0], np_ver[1]);

result.insert("NPY_VER".to_string(), np_ver);
insert!(result, "NPY_VER", np_ver);
insert!(result, "NPY_DISTUTILS_APPEND_FLAGS", "1");
}
result.insert("NPY_DISTUTILS_APPEND_FLAGS".to_string(), "1".to_string());

result
}
Expand All @@ -97,11 +89,11 @@ pub fn python_vars(output: &Output) -> HashMap<String, String> {
/// - R: Path to R executable
/// - R_USER: Path to R user directory
///
pub fn r_vars(output: &Output) -> HashMap<String, String> {
let mut result = HashMap::<String, String>::new();
pub fn r_vars(output: &Output) -> HashMap<String, Option<String>> {
let mut result = HashMap::new();

if let Some(r_ver) = output.variant().get("r-base") {
result.insert("R_VER".to_string(), r_ver.clone());
insert!(result, "R_VER", r_ver);

let r_bin = if output.host_platform().is_windows() {
output.prefix().join("Scripts/R.exe")
Expand All @@ -111,15 +103,15 @@ pub fn r_vars(output: &Output) -> HashMap<String, String> {

let r_user = output.prefix().join("Libs/R");

result.insert("R".to_string(), r_bin.to_string_lossy().to_string());
result.insert("R_USER".to_string(), r_user.to_string_lossy().to_string());
insert!(result, "R", r_bin.to_string_lossy());
insert!(result, "R_USER", r_user.to_string_lossy());
}

result
}

pub fn language_vars(output: &Output) -> HashMap<String, String> {
let mut result = HashMap::<String, String>::new();
pub fn language_vars(output: &Output) -> HashMap<String, Option<String>> {
let mut result = HashMap::new();

result.extend(python_vars(output));
result.extend(r_vars(output));
Expand All @@ -139,48 +131,40 @@ pub fn language_vars(output: &Output) -> HashMap<String, String> {
/// - LANG: Language (e.g. en_US.UTF-8)
/// - LC_ALL: Language (e.g. en_US.UTF-8)
/// - MAKEFLAGS: Make flags (e.g. -j4)
pub fn os_vars(prefix: &Path, platform: &Platform) -> HashMap<String, String> {
let mut vars = HashMap::<String, String>::new();
pub fn os_vars(prefix: &Path, platform: &Platform) -> HashMap<String, Option<String>> {
let mut vars = HashMap::new();

let path_var = if platform.is_windows() {
"Path"
} else {
"PATH"
};

vars.insert(
"CPU_COUNT".to_string(),
env::var("CPU_COUNT").unwrap_or_else(|_| num_cpus::get().to_string()),
);
vars.insert("LANG".to_string(), env::var("LANG").unwrap_or_default());
vars.insert("LC_ALL".to_string(), env::var("LC_ALL").unwrap_or_default());
vars.insert(
"MAKEFLAGS".to_string(),
env::var("MAKEFLAGS").unwrap_or_default(),
insert!(
vars,
"CPU_COUNT",
env::var("CPU_COUNT").unwrap_or_else(|_| num_cpus::get().to_string())
);
vars.insert("LANG".to_string(), env::var("LANG").ok());
vars.insert("LC_ALL".to_string(), env::var("LC_ALL").ok());
vars.insert("MAKEFLAGS".to_string(), env::var("MAKEFLAGS").ok());

let shlib_ext = if platform.is_windows() {
".dll".to_string()
".dll"
} else if platform.is_osx() {
".dylib".to_string()
".dylib"
} else if platform.is_linux() {
".so".to_string()
".so"
} else {
".not_implemented".to_string()
".not_implemented"
};

vars.insert("SHLIB_EXT".to_string(), shlib_ext);
if let Ok(path) = env::var(path_var) {
vars.insert(path_var.to_string(), path);
}
insert!(vars, "SHLIB_EXT", shlib_ext);
vars.insert(path_var.to_string(), env::var(path_var).ok());

if cfg!(target_family = "windows") {
if platform.is_windows() {
vars.extend(windows::env::default_env_vars(prefix, platform));
} else if cfg!(target_family = "unix") {
vars.extend(unix::env::default_env_vars(prefix));
}

if platform.is_osx() {
} else if platform.is_osx() {
vars.extend(macos::env::default_env_vars(prefix, platform));
} else if platform.is_linux() {
vars.extend(linux::env::default_env_vars(prefix, platform));
Expand All @@ -189,15 +173,9 @@ pub fn os_vars(prefix: &Path, platform: &Platform) -> HashMap<String, String> {
vars
}

macro_rules! insert {
($map:expr, $key:expr, $value:expr) => {
$map.insert($key.to_string(), $value.to_string());
};
}

/// Set environment variables that help to force color output.
fn force_color_vars() -> HashMap<String, String> {
let mut vars = HashMap::<String, String>::new();
fn force_color_vars() -> HashMap<String, Option<String>> {
let mut vars = HashMap::new();

insert!(vars, "CLICOLOR_FORCE", "1");
insert!(vars, "FORCE_COLOR", "1");
Expand All @@ -216,8 +194,8 @@ fn force_color_vars() -> HashMap<String, String> {

/// Return all variables that should be set during the build process, including
/// operating system specific environment variables.
pub fn vars(output: &Output, build_state: &str) -> HashMap<String, String> {
let mut vars = HashMap::<String, String>::new();
pub fn vars(output: &Output, build_state: &str) -> HashMap<String, Option<String>> {
let mut vars = HashMap::new();

insert!(vars, "CONDA_BUILD", "1");
insert!(vars, "PYTHONNOUSERSITE", "1");
Expand Down Expand Up @@ -324,10 +302,7 @@ pub fn vars(output: &Output, build_state: &str) -> HashMap<String, String> {
// for reproducibility purposes, set the SOURCE_DATE_EPOCH to the configured timestamp
// this value will be taken from the previous package for rebuild purposes
let timestamp_epoch_secs = output.build_configuration.timestamp.timestamp();
vars.insert(
"SOURCE_DATE_EPOCH".to_string(),
timestamp_epoch_secs.to_string(),
);
insert!(vars, "SOURCE_DATE_EPOCH", timestamp_epoch_secs);

vars
}
42 changes: 14 additions & 28 deletions src/linux/env.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ use rattler_conda_types::Platform;
use crate::unix;

/// Get default env vars for Linux
pub fn default_env_vars(prefix: &Path, target_platform: &Platform) -> HashMap<String, String> {
pub fn default_env_vars(
prefix: &Path,
target_platform: &Platform,
) -> HashMap<String, Option<String>> {
let mut vars = unix::env::default_env_vars(prefix);

let build_distro = match target_platform {
Expand All @@ -21,42 +24,25 @@ pub fn default_env_vars(prefix: &Path, target_platform: &Platform) -> HashMap<St
// filtered so it only contains the result of `linux_vars`
// which, before this change was empty, and after it only
// contains other QEMU env vars.
vars.insert(
"CFLAGS".to_string(),
std::env::var("CFLAGS").unwrap_or_default(),
);
vars.insert(
"CXXFLAGS".to_string(),
std::env::var("CXXFLAGS").unwrap_or_default(),
);
vars.insert(
"LDFLAGS".to_string(),
std::env::var("LDFLAGS").unwrap_or_default(),
);
vars.insert("CFLAGS".to_string(), std::env::var("CFLAGS").ok());
vars.insert("CXXFLAGS".to_string(), std::env::var("CXXFLAGS").ok());
vars.insert("LDFLAGS".to_string(), std::env::var("LDFLAGS").ok());
vars.insert(
"QEMU_LD_PREFIX".to_string(),
std::env::var("QEMU_LD_PREFIX").unwrap_or_default(),
);
vars.insert(
"QEMU_UNAME".to_string(),
std::env::var("QEMU_UNAME").unwrap_or_default(),
);
vars.insert(
"DEJAGNU".to_string(),
std::env::var("DEJAGNU").unwrap_or_default(),
);
vars.insert(
"DISPLAY".to_string(),
std::env::var("DISPLAY").unwrap_or_default(),
std::env::var("QEMU_LD_PREFIX").ok(),
);
vars.insert("QEMU_UNAME".to_string(), std::env::var("QEMU_UNAME").ok());
vars.insert("DEJAGNU".to_string(), std::env::var("DEJAGNU").ok());
vars.insert("DISPLAY".to_string(), std::env::var("DISPLAY").ok());
vars.insert(
"LD_RUN_PATH".to_string(),
std::env::var("LD_RUN_PATH")
.unwrap_or_else(|_| prefix.join("lib").to_string_lossy().to_string()),
.ok()
.or_else(|| Some(prefix.join("lib").to_string_lossy().to_string())),
);
vars.insert(
"BUILD".to_string(),
format!("{}-conda_{}-linux-gnu", build_arch, build_distro),
Some(format!("{}-conda_{}-linux-gnu", build_arch, build_distro)),
);

vars
Expand Down
15 changes: 10 additions & 5 deletions src/macos/env.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,14 @@
use rattler_conda_types::Platform;
use std::{collections::HashMap, path::Path};

use crate::unix;

/// Get default env vars for macOS
pub fn default_env_vars(_prefix: &Path, target_platform: &Platform) -> HashMap<String, String> {
let mut vars = HashMap::new();
pub fn default_env_vars(
prefix: &Path,
target_platform: &Platform,
) -> HashMap<String, Option<String>> {
let mut vars = unix::env::default_env_vars(prefix);
let t_string = target_platform.to_string();
let arch = t_string.split('-').collect::<Vec<&str>>()[1];
let (osx_arch, deployment_target, build) = match arch {
Expand All @@ -13,11 +18,11 @@ pub fn default_env_vars(_prefix: &Path, target_platform: &Platform) -> HashMap<S
_ => ("x86_64", "10.9", "x86_64-apple-darwin13.4.0"),
};

vars.insert("OSX_ARCH".to_string(), osx_arch.to_string());
vars.insert("OSX_ARCH".to_string(), Some(osx_arch.to_string()));
vars.insert(
"MACOSX_DEPLOYMENT_TARGET".to_string(),
deployment_target.to_string(),
Some(deployment_target.to_string()),
);
vars.insert("BUILD".to_string(), build.to_string());
vars.insert("BUILD".to_string(), Some(build.to_string()));
vars
}
11 changes: 7 additions & 4 deletions src/package_test/run_test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,10 @@ impl Tests {
let platform = Platform::current();
let mut env_vars = env_vars::os_vars(environment, &platform);
env_vars.retain(|key, _| key != ShellEnum::default().path_var(&platform));
env_vars.extend(pkg_vars.clone());
env_vars.extend(pkg_vars.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
env_vars.insert(
"PREFIX".to_string(),
environment.to_string_lossy().to_string(),
Some(environment.to_string_lossy().to_string()),
);
let tmp_dir = tempfile::tempdir()?;

Expand Down Expand Up @@ -580,8 +580,11 @@ impl CommandsTest {
let platform = Platform::current();
let mut env_vars = env_vars::os_vars(prefix, &platform);
env_vars.retain(|key, _| key != ShellEnum::default().path_var(&platform));
env_vars.extend(pkg_vars.clone());
env_vars.insert("PREFIX".to_string(), run_env.to_string_lossy().to_string());
env_vars.extend(pkg_vars.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
env_vars.insert(
"PREFIX".to_string(),
Some(run_env.to_string_lossy().to_string()),
);

// copy all test files to a temporary directory and set it as the working directory
let tmp_dir = tempfile::tempdir()?;
Expand Down
Loading
Loading