From 2a5bb22b5a9f062b53dd8df29cf4fa38accacd9e Mon Sep 17 00:00:00 2001 From: be-marc Date: Thu, 29 Aug 2024 14:01:10 +0200 Subject: [PATCH 1/2] feat: set blas threads to 1 --- R/worker.R | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/R/worker.R b/R/worker.R index d8543e43a..d8fe15fb5 100644 --- a/R/worker.R +++ b/R/worker.R @@ -262,6 +262,15 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, # reduce data.table and blas threads to 1 if (!is_sequential) { setDTthreads(1, restore_after_fork = TRUE) + old_blas = Sys.getenv("OPENBLAS_NUM_THREADS") + old_mkl = Sys.getenv("MKL_NUM_THREADS") + Sys.setenv(OPENBLAS_NUM_THREADS = 1) + Sys.setenv(MKL_NUM_THREADS = 1) + + on.exit({ + Sys.setenv(OPENBLAS_NUM_THREADS = old_blas) + Sys.setenv(MKL_NUM_THREADS = old_mkl) + }, add = TRUE) # RhpcBLASctl is licensed under AGPL and therefore should be in suggest #1023 if (require_namespaces("RhpcBLASctl", quietly = TRUE)) { From 28000c14cb6684d3bd0a763926fc11d0ed2f9a4d Mon Sep 17 00:00:00 2001 From: Michel Lang Date: Thu, 29 Aug 2024 18:09:07 +0200 Subject: [PATCH 2/2] rely on RhpcBLASctl if installed --- R/worker.R | 19 ++++++++++--------- man-roxygen/section_parallelization.R | 2 ++ man/Learner.Rd | 2 +- man/benchmark.Rd | 2 ++ man/resample.Rd | 2 ++ 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/R/worker.R b/R/worker.R index d8fe15fb5..3ee363ebc 100644 --- a/R/worker.R +++ b/R/worker.R @@ -262,21 +262,22 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, # reduce data.table and blas threads to 1 if (!is_sequential) { setDTthreads(1, restore_after_fork = TRUE) - old_blas = Sys.getenv("OPENBLAS_NUM_THREADS") - old_mkl = Sys.getenv("MKL_NUM_THREADS") - Sys.setenv(OPENBLAS_NUM_THREADS = 1) - Sys.setenv(MKL_NUM_THREADS = 1) - - on.exit({ - Sys.setenv(OPENBLAS_NUM_THREADS = old_blas) - Sys.setenv(MKL_NUM_THREADS = old_mkl) - }, add = TRUE) # RhpcBLASctl is licensed under AGPL and therefore should be in suggest #1023 if (require_namespaces("RhpcBLASctl", quietly = TRUE)) { old_blas_threads = RhpcBLASctl::blas_get_num_procs() on.exit(RhpcBLASctl::blas_set_num_threads(old_blas_threads), add = TRUE) RhpcBLASctl::blas_set_num_threads(1) + } else { # try the bare minimum to disable threading of the most popular blas implementations + old_blas = Sys.getenv("OPENBLAS_NUM_THREADS") + old_mkl = Sys.getenv("MKL_NUM_THREADS") + Sys.setenv(OPENBLAS_NUM_THREADS = 1) + Sys.setenv(MKL_NUM_THREADS = 1) + + on.exit({ + Sys.setenv(OPENBLAS_NUM_THREADS = old_blas) + Sys.setenv(MKL_NUM_THREADS = old_mkl) + }, add = TRUE) } } # restore logger thresholds diff --git a/man-roxygen/section_parallelization.R b/man-roxygen/section_parallelization.R index 5d2d1f926..6c4fed578 100644 --- a/man-roxygen/section_parallelization.R +++ b/man-roxygen/section_parallelization.R @@ -4,3 +4,5 @@ #' One job is one resampling iteration, and all jobs are send to an apply function #' from \CRANpkg{future.apply} in a single batch. #' To select a parallel backend, use [future::plan()]. +#' More on parallelization can be found in the book: +#' \url{https://mlr3book.mlr-org.com/chapters/chapter10/advanced_technical_aspects_of_mlr3.html} diff --git a/man/Learner.Rd b/man/Learner.Rd index 4381c1758..29da44fa9 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -300,7 +300,7 @@ Set of hyperparameters.} Controls how to execute the code in internal train and predict methods. Must be a named character vector with names \code{"train"} and \code{"predict"}. Possible values are \code{"none"}, \code{"try"}, \code{"evaluate"} (requires package \CRANpkg{evaluate}) and \code{"callr"} (requires package \CRANpkg{callr}). -When encapsulation is activated, a fallback learner must be set. +When encapsulation is activated, a fallback learner must be set, If no learner is set in \verb{$fallback}, the default fallback learner is used (see \code{mlr_reflections$task_types}). See \code{\link[mlr3misc:encapsulate]{mlr3misc::encapsulate()}} for more details.} diff --git a/man/benchmark.Rd b/man/benchmark.Rd index 2715e223d..9cfc995f7 100644 --- a/man/benchmark.Rd +++ b/man/benchmark.Rd @@ -99,6 +99,8 @@ This function can be parallelized with the \CRANpkg{future} package. One job is one resampling iteration, and all jobs are send to an apply function from \CRANpkg{future.apply} in a single batch. To select a parallel backend, use \code{\link[future:plan]{future::plan()}}. +More on parallelization can be found in the book: +\url{https://mlr3book.mlr-org.com/chapters/chapter10/advanced_technical_aspects_of_mlr3.html} } \section{Progress Bars}{ diff --git a/man/resample.Rd b/man/resample.Rd index b89e9be03..b972108ef 100644 --- a/man/resample.Rd +++ b/man/resample.Rd @@ -100,6 +100,8 @@ This function can be parallelized with the \CRANpkg{future} package. One job is one resampling iteration, and all jobs are send to an apply function from \CRANpkg{future.apply} in a single batch. To select a parallel backend, use \code{\link[future:plan]{future::plan()}}. +More on parallelization can be found in the book: +\url{https://mlr3book.mlr-org.com/chapters/chapter10/advanced_technical_aspects_of_mlr3.html} } \section{Progress Bars}{