From 8ba1cfaba425476b8140474ed13d0e5d1f33d164 Mon Sep 17 00:00:00 2001 From: jeremie Date: Fri, 11 Aug 2023 15:52:01 -0400 Subject: [PATCH 1/3] use default dynamic threads --- benchmarks/regressor.jl | 18 +++++++++++------- src/eval.jl | 20 ++++++++++---------- src/fit-utils.jl | 14 +++++++------- src/fit.jl | 4 ++-- src/gpu/fit.jl | 4 ++-- src/loss.jl | 20 ++++++++++---------- src/predict.jl | 10 +++++----- 7 files changed, 47 insertions(+), 43 deletions(-) diff --git a/benchmarks/regressor.jl b/benchmarks/regressor.jl index b78b38af..a4ab6ded 100644 --- a/benchmarks/regressor.jl +++ b/benchmarks/regressor.jl @@ -15,6 +15,10 @@ import CUDA ### perf depth # desktop | 1e6 | depth 11 | cpu: 28s gpu: 73 sec | xgboost: 26s # desktop | 10e6 | depth 11 | cpu 205s gpu: 109 sec | xgboost 260s + +#threads +# laptop depth 6: 12.717845 seconds (2.08 M allocations: 466.228 MiB) + nobs = Int(1e6) num_feat = Int(100) nrounds = 200 @@ -55,13 +59,13 @@ params_xgb = Dict( :max_bin => 64, ) -dtrain = DMatrix(x_train, y_train) -watchlist = Dict("train" => DMatrix(x_train, y_train)); -@time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric=metric_xgb, params_xgb...); -# @btime m_xgb = xgboost($dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...); -@info "predict" -@time pred_xgb = XGBoost.predict(m_xgb, x_train); -# @btime XGBoost.predict($m_xgb, $x_train); +# dtrain = DMatrix(x_train, y_train) +# watchlist = Dict("train" => DMatrix(x_train, y_train)); +# @time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric=metric_xgb, params_xgb...); +# # @btime m_xgb = xgboost($dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...); +# @info "predict" +# @time pred_xgb = XGBoost.predict(m_xgb, x_train); +# # @btime XGBoost.predict($m_xgb, $x_train); # @info "lightgbm train:" # m_gbm = LGBMRegression( diff --git a/src/eval.jl b/src/eval.jl index ce952971..257d23d5 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -5,7 +5,7 @@ function mse( eval::AbstractVector; kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) eval[i] = w[i] * (p[1, i] - y[i])^2 end return sum(eval) / sum(w) @@ -20,7 +20,7 @@ function mae( eval::AbstractVector; kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) eval[i] = w[i] * abs(p[1, i] - y[i]) end return sum(eval) / sum(w) @@ -33,7 +33,7 @@ function logloss( eval::AbstractVector; kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) pred = sigmoid(p[1, i]) eval[i] = w[i] * (-y[i] * log(pred) + (y[i] - 1) * log(1 - pred)) end @@ -48,7 +48,7 @@ function mlogloss( kwargs... ) where {T} K = size(p, 1) - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) isum = zero(T) @inbounds for k in 1:K isum += exp(p[k, i]) @@ -65,7 +65,7 @@ function poisson( eval::AbstractVector; kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) pred = exp(p[1, i]) eval[i] = w[i] * 2 * (y[i] * (log(y[i]) - log(pred)) + pred - y[i]) end @@ -79,7 +79,7 @@ function gamma( eval::AbstractVector; kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) pred = exp(p[1, i]) eval[i] = w[i] * 2 * (log(pred / y[i]) + y[i] / pred - 1) end @@ -94,7 +94,7 @@ function tweedie( kwargs... ) where {T} rho = T(1.5) - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) pred = exp(p[1, i]) eval[i] = w[i] * @@ -114,7 +114,7 @@ function gaussian_mle( eval::AbstractVector; kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) eval[i] = -w[i] * (p[2, i] + (y[i] - p[1, i])^2 / (2 * exp(2 * p[2, i]))) end return sum(eval) / sum(w) @@ -127,7 +127,7 @@ function logistic_mle( eval::AbstractVector; kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) eval[i] = w[i] * (log(1 / 4 * sech(exp(-p[2, i]) * (y[i] - p[1, i]))^2) - p[2, i]) end return sum(eval) / sum(w) @@ -141,7 +141,7 @@ function wmae( alpha=0.5, kwargs... ) where {T} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) eval[i] = w[i] * ( alpha * max(y[i] - p[1, i], zero(T)) + diff --git a/src/fit-utils.jl b/src/fit-utils.jl index f49163a4..7f814294 100644 --- a/src/fit-utils.jl +++ b/src/fit-utils.jl @@ -11,7 +11,7 @@ function get_edges(X::AbstractMatrix{T}; fnames, nbins, rng=Random.TaskLocalRNG( edges = Vector{Vector{T}}(undef, nfeats) featbins = Vector{UInt8}(undef, nfeats) feattypes = Vector{Bool}(undef, nfeats) - @threads :static for j in 1:size(X, 2) + @threads for j in 1:size(X, 2) edges[j] = quantile(view(X, idx, j), (1:nbins-1) / nbins) if length(edges[j]) == 1 edges[j] = [minimum(view(X, idx, j))] @@ -30,7 +30,7 @@ function get_edges(df; fnames, nbins, rng=Random.TaskLocalRNG()) nfeats = length(fnames) featbins = Vector{UInt8}(undef, nfeats) feattypes = Vector{Bool}(undef, nfeats) - @threads :static for j in eachindex(fnames) + @threads for j in eachindex(fnames) col = view(Tables.getcolumn(df, fnames[j]), idx) if eltype(col) <: Bool edges[j] = [false, true] @@ -63,7 +63,7 @@ Transform feature data into a UInt8 binarized matrix. """ function binarize(X::AbstractMatrix; fnames, edges) x_bin = zeros(UInt8, size(X)) - @threads :static for j in axes(X, 2) + @threads for j in axes(X, 2) x_bin[:, j] .= searchsortedfirst.(Ref(edges[j]), view(X, :, j)) end return x_bin @@ -72,7 +72,7 @@ end function binarize(df; fnames, edges) nobs = length(Tables.getcolumn(df, 1)) x_bin = zeros(UInt8, nobs, length(fnames)) - @threads :static for j in eachindex(fnames) + @threads for j in eachindex(fnames) col = Tables.getcolumn(df, fnames[j]) if eltype(col) <: Bool x_bin[:, j] .= col .+ 1 @@ -232,7 +232,7 @@ function update_hist!( is::AbstractVector, js::AbstractVector, ) where {L<:GradientRegression} - @threads :static for j in js + @threads for j in js @inbounds @simd for i in is bin = x_bin[i, j] hist[j][1, bin] += ∇[1, i] @@ -255,7 +255,7 @@ function update_hist!( is::AbstractVector, js::AbstractVector, ) where {L<:MLE2P} - @threads :static for j in js + @threads for j in js @inbounds @simd for i in is bin = x_bin[i, j] hist[j][1, bin] += ∇[1, i] @@ -280,7 +280,7 @@ function update_hist!( is::AbstractVector, js::AbstractVector, ) where {L} - @threads :static for j in js + @threads for j in js @inbounds for i in is bin = x_bin[i, j] @inbounds @simd for k in axes(∇, 1) diff --git a/src/fit.jl b/src/fit.jl index f6d2aac0..4b508426 100644 --- a/src/fit.jl +++ b/src/fit.jl @@ -91,7 +91,7 @@ function grow_tree!( update_hist!(L, nodes[n].h, ∇, x_bin, nodes[n].is, js) end end - @threads :static for n ∈ sort(n_current) + @threads for n ∈ sort(n_current) update_gains!(nodes[n], js, params, feattypes, monotone_constraints) end end @@ -215,7 +215,7 @@ function grow_otree!( update_hist!(L, nodes[n].h, ∇, x_bin, nodes[n].is, js) end end - @threads :static for n ∈ n_current + @threads for n ∈ n_current update_gains!(nodes[n], js, params, feattypes, monotone_constraints) end diff --git a/src/gpu/fit.jl b/src/gpu/fit.jl index b6751599..b0935451 100644 --- a/src/gpu/fit.jl +++ b/src/gpu/fit.jl @@ -90,7 +90,7 @@ function grow_tree!( update_hist_gpu!(nodes[n].h, h∇, ∇, x_bin, nodes[n].is, jsg, js) end end - @threads :static for n ∈ sort(n_current) + @threads for n ∈ sort(n_current) update_gains!(nodes[n], js, params, feattypes, monotone_constraints) end end @@ -217,7 +217,7 @@ function grow_otree!( update_hist_gpu!(nodes[n].h, h∇, ∇, x_bin, nodes[n].is, jsg, js) end end - @threads :static for n ∈ n_current + @threads for n ∈ n_current update_gains!(nodes[n], js, params, feattypes, monotone_constraints) end diff --git a/src/loss.jl b/src/loss.jl index 85f42548..07d006dc 100644 --- a/src/loss.jl +++ b/src/loss.jl @@ -1,6 +1,6 @@ # MSE function update_grads!(∇::Matrix, p::Matrix, y::Vector, ::EvoTreeRegressor{L}) where {L<:MSE} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) @inbounds ∇[1, i] = 2 * (p[1, i] - y[i]) * ∇[3, i] @inbounds ∇[2, i] = 2 * ∇[3, i] end @@ -8,7 +8,7 @@ end # LogLoss - on linear predictor function update_grads!(∇::Matrix, p::Matrix, y::Vector, ::EvoTreeRegressor{L}) where {L<:LogLoss} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) @inbounds pred = sigmoid(p[1, i]) @inbounds ∇[1, i] = (pred - y[i]) * ∇[3, i] @inbounds ∇[2, i] = pred * (1 - pred) * ∇[3, i] @@ -17,7 +17,7 @@ end # Poisson function update_grads!(∇::Matrix, p::Matrix, y::Vector, ::EvoTreeCount{L}) where {L<:Poisson} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) @inbounds pred = exp(p[1, i]) @inbounds ∇[1, i] = (pred - y[i]) * ∇[3, i] @inbounds ∇[2, i] = pred * ∇[3, i] @@ -26,7 +26,7 @@ end # Gamma function update_grads!(∇::Matrix, p::Matrix, y::Vector, ::EvoTreeRegressor{L}) where {L<:Gamma} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) @inbounds pred = exp(p[1, i]) @inbounds ∇[1, i] = 2 * (1 - y[i] / pred) * ∇[3, i] @inbounds ∇[2, i] = 2 * y[i] / pred * ∇[3, i] @@ -36,7 +36,7 @@ end # Tweedie function update_grads!(∇::Matrix, p::Matrix, y::Vector, ::EvoTreeRegressor{L}) where {L<:Tweedie} rho = eltype(p)(1.5) - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) @inbounds pred = exp(p[1, i]) @inbounds ∇[1, i] = 2 * (pred^(2 - rho) - y[i] * pred^(1 - rho)) * ∇[3, i] @inbounds ∇[2, i] = @@ -46,7 +46,7 @@ end # L1 function update_grads!(∇::Matrix, p::Matrix, y::Vector, params::EvoTreeRegressor{L}) where {L<:L1} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) @inbounds ∇[1, i] = (params.alpha * max(y[i] - p[1, i], 0) - (1 - params.alpha) * max(p[1, i] - y[i], 0)) * ∇[3, i] @@ -56,7 +56,7 @@ end # MLogLoss function update_grads!(∇::Matrix{T}, p::Matrix, y::Vector, ::EvoTreeClassifier{L}) where {L<:MLogLoss,T} K = size(p, 1) - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) isum = zero(T) @inbounds for k = 1:K isum += exp(p[k, i]) @@ -75,7 +75,7 @@ end # Quantile function update_grads!(∇::Matrix, p::Matrix, y::Vector, params::EvoTreeRegressor{L}) where {L<:Quantile} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) @inbounds ∇[1, i] = y[i] > p[1, i] ? params.alpha * ∇[3, i] : (params.alpha - 1) * ∇[3, i] @inbounds ∇[2, i] = y[i] - p[1, i] # δ² serves to calculate the quantile value - hence no weighting on δ² end @@ -85,7 +85,7 @@ end # pred[i][1] = μ # pred[i][2] = log(σ) function update_grads!(∇::Matrix, p::Matrix, y::Vector, ::Union{EvoTreeGaussian{L},EvoTreeMLE{L}}) where {L<:GaussianMLE} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) # first order @inbounds ∇[1, i] = (p[1, i] - y[i]) / exp(2 * p[2, i]) * ∇[5, i] @inbounds ∇[2, i] = (1 - (p[1, i] - y[i])^2 / exp(2 * p[2, i])) * ∇[5, i] @@ -100,7 +100,7 @@ end # pred[i][1] = μ # pred[i][2] = log(s) function update_grads!(∇::Matrix, p::Matrix, y::Vector, ::EvoTreeMLE{L}) where {L<:LogisticMLE} - @threads :static for i in eachindex(y) + @threads for i in eachindex(y) # first order @inbounds ∇[1, i] = -tanh((y[i] - p[1, i]) / (2 * exp(p[2, i]))) * exp(-p[2, i]) * ∇[5, i] diff --git a/src/predict.jl b/src/predict.jl index 7fd4f07e..300ebe66 100644 --- a/src/predict.jl +++ b/src/predict.jl @@ -1,5 +1,5 @@ function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, feattypes::Vector{Bool}) where {L<:GradientRegression,K,T} - @inbounds @threads for i in axes(x_bin, 1) + @threads for i in axes(x_bin, 1) nid = 1 @inbounds while tree.split[nid] feat = tree.feat[nid] @@ -12,7 +12,7 @@ function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, featty end function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, feattypes::Vector{Bool}) where {L<:LogLoss,K,T} - @inbounds @threads for i in axes(x_bin, 1) + @threads for i in axes(x_bin, 1) nid = 1 @inbounds while tree.split[nid] feat = tree.feat[nid] @@ -25,7 +25,7 @@ function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, featty end function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, feattypes::Vector{Bool}) where {L<:MLE2P,K,T} - @inbounds @threads for i in axes(x_bin, 1) + @threads for i in axes(x_bin, 1) nid = 1 @inbounds while tree.split[nid] feat = tree.feat[nid] @@ -39,7 +39,7 @@ function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, featty end function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, feattypes::Vector{Bool}) where {L<:MLogLoss,K,T} - @inbounds @threads for i in axes(x_bin, 1) + @threads for i in axes(x_bin, 1) nid = 1 @inbounds while tree.split[nid] feat = tree.feat[nid] @@ -60,7 +60,7 @@ end Generic fallback to add predictions of `tree` to existing `pred` matrix. """ function predict!(pred::Matrix{T}, tree::Tree{L,K}, x_bin::Matrix{UInt8}, feattypes::Vector{Bool}) where {L,K,T} - @inbounds @threads for i in axes(x_bin, 1) + @threads for i in axes(x_bin, 1) nid = 1 @inbounds while tree.split[nid] feat = tree.feat[nid] From 3baf19e55b108b4e110f9debcaa3cd012af6b5db Mon Sep 17 00:00:00 2001 From: jeremie Date: Fri, 11 Aug 2023 15:53:24 -0400 Subject: [PATCH 2/3] bump version --- Project.toml | 2 +- benchmarks/regressor.jl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index a78cda25..b15833bb 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "EvoTrees" uuid = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" authors = ["jeremiedb "] -version = "0.15.2" +version = "0.15.3" [deps] BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" diff --git a/benchmarks/regressor.jl b/benchmarks/regressor.jl index a4ab6ded..d38804b6 100644 --- a/benchmarks/regressor.jl +++ b/benchmarks/regressor.jl @@ -59,13 +59,13 @@ params_xgb = Dict( :max_bin => 64, ) -# dtrain = DMatrix(x_train, y_train) -# watchlist = Dict("train" => DMatrix(x_train, y_train)); -# @time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric=metric_xgb, params_xgb...); -# # @btime m_xgb = xgboost($dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...); -# @info "predict" -# @time pred_xgb = XGBoost.predict(m_xgb, x_train); -# # @btime XGBoost.predict($m_xgb, $x_train); +dtrain = DMatrix(x_train, y_train) +watchlist = Dict("train" => DMatrix(x_train, y_train)); +@time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric=metric_xgb, params_xgb...); +# @btime m_xgb = xgboost($dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...); +@info "predict" +@time pred_xgb = XGBoost.predict(m_xgb, x_train); +# @btime XGBoost.predict($m_xgb, $x_train); # @info "lightgbm train:" # m_gbm = LGBMRegression( From a70d925e4196e22f95e5b09d3af2920a8bab98fd Mon Sep 17 00:00:00 2001 From: jeremie Date: Sat, 12 Aug 2023 13:56:43 -0400 Subject: [PATCH 3/3] new params default: nrounds=100, nbins=64 max-depth=6 --- Project.toml | 2 +- experiments/shuffling.jl | 144 +++++++++++++++++++++++++++++++++++++++ src/fit-utils.jl | 4 +- src/models.jl | 30 ++++---- 4 files changed, 162 insertions(+), 18 deletions(-) create mode 100644 experiments/shuffling.jl diff --git a/Project.toml b/Project.toml index b15833bb..7e1bc254 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "EvoTrees" uuid = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" authors = ["jeremiedb "] -version = "0.15.3" +version = "0.16.0" [deps] BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" diff --git a/experiments/shuffling.jl b/experiments/shuffling.jl new file mode 100644 index 00000000..8cd51bba --- /dev/null +++ b/experiments/shuffling.jl @@ -0,0 +1,144 @@ +using DataFrames +using Distributions +using EvoTrees +using LinearAlgebra +using GLM +using Random + +δ = 1.0e-6 +b = fill(1.0 - δ, 3, 3) + δ * I +z = zeros(3, 3) +y = fill(0.5, 3) +dist = MvNormal([ + b z z 0.8*y + z b z y + z z b 1.2*y + 0.8*y' y' 1.2*y' 1.0]) +Random.seed!(1) +mat = rand(dist, 10_000); +df = DataFrame(transpose(mat), [string.("x", 1:9); "y"]); +target_name = "y" + +################################# +# Tables API +################################# +config = EvoTreeRegressor(seed=123) +m1 = fit_evotree(config, + df; + target_name="y", + verbosity=0); +EvoTrees.importance(m1) + +config = EvoTreeRegressor(seed=124) +m2 = fit_evotree(config, + df; + target_name="y", + verbosity=0); +EvoTrees.importance(m2) + +# permuted tables doesn't return the same result - numerical rounding error? +df2 = df[!, 10:-1:1] +config = EvoTreeRegressor() +m3 = fit_evotree(config, + df2; + target_name="y", + verbosity=0); +EvoTrees.importance(m3) + +# manual check on col permutations +config = EvoTreeRegressor(max_depth=4) +m1, cache1 = EvoTrees.init(config, df; target_name); +EvoTrees.grow_evotree!(m1, cache1, config, EvoTrees.CPU) +EvoTrees.importance(m1) + +df2 = df[!, 10:-1:1]; +config = EvoTreeRegressor(max_depth=4) +m2, cache2 = EvoTrees.init(config, df2; target_name); +EvoTrees.grow_evotree!(m2, cache2, config, EvoTrees.CPU) +EvoTrees.importance(m2) + +all(cache1.x_bin .== cache2.x_bin[:, 9:-1:1]) +all(cache1.edges .== cache2.edges[9:-1:1]) +m1.trees[2] +m2.trees[2] + +m1.trees[2].feat +m2.trees[2].feat + +Int.(m1.trees[2].cond_bin) +Int.(m2.trees[2].cond_bin) + + +config = EvoTreeRegressor(nrounds=100, eta=0.05, colsample=1.0) +m3 = fit_evotree(config, + df; + target_name="y", + verbosity=0); +EvoTrees.importance(m3) + +################################# +# Tables API +################################# +config = EvoTreeRegressor(colsample=0.5) +m1 = fit_evotree(config, + df; + target_name="y", + verbosity=0); +EvoTrees.importance(m1) + +m2 = fit_evotree(config, + df; + target_name="y", + verbosity=0); +EvoTrees.importance(m2) + +################################# +# Matrix API +################################# +x_train = Matrix(mat[1:9, :]') +y_train = mat[10, :] + +config = EvoTreeRegressor() +m1 = fit_evotree(config; + x_train, + y_train, + verbosity=0); +EvoTrees.importance(m1) + +m2 = fit_evotree(config; + x_train, + y_train, + verbosity=0); +EvoTrees.importance(m2) + +using GLM +x_train = Matrix(mat[1:9, :]') +y_train = mat[10, :] +lm(x_train, y_train) + +################################# +# Matrix debug API +################################# +x_train = Matrix(mat[1:9, :]') +y_train = mat[10, :] + +config = EvoTreeRegressor() +m1, cache1 = EvoTrees.init(config, x_train, y_train); +EvoTrees.grow_evotree!(m1, cache1, config, EvoTrees.CPU) +EvoTrees.importance(m1) + +m2, cache2 = EvoTrees.init(config, x_train, y_train); +EvoTrees.grow_evotree!(m2, cache2, config, EvoTrees.CPU) +EvoTrees.importance(m2) + +using MLJ +using EvoTrees +using MLJLinearModels +X, y = make_regression() +model = Stack( + metalearner=LinearRegressor(), + resampling=CV(nfolds=2), + tree=EvoTreeRegressor() +) +mach = machine(model, X, y) +fit!(mach) diff --git a/src/fit-utils.jl b/src/fit-utils.jl index 7f814294..2931ee1d 100644 --- a/src/fit-utils.jl +++ b/src/fit-utils.jl @@ -6,7 +6,7 @@ Get the braking points of the feature data. """ function get_edges(X::AbstractMatrix{T}; fnames, nbins, rng=Random.TaskLocalRNG()) where {T} nobs = min(size(X, 1), 1000 * nbins) - idx = rand(rng, 1:size(X, 1), nobs) + idx = sample(rng, 1:size(X, 1), nobs, replace=false, ordered=true) nfeats = size(X, 2) edges = Vector{Vector{T}}(undef, nfeats) featbins = Vector{UInt8}(undef, nfeats) @@ -25,7 +25,7 @@ end function get_edges(df; fnames, nbins, rng=Random.TaskLocalRNG()) _nobs = length(Tables.getcolumn(df, 1)) nobs = min(_nobs, 1000 * nbins) - idx = rand(rng, 1:_nobs, nobs) + idx = sample(rng, 1:_nobs, nobs, replace=false, ordered=true) edges = Vector{Any}([Vector{eltype(Tables.getcolumn(df, col))}() for col in fnames]) nfeats = length(fnames) featbins = Vector{UInt8}(undef, nfeats) diff --git a/src/models.jl b/src/models.jl index 74495393..39a5eb04 100644 --- a/src/models.jl +++ b/src/models.jl @@ -61,15 +61,15 @@ function EvoTreeRegressor(; kwargs...) # defaults arguments args = Dict{Symbol,Any}( :loss => :mse, - :nrounds => 10, + :nrounds => 100, :lambda => 0.0, :gamma => 0.0, # min gain to split :eta => 0.1, # learning rate - :max_depth => 5, + :max_depth => 6, :min_weight => 1.0, # minimal weight, different from xgboost (but same for linear) :rowsample => 1.0, :colsample => 1.0, - :nbins => 32, + :nbins => 64, :alpha => 0.5, :monotone_constraints => Dict{Int,Int}(), :tree_type => "binary", @@ -151,15 +151,15 @@ function EvoTreeCount(; kwargs...) # defaults arguments args = Dict{Symbol,Any}( - :nrounds => 10, + :nrounds => 100, :lambda => 0.0, :gamma => 0.0, # min gain to split :eta => 0.1, # learning rate - :max_depth => 5, + :max_depth => 6, :min_weight => 1.0, # minimal weight, different from xgboost (but same for linear) :rowsample => 1.0, :colsample => 1.0, - :nbins => 32, + :nbins => 64, :alpha => 0.5, :monotone_constraints => Dict{Int,Int}(), :tree_type => "binary", @@ -217,15 +217,15 @@ function EvoTreeClassifier(; kwargs...) # defaults arguments args = Dict{Symbol,Any}( - :nrounds => 10, + :nrounds => 100, :lambda => 0.0, :gamma => 0.0, # min gain to split :eta => 0.1, # learning rate - :max_depth => 5, + :max_depth => 6, :min_weight => 1.0, # minimal weight, different from xgboost (but same for linear) :rowsample => 1.0, :colsample => 1.0, - :nbins => 32, + :nbins => 64, :alpha => 0.5, :tree_type => "binary", :rng => 123, @@ -283,15 +283,15 @@ function EvoTreeMLE(; kwargs...) # defaults arguments args = Dict{Symbol,Any}( :loss => :gaussian_mle, - :nrounds => 10, + :nrounds => 100, :lambda => 0.0, :gamma => 0.0, # min gain to split :eta => 0.1, # learning rate - :max_depth => 5, + :max_depth => 6, :min_weight => 8.0, # minimal weight, different from xgboost (but same for linear) :rowsample => 1.0, :colsample => 1.0, - :nbins => 32, + :nbins => 64, :alpha => 0.5, :monotone_constraints => Dict{Int,Int}(), :tree_type => "binary", @@ -366,15 +366,15 @@ function EvoTreeGaussian(; kwargs...) # defaults arguments args = Dict{Symbol,Any}( - :nrounds => 10, + :nrounds => 100, :lambda => 0.0, :gamma => 0.0, # min gain to split :eta => 0.1, # learning rate - :max_depth => 5, + :max_depth => 6, :min_weight => 8.0, # minimal weight, different from xgboost (but same for linear) :rowsample => 1.0, :colsample => 1.0, - :nbins => 32, + :nbins => 64, :alpha => 0.5, :monotone_constraints => Dict{Int,Int}(), :tree_type => "binary",