https://juntian.me/AMA/

[utterances-bot]

🙋 Ask Me Anything · Jun Tian

https://juntian.me/AMA/

[martinmestre]

Hi Jun Tian, thanks for your explanations. I am trying to use pmap, but there is a problem with my packages not being visible to the workers. This is my code:

"""Perform optimization for any fermion mass (ϵ) by running in parallel
for a grid in a global region of paramter space.
Using Distributed.jl
"""

using Pkg
Pkg.activate(".")
@everywhere begin
    using PyCall
    using Optimization, OptimizationNOMAD
    using FiniteDiff
    using DelimitedFiles
    using CSV
    using DataFrames, DataFramesMeta
    using Distributed
end
# %%
@everywhere begin
pushfirst!(PyVector(pyimport("sys")."path"), "")
importLib = pyimport("importlib")
stream = pyimport("stream")
potentials = pyimport("potential_classes")
u = pyimport("astropy.units")
importLib.reload(stream)
importLib.reload(potentials)
end
# %%

println("Threads=", Threads.nthreads())

"""Loop in ϵ."""

"""Anti-normalization function."""
@everywhere function back_orig(x, a, b)
    return (b-a).*x + a
end

"""χ² wrap."""
@everywhere function χ²Full(x, p)
    m = p[1]
    ic = p[2]
    r☼ = p[3]
    lb = p[4]
    ub = p[5]
    θ, ω, β = back_orig(x, lb, ub)
    @show  θ, ω, β
    return stream.chi2_full(θ, ω, β, m, ic, r☼)
end


"""Worker function."""
@everywhere function worker(m, ic, r☼, lb, ub)
    len = length(lb)
    x₀ = 0.5*ones(len)
    p = (m, ic, r☼, lb, ub)
    prob = OptimizationProblem(χ²Full, x₀, p, lb=zeros(len), ub=ones(len))
    println("prob=$prob")
    sol = Optimization.solve(prob, NOMADOpt(); maxiters=2)
    return sol
end

"""Build grid."""
function build_grid(lb_g, ub_g, n_grid)
    n_full = n_grid^3
    lb_a = Vector{Vector{Float64}}(undef,n_full)
    ub_a = Vector{Vector{Float64}}(undef,n_full)
    x₀_a = Vector{Vector{Float64}}(undef,n_full)
    c₁ = collect(range(lb_g[1], ub_g[1], n_grid+1))
    c₂ = collect(range(lb_g[2], ub_g[2], n_grid+1))
    c₃ = collect(range(lb_g[3], ub_g[3], n_grid+1))
    for i ∈ 1:n_grid
        for j ∈ 1:n_grid
            for k ∈ 1:n_grid
                n = (i-1)*n_grid^2+(j-1)*n_grid+k
                lb_a[n] = [c₁[i], c₂[j], c₃[k]]
                ub_a[n] = [c₁[i+1], c₂[j+1], c₃[k+1]]
                x₀_a[n] = 0.5*(lb_a[n]+ub_a[n])
            end
        end
    end
    return lb_a, ub_a, x₀_a
end

"""Parallel function."""
function cooperative(m, ic, r☼, lb_g, ub_g, n_grid)
    lb_a, ub_a, x₀_a = build_grid(lb_g, ub_g, n_grid)
    pars(i) = [m, ic, r☼, lb_a[i], ub_a[i]]
    @show pars(i)
    # res = pmap(i->worker(pars(i)...), eachindex(x₀_a))
    return 1.0
end
# %%

"""Initial orbit conditions file."""

const ic_file = "param_fit_orbit_from_IbataPolysGaiaDR2-data_fixedpot.txt"
const ic = readdlm(ic_file)

"""Metaparameters."""
const m = 300.0
const sol_file = "param_optim_pot_m$(Int(m)).txt"
const r☼ = 8.122
const lb_g = [35., 25., 1.e-5]
const ub_g = [45., 31., 0.005]
const maxiters = 1000
const n_grid = 2
@show m sol_file r☼ maxiters

"""Running."""
lb_l = [40., 29., 0.001]
ub_l = [41., 30., 0.002]
# sol = worker(m, ic, r☼, lb_l, ub_l)
sol = cooperative(m, ic, r☼, lb_g, ub_g, n_grid)

@show sol
writedlm(sol_file, back_orig(sol.u, lb_l, ub_l))
# %%

And the output says:

julia> include("optim_pot.jl")
  Activating project at `/mnt/md1/mmestre/work/2020/halostream/sources/stream_fit/pipeline_paper`
ERROR: LoadError: On worker 2:
ArgumentError: Package PyCall not found in current path.
- Run `import Pkg; Pkg.add("PyCall")` to install the PyCall package.
Stacktrace:
  [1] macro expansion
    @ ./loading.jl:1595 [inlined]
  [2] macro expansion
    @ ./lock.jl:267 [inlined]
  [3] require
    @ ./loading.jl:1576
  [4] top-level scope
    @ /mnt/md1/mmestre/work/2020/halostream/sources/stream_fit/pipeline_paper/optim_pot.jl:9
  [5] eval
    @ ./boot.jl:370
  [6] #invokelatest#2
    @ ./essentials.jl:816
  [7] invokelatest
    @ ./essentials.jl:813
  [8] #114
    @ ~/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/process_messages.jl:301
  [9] run_work_thunk
    @ ~/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/process_messages.jl:70
 [10] run_work_thunk
    @ ~/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/process_messages.jl:79
 [11] #100
    @ ./task.jl:514

...and 7 more exceptions.

Stacktrace:
 [1] sync_end(c::Channel{Any})
   @ Base ./task.jl:445
 [2] macro expansion
   @ ./task.jl:477 [inlined]
 [3] remotecall_eval(m::Module, procs::Vector{Int64}, ex::Expr)
   @ Distributed ~/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/macros.jl:219
 [4] top-level scope
   @ ~/.julia/juliaup/julia-1.9.0+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/macros.jl:203
 [5] include(fname::String)
   @ Base.MainInclude ./client.jl:478
 [6] top-level scope
   @ REPL[3]:1
in expression starting at /mnt/md1/mmestre/work/2020/halostream/sources/stream_fit/pipeline_paper/optim_pot.jl:8

I would really appreciate any help.
Thanks.
Best.

[findmyway]

It seems you are not passing the current active project environment to workers. How do you add workers? I can't find it based on your description above.