From d1da8358121d148f15a31ecdfa09dcfab1bcff3c Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 13 Aug 2024 01:23:33 +0100 Subject: [PATCH] progress printing --- src/Artifacts.jl | 16 ++++----- src/Operations.jl | 87 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 9620a6b2e8..799ce85bb9 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -395,32 +395,30 @@ function ensure_artifact_installed(name::String, artifacts_toml::String; pkg_uuid::Union{Base.UUID,Nothing}=nothing, verbose::Bool = false, quiet_download::Bool = false, + progress::Union{Function,Nothing} = nothing, io::IO=stderr_f()) meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform) if meta === nothing error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'") end - return ensure_artifact_installed(name, meta, artifacts_toml; platform=platform, - verbose=verbose, quiet_download=quiet_download, io=io) + return ensure_artifact_installed(name, meta, artifacts_toml; + platform, verbose, quiet_download, progress, io) end function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String; platform::AbstractPlatform = HostPlatform(), verbose::Bool = false, quiet_download::Bool = false, - progress_callback::Union{Function,Nothing} = nothing, - io::IO=stderr_f(), - progress::Union{Function,Nothing}=nothing) + progress::Union{Function,Nothing} = nothing, + io::IO=stderr_f()) hash = SHA1(meta["git-tree-sha1"]) if !artifact_exists(hash) - if isnothing(progress_callback) || verbose == true + if isnothing(progress) || verbose == true return try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, verbose, quiet_download, io) else - return Threads.@spawn begin - try_artifact_download_sources(name, meta, artifacts_toml; platform, quiet_download=true, io, progress) - end + return try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, quiet_download=true, io, progress) end else return artifact_path(hash) diff --git a/src/Operations.jl b/src/Operations.jl index 1a24e4b178..1173b122e9 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -830,16 +830,41 @@ function download_artifacts(ctx::Context; end push!(pkg_roots, dirname(env.project_file)) used_artifact_tomls = Set{String}() - download_jobs = Channel{Function}(Inf) - # download_progresses = # Dict for each live job of (total, now) -> nothing + download_jobs = Channel{Function}(Inf) # ctx.num_concurrent_downloads + + download_states = Dict{String, Tuple{Bool,MiniProgressBar}}() + is_done = false + ansi_moveup(n::Int) = string("\e[", n, "A") + ansi_movecol1 = "\e[1G" + ansi_cleartoend = "\e[0J" + ansi_cleartoendofline = "\e[0K" + ansi_enablecursor = "\e[?25h" + ansi_disablecursor = "\e[?25l" + termwidth = displaysize(io)[2] + + longest_name = 0 + for pkg_root in pkg_roots + for (artifacts_toml, artifacts) in collect_artifacts(pkg_root; platform) + for name in keys(artifacts) + longest_name = max(longest_name, textwidth(name)) + end + end + end + for pkg_root in pkg_roots for (artifacts_toml, artifacts) in collect_artifacts(pkg_root; platform) # For each Artifacts.toml, install each artifact we've collected from it for name in keys(artifacts) - # @info name + is_done && break + bar = MiniProgressBar(; indent=2, header = rpad(name, longest_name), color = Base.info_color(), percentage=true, always_reprint=true) + progress = (total, current) -> (bar.max = total; bar.current = current) + download_states[name] = (true, bar) push!(download_jobs, - () -> ensure_artifact_installed(name, artifacts[name], artifacts_toml; - verbose, quiet_download=!(usable_io(io)), io=io) + () -> begin + ensure_artifact_installed(name, artifacts[name], artifacts_toml; + verbose, quiet_download=!(usable_io(io)), io, progress) + download_states[name] = (false, bar) + end ) end push!(used_artifact_tomls, artifacts_toml) @@ -847,12 +872,54 @@ function download_artifacts(ctx::Context; end close(download_jobs) - sema = Base.Semaphore(20) - # TODO: figure out printing & error handling - @sync for f in download_jobs - Threads.@spawn Base.acquire(sema) do - f() + @sync begin + t_print = Threads.@spawn begin + try + first = true + timer = Timer(0, interval=1/10) + main_bar = MiniProgressBar(; indent=0, header = "Downloading artifacts", color = :green, percentage=false, always_reprint=true) + main_bar.max = length(download_states) + while !is_done + main_bar.current = length(filter(x -> !x[2][1], download_states)) + str = sprint(context=io) do iostr + first || print(iostr, ansi_cleartoend) + n_printed = 1 + show_progress(iostr, main_bar; termwidth, carriagereturn=false) + println(iostr) + for (name, (running, bar)) in download_states + running || continue + show_progress(iostr, bar; termwidth, carriagereturn=false) + println(iostr) + n_printed += 1 + end + is_done || print(iostr, ansi_moveup(n_printed), ansi_movecol1) + first = false + end + print(io, str) + wait(timer) + end + catch e + e isa InterruptException || rethrow() + is_done = true + end + end + Base.errormonitor(t_print) + + # TODO: figure out error handling/reporting + sema = Base.Semaphore(ctx.num_concurrent_downloads) + @sync for f in download_jobs + is_done && break + t = Threads.@spawn begin + try + Base.acquire(f, sema) + catch e + e isa InterruptException || rethrow() + is_done = true + end + end + Base.errormonitor(t) end + is_done = true end for f in used_artifact_tomls