parallelize artifact downloads

src/API.jl

@@ -1268,7 +1268,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
     # Install all packages
     new_apply = Operations.download_source(ctx)
     # Install all artifacts
-    Operations.download_artifacts(ctx.env; platform, verbose, io=ctx.io)
+    Operations.download_artifacts(ctx; platform, verbose)
     # Run build scripts
     allow_build && Operations.build_versions(ctx, union(new_apply, new_git); verbose=verbose)
 

src/Artifacts.jl

@@ -301,6 +301,7 @@ function download_artifact(
     verbose::Bool = false,
     quiet_download::Bool = false,
     io::IO=stderr_f(),
+    progress::Union{Function, Nothing} = nothing,
 )
     if artifact_exists(tree_hash)
         return true
@@ -323,8 +324,8 @@ function download_artifact(
     temp_dir = mktempdir(artifacts_dir)
 
     try
-        download_verify_unpack(tarball_url, tarball_hash, temp_dir, ignore_existence=true, verbose=verbose,
-                quiet_download=quiet_download, io=io)
+        download_verify_unpack(tarball_url, tarball_hash, temp_dir;
+                                ignore_existence=true, verbose, quiet_download, io, progress)
         calc_hash = SHA1(GitTools.tree_hash(temp_dir))
 
         # Did we get what we expected?  If not, freak out.
@@ -394,82 +395,100 @@ function ensure_artifact_installed(name::String, artifacts_toml::String;
                                    pkg_uuid::Union{Base.UUID,Nothing}=nothing,
                                    verbose::Bool = false,
                                    quiet_download::Bool = false,
+                                   progress::Union{Function,Nothing} = nothing,
                                    io::IO=stderr_f())
     meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform)
     if meta === nothing
         error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'")
     end
 
-    return ensure_artifact_installed(name, meta, artifacts_toml; platform=platform,
-                                     verbose=verbose, quiet_download=quiet_download, io=io)
+    return ensure_artifact_installed(name, meta, artifacts_toml;
+                                    platform, verbose, quiet_download, progress, io)
 end
 
 function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String;
                                    platform::AbstractPlatform = HostPlatform(),
                                    verbose::Bool = false,
                                    quiet_download::Bool = false,
+                                   progress::Union{Function,Nothing} = nothing,
                                    io::IO=stderr_f())
-    hash = SHA1(meta["git-tree-sha1"])
 
+    hash = SHA1(meta["git-tree-sha1"])
     if !artifact_exists(hash)
-        errors = Any[]
-        # first try downloading from Pkg server
-        # TODO: only do this if Pkg server knows about this package
-        if (server = pkg_server()) !== nothing
-            url = "$server/artifact/$hash"
-            download_success = let url=url
-                @debug "Downloading artifact from Pkg server" name artifacts_toml platform url
-                with_show_download_info(io, name, quiet_download) do
-                    download_artifact(hash, url; verbose=verbose, quiet_download=quiet_download, io=io)
-                end
-            end
-            # download_success is either `true` or an error object
-            if download_success === true
-                return artifact_path(hash)
-            else
-                @debug "Failed to download artifact from Pkg server" download_success
-                push!(errors, (url, download_success))
-            end
+        if isnothing(progress) || verbose == true
+            return try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, verbose, quiet_download, io)
+        else
+            return () -> try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, quiet_download=true, io, progress)
         end
+    else
+        return artifact_path(hash)
+    end
+end
 
-        # If this artifact does not exist on-disk already, ensure it has download
-        # information, then download it!
-        if !haskey(meta, "download")
-            error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'")
+function try_artifact_download_sources(
+            name::String, hash::SHA1, meta::Dict, artifacts_toml::String;
+            platform::AbstractPlatform=HostPlatform(),
+            verbose::Bool=false,
+            quiet_download::Bool=false,
+            io::IO=stderr_f(),
+            progress::Union{Function,Nothing}=nothing)
+
+    errors = Any[]
+    # first try downloading from Pkg server
+    # TODO: only do this if Pkg server knows about this package
+    if (server = pkg_server()) !== nothing
+        url = "$server/artifact/$hash"
+        download_success = let url = url
+            @debug "Downloading artifact from Pkg server" name artifacts_toml platform url
+            with_show_download_info(io, name, quiet_download) do
+                download_artifact(hash, url; verbose, quiet_download, io, progress)
+            end
         end
+        # download_success is either `true` or an error object
+        if download_success === true
+            return artifact_path(hash)
+        else
+            @debug "Failed to download artifact from Pkg server" download_success
+            push!(errors, (url, download_success))
+        end
+    end
 
-        # Attempt to download from all sources
-        for entry in meta["download"]
-            url = entry["url"]
-            tarball_hash = entry["sha256"]
-            download_success = let url=url
-                @debug "Downloading artifact" name artifacts_toml platform url
-                with_show_download_info(io, name, quiet_download) do
-                    download_artifact(hash, url, tarball_hash; verbose=verbose, quiet_download=quiet_download, io=io)
-                end
-            end
-            # download_success is either `true` or an error object
-            if download_success === true
-                return artifact_path(hash)
-            else
-                @debug "Failed to download artifact" download_success
-                push!(errors, (url, download_success))
+    # If this artifact does not exist on-disk already, ensure it has download
+    # information, then download it!
+    if !haskey(meta, "download")
+        error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'")
+    end
+
+    # Attempt to download from all sources
+    for entry in meta["download"]
+        url = entry["url"]
+        tarball_hash = entry["sha256"]
+        download_success = let url = url
+            @debug "Downloading artifact" name artifacts_toml platform url
+            with_show_download_info(io, name, quiet_download) do
+                download_artifact(hash, url, tarball_hash; verbose, quiet_download, io, progress)
             end
         end
-        errmsg = """
-        Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'.
-        Sources attempted:
-        """
-        for (url, err) in errors
-            errmsg *= "- $(url)\n"
-            errmsg *= "    Error: $(sprint(showerror, err))\n"
+        # download_success is either `true` or an error object
+        if download_success === true
+            return artifact_path(hash)
+        else
+            @debug "Failed to download artifact" download_success
+            push!(errors, (url, download_success))
         end
-        error(errmsg)
-    else
-        return artifact_path(hash)
     end
+    errmsg = """
+    Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'.
+    Sources attempted:
+    """
+    for (url, err) in errors
+        errmsg *= "- $(url)\n"
+        errmsg *= "    Error: $(sprint(showerror, err))\n"
+    end
+    error(errmsg)
 end
 
+
 function with_show_download_info(f, io, name, quiet_download)
     fancyprint = can_fancyprint(io)
     if !quiet_download
@@ -485,7 +504,7 @@ function with_show_download_info(f, io, name, quiet_download)
         if !quiet_download
             fancyprint && print(io, "\033[1A") # move cursor up one line
             fancyprint && print(io, "\033[2K") # clear line
-            if success 
+            if success
                 fancyprint && printpkgstyle(io, :Downloaded, "artifact: $name")
             else
                 printpkgstyle(io, :Failure, "artifact: $name", color = :red)

src/MiniProgressBars.jl

@@ -9,11 +9,11 @@ Base.@kwdef mutable struct MiniProgressBar
     header::String = ""
     color::Symbol = :nothing
     width::Int = 40
-    current::Int = 0.0
-    prev::Int = 0.0
+    current::Int = 0
+    prev::Int = 0
     has_shown::Bool = false
     time_shown::Float64 = 0.0
-    percentage::Bool = true
+    mode::Symbol = :percentage # :percentage :int :data
     always_reprint::Bool = false
     indent::Int = 4
 end
@@ -47,10 +47,14 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere
     p.prev = p.current
     p.has_shown = true
 
-    progress_text = if p.percentage
+    progress_text = if p.mode == :percentage
         @sprintf "%2.1f %%" perc
-    else
+    elseif p.mode == :int
         string(p.current, "/",  p.max)
+    elseif p.mode == :data
+        string(Base.format_bytes(p.current), "/", Base.format_bytes(p.max))
+    else
+        error("Unknown mode $(p.mode)")
     end
     termwidth = @something termwidth displaysize(io)[2]
     max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width))