diff --git a/.gitignore b/.gitignore index 99e2cea..ebbe1d6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.jl.cov *.jl.mem .DS_Store +.ipynb_checkpoints/ /Manifest.toml /dev/ /docs/build/ diff --git a/Project.toml b/Project.toml index dfe4817..c6ad79b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "UrlDownload" uuid = "856ac37a-3032-4c1c-9122-f86d88358c8b" authors = ["Andrey Oskin"] -version = "0.2.0" +version = "0.2.1" [deps] HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" diff --git a/src/UrlDownload.jl b/src/UrlDownload.jl index dbf771d..3dc5ddd 100644 --- a/src/UrlDownload.jl +++ b/src/UrlDownload.jl @@ -26,11 +26,11 @@ const sym2func = Dict( ) const Compressor = Dict( - :gzip => (lib = :CodecZlib, stream = :GzipDecompressorStream), - :zstd => (lib = :CodecZstd, stream = :ZstdDecompressorStream), - :xz => (lib = :CodecXz, stream = :XzDecompressorStream), - :lz4 => (lib = :CodecLz4, stream = :LZ4FrameDecompressorStream), - :bzip2 => (lib = :CodecBzip2, stream = :Bzip2DecompressorStream) + :gzip => (lib = :CodecZlib, stream = :GzipDecompressorStream, transcode = :GzipDecompressor), + :zstd => (lib = :CodecZstd, stream = :ZstdDecompressorStream, transcode = :ZstdDecompressor), + :xz => (lib = :CodecXz, stream = :XzDecompressorStream, transcode = :XzDecompressor), + :lz4 => (lib = :CodecLz4, stream = :LZ4FrameDecompressorStream, transcode = :LZ4FrameDecompressor), + :bzip2 => (lib = :CodecBzip2, stream = :Bzip2DecompressorStream, transcode = :Bzip2Decompressor) ) function load_feather(buf; kw...) @@ -186,8 +186,9 @@ function urldownload(url, progress = false; eof(stream) && return # nothing to process yet body = UInt8[] - total_bytes = Int(floor(parse(Float64, HTTP.header(resp, "Content-Length", "NaN")))) if progress + total_bytes = floor(parse(Float64, HTTP.header(resp, "Content-Length", "NaN"))) + total_bytes = isnan(total_bytes) ? typemax(Int) : Int(total_bytes) p = Progress(total_bytes, update_period) end @@ -229,7 +230,10 @@ function urldownload(url, progress = false; csvlibfile = getfield(csvlib, :File) return Base.invokelatest(csvlibfile, Base.invokelatest(stream, IOBuffer(body)); kw...) else - return parser(body; kw...) + lib = checked_import(Compressor[compress].lib) + transcoder = getfield(lib, Compressor[compress].transcode) + data = Base.invokelatest(transcode, transcoder, body) + return parser(data; kw...) end else error("Unknown compress format: $compress")