From 9b4c22e344940ae564c06950e530ec10de022c70 Mon Sep 17 00:00:00 2001 From: Jaakko Ruohio Date: Mon, 18 Feb 2019 09:21:16 -0600 Subject: [PATCH 1/2] Fix last copy and expunge cruft --- .travis.yml | 12 +++++++----- REQUIRE | 5 ++--- src/MAT.jl | 7 ++----- src/MAT_HDF5.jl | 44 ++++++++++++++++++++------------------------ src/MAT_v5.jl | 35 +++++++++++++++++------------------ test/read.jl | 12 +++--------- test/runtests.jl | 4 +--- test/write.jl | 10 ++++++---- 8 files changed, 58 insertions(+), 71 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84b4b01..8aeb2e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,16 @@ os: - linux - osx julia: - - 0.6 - - 0.7 - 1.0 + - 1.1 - nightly +addons: +# apt: +# packages: +# - hdf5-tools +before_script: +# - if [ $TRAVIS_OS_NAME = osx ]; then brew install gcc; brew link --overwrite gcc; brew install hdf5; fi notifications: email: false -script: - - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - - julia -e 'Pkg.clone(pwd()); Pkg.build("MAT"); Pkg.test("MAT"; coverage=true)' after_success: - julia -e 'cd(Pkg.dir("MAT")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' diff --git a/REQUIRE b/REQUIRE index 264fa34..d6efe21 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,4 @@ -julia 0.6 -HDF5 +julia 0.7 +HDF5 0.11.0 BufferedStreams 0.2.0 Libz -Compat 1.0 diff --git a/src/MAT.jl b/src/MAT.jl index 5baac22..57d254b 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -22,12 +22,9 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -VERSION >= v"0.4.0-dev+6521" && __precompile__() - module MAT -using HDF5, Compat -using Compat.SparseArrays +using HDF5, SparseArrays include("MAT_HDF5.jl") include("MAT_v5.jl") @@ -144,7 +141,7 @@ end Write a dictionary containing variable names as keys and values as values to a Matlab file, opening and closing it automatically. """ -function matwrite(filename::AbstractString, dict::AbstractDict{S, T}) where {S,T} +function matwrite(filename::AbstractString, dict::AbstractDict{S, T}) where {S, T} file = matopen(filename, "w") try for (k, v) in dict diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 952152c..22f1efa 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -28,15 +28,7 @@ module MAT_HDF5 -using HDF5 -using Compat -using Compat.SparseArrays - -@static if VERSION < v"0.7-" - _finalizer(f, x) = finalizer(x, f) -else - _finalizer = finalizer -end +using HDF5, SparseArrays import Base: read, write, close import HDF5: names, exists, HDF5ReferenceObj, HDF5BitsKind @@ -53,7 +45,7 @@ mutable struct MatlabHDF5File <: HDF5.DataFile function MatlabHDF5File(plain, toclose::Bool=true, writeheader::Bool=false, refcounter::Int=0) f = new(plain, toclose, writeheader, refcounter) if toclose - _finalizer(close, f) + finalizer(close, f) end f end @@ -70,7 +62,11 @@ function close(f::MatlabHDF5File) if f.writeheader magic = zeros(UInt8, 512) identifier = "MATLAB 7.3 MAT-file" # minimal but sufficient - magic[1:length(identifier)] = Vector{UInt8}(identifier) + GC.@preserve magic identifier begin + magicptr = pointer(magic) + idptr = pointer(identifier) + unsafe_copyto!(magicptr, idptr, length(identifier)) + end magic[126] = 0x02 magic[127] = 0x49 magic[128] = 0x4d @@ -123,7 +119,7 @@ const sparse_attr_matlab = "MATLAB_sparse" const int_decode_attr_matlab = "MATLAB_int_decode" ### Reading -function read_complex(dtype::HDF5Datatype, dset::HDF5Dataset, ::Type{Array{T}}) where {T} +function read_complex(dtype::HDF5Datatype, dset::HDF5Dataset, ::Type{Array{T}}) where T if !check_datatype_complex(dtype) close(dtype) error("Unrecognized compound data type when reading ", name(dset)) @@ -301,7 +297,7 @@ toarray(x::Bool) = UInt8[x] toarray(x) = [x] # Write the MATLAB type string for dset -m_writetypeattr(dset, ::Type{Complex{T}}) where {T} = m_writetypeattr(dset, T) +m_writetypeattr(dset, ::Type{Complex{T}}) where T = m_writetypeattr(dset, T) function m_writetypeattr(dset, T) if !haskey(type2str_matlab, T) error("Type ", T, " is not (yet) supported") @@ -316,7 +312,7 @@ function m_writetypeattr(dset, T) end # Writes an empty scalar or array -function m_writeempty(parent::HDF5Parent, name::String, data::Array) +function m_writeempty(parent::HDF5Parent, name::String, data::AbstractArray) adata = [size(data)...] dset, dtype = d_create(parent, name, adata) try @@ -330,7 +326,7 @@ function m_writeempty(parent::HDF5Parent, name::String, data::Array) end # Write an array to a dataset in a MATLAB file, returning the dataset -function m_writearray(parent::HDF5Parent, name::String, adata::Array{T}) where {T<:HDF5BitsOrBool} +function m_writearray(parent::HDF5Parent, name::String, adata::AbstractArray{T}) where {T<:HDF5BitsOrBool} dset, dtype = d_create(parent, name, adata) try HDF5.writearray(dset, dtype.id, adata) @@ -342,14 +338,14 @@ function m_writearray(parent::HDF5Parent, name::String, adata::Array{T}) where { close(dtype) end end -function m_writearray(parent::HDF5Parent, name::String, adata::Array{Complex{T}}) where {T<:HDF5BitsOrBool} +function m_writearray(parent::HDF5Parent, name::String, adata::AbstractArray{Complex{T}}) where {T<:HDF5BitsOrBool} dtype = build_datatype_complex(T) try stype = dataspace(adata) obj_id = HDF5.h5d_create(parent.id, name, dtype.id, stype.id) dset = HDF5Dataset(obj_id, file(parent)) try - arr = reinterpret(T, adata, tuple(2, size(adata)...)) + arr = reshape(reinterpret(T, adata), tuple(2, size(adata)...)) HDF5.writearray(dset, dtype.id, arr) catch e close(dset) @@ -378,16 +374,16 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data:: end # Write sparse arrays -function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::SparseMatrixCSC{T}) where {T} +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::SparseMatrixCSC{T}) where T g = g_create(parent, name) try m_writetypeattr(g, T) a_write(g, sparse_attr_matlab, UInt64(size(data, 1))) if !isempty(data.nzval) close(m_writearray(g, "data", toarray(data.nzval))) - close(m_writearray(g, "ir", add!(isa(data.rowval, Vector{UInt64}) ? copy(data.rowval) : convert(Vector{UInt64}, data.rowval), reinterpret(UInt64, convert(Int64, -1))))) + close(m_writearray(g, "ir", add!(isa(data.rowval, Vector{UInt64}) ? copy(data.rowval) : convert(Vector{UInt64}, data.rowval), typemax(UInt64)))) end - close(m_writearray(g, "jc", add!(isa(data.colptr, Vector{UInt64}) ? copy(data.colptr) : convert(Vector{UInt64}, data.colptr), reinterpret(UInt64, convert(Int64, -1))))) + close(m_writearray(g, "jc", add!(isa(data.colptr, Vector{UInt64}) ? copy(data.colptr) : convert(Vector{UInt64}, data.colptr), typemax(UInt64)))) finally close(g) end @@ -431,7 +427,7 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, str::A end # Write cell arrays -function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Array{T}) where {T} +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Array{T}) where T pathrefs = "/#refs#" fid = file(parent) local g @@ -538,7 +534,7 @@ end ## Type conversion operations ## -struct MatlabString; end +struct MatlabString end const str2type_matlab = Dict( "canonical empty" => nothing, @@ -607,14 +603,14 @@ function read(obj::HDF5Object, ::Type{Bool}) tf = read(obj, UInt8) tf > 0 end -function read(obj::HDF5Object, ::Type{Array{Bool}}) +function read(obj::HDF5Dataset, ::Type{Array{Bool}}) if HDF5.isnull(obj) return Bool[] end # Use the low-level HDF5 API to put the data directly into a Bool array tf = Array{Bool}(undef, size(obj)) HDF5.h5d_read(obj.id, HDF5.hdf5_type_id(UInt8), tf, obj.xfer) - tf + return tf end ## Utilities for handling complex numbers diff --git a/src/MAT_v5.jl b/src/MAT_v5.jl index a449146..789affb 100644 --- a/src/MAT_v5.jl +++ b/src/MAT_v5.jl @@ -26,18 +26,12 @@ # http://www.mathworks.com/help/pdf_doc/matlab/matfile_format.pdf module MAT_v5 -using Libz, BufferedStreams, HDF5, Compat -using Compat.SparseArrays +using Libz, BufferedStreams, HDF5, SparseArrays import Base: read, write, close import HDF5: names, exists -if VERSION < v"0.6.0-dev.1632" - round_uint8(data) = round(UInt8, data) - complex_array(a, b) = complex(a, b) -else - round_uint8(data) = round.(UInt8, data) - complex_array(a, b) = complex.(a, b) -end +round_uint8(data) = round.(UInt8, data) +complex_array(a, b) = complex.(a, b) mutable struct Matlabv5File <: HDF5.DataFile ios::IOStream @@ -85,9 +79,9 @@ const CONVERT_TYPES = Type[ Union{}, Float64, Float32, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64] -read_bswap(f::IO, swap_bytes::Bool, ::Type{T}) where {T} = +read_bswap(f::IO, swap_bytes::Bool, ::Type{T}) where T = swap_bytes ? bswap(read(f, T)) : read(f, T) -function read_bswap(f::IO, swap_bytes::Bool, ::Type{T}, dim::Union{Int, Tuple{Vararg{Int}}}) where {T} +function read_bswap(f::IO, swap_bytes::Bool, ::Type{T}, dim::Union{Int, Tuple{Vararg{Int}}}) where T d = read!(f, Array{T}(undef, dim)) if swap_bytes for i = 1:length(d) @@ -96,6 +90,15 @@ function read_bswap(f::IO, swap_bytes::Bool, ::Type{T}, dim::Union{Int, Tuple{Va end d end +function read_bswap(f::IO, swap_bytes::Bool, d::AbstractArray{T}) where T + readbytes!(f, reinterpret(UInt8, d)) + if swap_bytes + for i = 1:length(d) + @inbounds d[i] = bswap(d[i]) + end + end + d +end skip_padding(f::IO, nbytes::Int, hbytes::Int) = if nbytes % hbytes != 0 skip(f, hbytes-(nbytes % hbytes)) @@ -115,7 +118,7 @@ function read_header(f::IO, swap_bytes::Bool) end # Read data element as a vector of a given type -function read_element(f::IO, swap_bytes::Bool, ::Type{T}) where {T} +function read_element(f::IO, swap_bytes::Bool, ::Type{T}) where T (dtype, nbytes, hbytes) = read_header(f, swap_bytes) data = read_bswap(f, swap_bytes, T, Int(div(nbytes, sizeof(T)))) skip_padding(f, nbytes, hbytes) @@ -134,7 +137,7 @@ end # Read data element as encoded type with given dimensions, converting # to another type if necessary and collapsing one-element matrices to # scalars -function read_data(f::IO, swap_bytes::Bool, ::Type{T}, dimensions::Vector{Int32}) where {T} +function read_data(f::IO, swap_bytes::Bool, ::Type{T}, dimensions::Vector{Int32}) where T (dtype, nbytes, hbytes) = read_header(f, swap_bytes) read_type = READ_TYPES[dtype] if (read_type === UInt8) && (T === Bool) @@ -244,11 +247,7 @@ function read_sparse(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, flags:: SparseMatrixCSC(m, n, jc, ir, pr) end -if VERSION >= v"0.4.0-dev+1039" - truncate_to_uint8(x) = x % UInt8 -else - truncate_to_uint8(x) = convert(UInt8, x) -end +truncate_to_uint8(x) = x % UInt8 function read_string(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}) (dtype, nbytes, hbytes) = read_header(f, swap_bytes) diff --git a/test/read.jl b/test/read.jl index d4a4a97..6e00483 100644 --- a/test/read.jl +++ b/test/read.jl @@ -1,10 +1,4 @@ -using MAT - -@static if VERSION < v"0.7-" - using Base.Test -else - using Test -end +using MAT, Test function check(filename, result) matfile = matopen(filename) @@ -95,7 +89,7 @@ for _format in ["v6", "v7", "v7.3"] "a1x2" => [1.0 2.0], "a2x1" => zeros(2, 1)+[1.0, 2.0], "a2x2" => [1.0 3.0; 4.0 2.0], - "a2x2x2" => Compat.cat([1.0 3.0; 4.0 2.0], [1.0 2.0; 3.0 4.0], dims=3), + "a2x2x2" => cat([1.0 3.0; 4.0 2.0], [1.0 2.0; 3.0 4.0]; dims=3), "empty" => zeros(0, 0), "string" => "string" ) @@ -123,7 +117,7 @@ for _format in ["v6", "v7", "v7.3"] true false false false true false true false false - ] + ] ) check("logical.mat", result) diff --git a/test/runtests.jl b/test/runtests.jl index 3cf30b4..5b9bec1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,4 @@ -using Compat -using Compat.SparseArrays -using Compat.LinearAlgebra +using SparseArrays, LinearAlgebra include("read.jl") include("write.jl") diff --git a/test/write.jl b/test/write.jl index 8d52cdd..3b8c910 100644 --- a/test/write.jl +++ b/test/write.jl @@ -1,6 +1,6 @@ using MAT -tmpfile = string(tempname, ".mat") +tmpfile = string(tempname(), ".mat") function test_write(data) matwrite(tmpfile, data) @@ -33,6 +33,8 @@ test_write(Dict( )) test_write(Dict( + "ComplexInt" => Complex{Int}[1 -1 1+1im 1-1im -1+1im -1-1im 1im], + "ComplexF32" => ComplexF32[1.0 -1.0 1.0+1.0im 1.0-1.0im -1.0+1.0im -1.0-1.0im 1.0im], "ComplexF64" => [1.0 -1.0 1.0+1.0im 1.0-1.0im -1.0+1.0im -1.0-1.0im 1.0im], "ComplexPair" => [1 2-3im 4+5im] )) @@ -50,7 +52,7 @@ test_write(Dict( "a1x2" => [1.0 2.0], "a2x1" => zeros(2, 1)+[1.0, 2.0], "a2x2" => [1.0 3.0; 4.0 2.0], - "a2x2x2" => Compat.cat([1.0 3.0; 4.0 2.0], [1.0 2.0; 3.0 4.0], dims=3), + "a2x2x2" => cat([1.0 3.0; 4.0 2.0], [1.0 2.0; 3.0 4.0], dims=3), "empty" => zeros(0, 0), "string" => "string" )) @@ -81,7 +83,7 @@ test_write(Dict( @test_throws ErrorException test_write(Dict("another invalid key" => "invalid characters")) @test_throws ErrorException test_write(Dict("yetanotherinvalidkeyyetanotherinvalidkeyyetanotherinvalidkeyyetanotherinvalidkey" => "too long")) -type TestCompositeKind +struct TestCompositeKind field1::AbstractString end fid = matopen(tmpfile, "w") @@ -103,6 +105,6 @@ sd = SortedDict(Dict( "ComplexF64" => [1.0 -1.0 1.0+1.0im 1.0-1.0im -1.0+1.0im -1.0-1.0im 1.0im], "simple_string" => "the quick brown fox", "a1x2" => [1.0 2.0], - "sparse_empty" => sparse(Matrix{Float64}(0, 0)) + "sparse_empty" => sparse(Matrix{Float64}(undef, 0, 0)) )) test_write(sd) From c21fc9a0bd7fcab36d928f3efe81b8d01c022e65 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 18 Feb 2019 09:22:21 -0600 Subject: [PATCH 2/2] Test 0.7 on Travis if we're going to support it --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8aeb2e3..805a13c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ os: - linux - osx julia: + - 0.7 - 1.0 - 1.1 - nightly