From bea378a7f799fa4b35ef5990dd2e3b573e220f61 Mon Sep 17 00:00:00 2001 From: foreverallama Date: Wed, 20 Aug 2025 22:25:18 +0530 Subject: [PATCH 01/20] Add support for loading mxOPAQUE_CLASS types * MAT_subsys.jl: New file MAT_subsys with methods to set, parse and retrieve subsystem data * MAT_v5.jl: New method "read_opaque" to handle mxOPAQUE_CLASS * MAT_v5.jl: New method "read_subsystem" to handle subsystem data * MAT.jl (matread): Update to clear subsystem and object cache after load Support for loading mxOPAQUE_CLASS objects in v7.3 HDF5 format * MAT_HDF5.jl (matopen): New argument Endian indicator, Reads and parses subsystem on load * MAT_HDF5.jl (close): Update to write endian header based on system endianness * MAT_HDF5.jl (m_read::HDF5.Dataset): Update to handle MATLAB_object_decode (mxOPAQUE_CLASS) types * MAT_HDF5.jl (m_read::HDF5.Group): Update to read subsystem data and function_handles * MAT.jl (matopen): Update function calls Updated test for struct_table_datetime.mat to ensure accurate deserialization (including nested properties) in both v7 and v7.3 formats * test/read.jl: Update tests for "function_handles.mat" and "struct_table_datetime.mat" --- src/MAT.jl | 10 +- src/MAT_HDF5.jl | 86 ++++++--- src/MAT_subsys.jl | 297 ++++++++++++++++++++++++++++++ src/MAT_v5.jl | 56 +++++- test/read.jl | 45 ++++- test/v7/struct_table_datetime.mat | Bin 0 -> 1451 bytes 6 files changed, 449 insertions(+), 45 deletions(-) create mode 100644 src/MAT_subsys.jl create mode 100644 test/v7/struct_table_datetime.mat diff --git a/src/MAT.jl b/src/MAT.jl index a11efee..89efdb3 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -26,11 +26,12 @@ module MAT using HDF5, SparseArrays +include("MAT_subsys.jl") include("MAT_HDF5.jl") include("MAT_v5.jl") include("MAT_v4.jl") -using .MAT_HDF5, .MAT_v5, .MAT_v4 +using .MAT_HDF5, .MAT_v5, .MAT_v4, .MAT_subsys export matopen, matread, matwrite, @read, @write @@ -40,7 +41,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo # When creating new files, create as HDF5 by default fs = filesize(filename) if cr && (tr || fs == 0) - return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress) + return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, Base.ENDIAN_BOM == 0x04030201) elseif fs == 0 error("File \"$filename\" does not exist and create was not specified") end @@ -76,7 +77,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo seek(rawfid, offset) if read!(rawfid, Vector{UInt8}(undef, 8)) == HDF5_HEADER close(rawfid) - return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress) + return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, endian_indicator == 0x494D) end end @@ -133,6 +134,7 @@ function matread(filename::AbstractString) try vars = read(file) finally + MAT_subsys.clear_subsys!() close(file) end vars @@ -165,7 +167,7 @@ function matwrite(filename::AbstractString, dict::AbstractDict{S, T}; compress:: end else - + file = matopen(filename, "w"; compress = compress) try for (k, v) in dict diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index b0d36d6..05fdede 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -29,6 +29,7 @@ module MAT_HDF5 using HDF5, SparseArrays +using ..MAT_subsys import Base: names, read, write, close import HDF5: Reference @@ -69,8 +70,13 @@ function close(f::MatlabHDF5File) unsafe_copyto!(magicptr, idptr, length(identifier)) end magic[126] = 0x02 - magic[127] = 0x49 - magic[128] = 0x4d + if Base.ENDIAN_BOM == 0x04030201 + magic[127] = 0x49 + magic[128] = 0x4d + else + magic[127] = 0x4d + magic[128] = 0x49 + end rawfid = open(f.plain.filename, "r+") write(rawfid, magic) close(rawfid) @@ -80,7 +86,7 @@ function close(f::MatlabHDF5File) nothing end -function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool) +function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool, endian_indicator::Bool) local f if ff && !wr error("Cannot append to a read-only file") @@ -109,6 +115,11 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo fid.refcounter = length(g)-1 close(g) end + subsys_refs = "#subsystem#" + if haskey(fid.plain, subsys_refs) + subsys_data = m_read(fid.plain[subsys_refs]) + MAT_subsys.load_subsys!(subsys_data, endian_indicator) + end fid end @@ -118,6 +129,7 @@ const name_type_attr_matlab = "MATLAB_class" const empty_attr_matlab = "MATLAB_empty" const sparse_attr_matlab = "MATLAB_sparse" const int_decode_attr_matlab = "MATLAB_int_decode" +const object_type_attr_matlab = "MATLAB_object_decode" ### Reading function read_complex(dtype::HDF5.Datatype, dset::HDF5.Dataset, ::Type{T}) where T @@ -128,6 +140,21 @@ function read_complex(dtype::HDF5.Datatype, dset::HDF5.Dataset, ::Type{T}) where return read(dset, Complex{T}) end +function read_cell(dset::HDF5.Dataset) + refs = read(dset, Reference) + out = Array{Any}(undef, size(refs)) + f = HDF5.file(dset) + for i = 1:length(refs) + dset = f[refs[i]] + try + out[i] = m_read(dset) + finally + close(dset) + end + end + return out +end + function m_read(dset::HDF5.Dataset) if haskey(dset, empty_attr_matlab) # Empty arrays encode the dimensions as the dataset @@ -150,36 +177,46 @@ function m_read(dset::HDF5.Dataset) end mattype = haskey(dset, name_type_attr_matlab) ? read_attribute(dset, name_type_attr_matlab) : "cell" + objecttype = haskey(dset, object_type_attr_matlab) ? read_attribute(dset, object_type_attr_matlab) : nothing - if mattype == "cell" + if mattype == "cell" && objecttype === nothing # Cell arrays, represented as an array of refs - refs = read(dset, Reference) - out = Array{Any}(undef, size(refs)) - f = HDF5.file(dset) - for i = 1:length(refs) - dset = f[refs[i]] - try - out[i] = m_read(dset) - finally - close(dset) - end + return read_cell(dset) + elseif objecttype !== nothing + if objecttype != 3 + @warn "MATLAB Object Type $mattype is currently not supported." + return missing + end + if mattype == "FileWrapper__" + return read_cell(dset) + end + if haskey(dset, "MATLAB_fields") + @warn "Enumeration Instances are not supported currently." + return missing end - return out elseif !haskey(str2type_matlab,mattype) - @warn "MATLAB $mattype values are currently not supported" + @warn "MATLAB $mattype values are currently not supported." return missing end # Regular arrays of values # Convert to Julia type - T = str2type_matlab[mattype] + if objecttype === nothing + T = str2type_matlab[mattype] + else + T = UInt32 # FIXME: Default for MATLAB objects? + end # Check for a COMPOUND data set, and if so handle complex numbers specially dtype = datatype(dset) try class_id = HDF5.API.h5t_get_class(dtype.id) d = class_id == HDF5.API.H5T_COMPOUND ? read_complex(dtype, dset, T) : read(dset, T) - length(d) == 1 ? d[1] : d + if objecttype !== nothing + return MAT_subsys.load_mcos_object(d, "MCOS") + else + return length(d) == 1 ? d[1] : d + end finally close(dtype) end @@ -194,7 +231,11 @@ end # reading a struct, struct array, or sparse matrix function m_read(g::HDF5.Group) - mattype = read_attribute(g, name_type_attr_matlab) + if HDF5.name(g) == "/#subsystem#" + mattype = "#subsystem#" + else + mattype = read_attribute(g, name_type_attr_matlab) + end if mattype != "struct" # Check if this is a sparse matrix. fn = keys(g) @@ -226,10 +267,11 @@ function m_read(g::HDF5.Group) end return SparseMatrixCSC(convert(Int, read_attribute(g, sparse_attr_matlab)), length(jc)-1, jc, ir, data) elseif mattype == "function_handle" - @warn "MATLAB $mattype values are currently not supported" - return missing + # Fall through else - @warn "Unknown non-struct group of type $mattype detected; attempting to read as struct" + if mattype != "#subsystem#" + @warn "Unknown non-struct group of type $mattype detected; attempting to read as struct" + end end end if haskey(g, "MATLAB_fields") diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl new file mode 100644 index 0000000..57af12e --- /dev/null +++ b/src/MAT_subsys.jl @@ -0,0 +1,297 @@ +# MAT_subsys.jl +# Tools for processing MAT-file subsystem data in Julia +# +# Copyright (C) 2025 Nithin Lakshmisha +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +module MAT_subsys + +const FWRAP_VERSION = 4 + +mutable struct Subsys + num_names::UInt32 + mcos_names::Vector{String} + class_id_metadata::Vector{UInt32} + object_id_metadata::Vector{UInt32} + saveobj_prop_metadata::Vector{UInt32} + obj_prop_metadata::Vector{UInt32} + dynprop_metadata::Vector{UInt32} + _u6_metadata::Vector{UInt32} + _u7_metadata::Vector{UInt32} + prop_vals_saved::Vector{Any} + _c3::Any + _c2::Any + prop_vals_defaults::Any + handle_data::Any + java_data::Any + + Subsys() = new( + UInt32(0), + String[], + UInt32[], + UInt32[], + UInt32[], + UInt32[], + UInt32[], + UInt32[], + UInt32[], + Any[], + nothing, + nothing, + nothing, + nothing, + nothing + ) +end + +const subsys_cache = Ref{Union{Nothing,Subsys}}(nothing) +const object_cache = Ref{Union{Nothing, Dict{UInt32, Dict{String,Any}}}}(nothing) + +function clear_subsys!() + subsys_cache[] = nothing + object_cache[] = nothing +end + +function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) + subsys_cache[] = Subsys() + object_cache[] = Dict{UInt32, Dict{String,Any}}() + subsys_cache[].handle_data = get(subsystem_data, "handle", nothing) + subsys_cache[].java_data = get(subsystem_data, "java", nothing) + mcos_data = get(subsystem_data, "MCOS", nothing) + if mcos_data === nothing + return + end + + if mcos_data isa Tuple + # Backward compatibility with MAT_v5 + mcos_data = mcos_data[2] + end + fwrap_metadata = vec(mcos_data[1, 1]) + + # FIXME: Is this the best way to read? + # Integers are written as uint8 (with swap), interpret as uint32 + version = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[1:4]) : fwrap_metadata[1:4])[1] + if version <= 1 || version > FWRAP_VERSION + error("Cannot read subsystem: Unsupported FileWrapper version: $version") + end + + subsys_cache[].num_names = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[5:8]) : fwrap_metadata[5:8])[1] + region_offsets = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[9:40]) : fwrap_metadata[9:40]) + + # Class and Property Names stored as list of null-terminated strings + start = 41 + pos = start + name_count = 0 + while name_count < subsys_cache[].num_names + if fwrap_metadata[pos] == 0x00 + push!(subsys_cache[].mcos_names, String(fwrap_metadata[start:pos-1])) + name_count += 1 + start = pos + 1 + if name_count == subsys_cache[].num_names + break + end + end + pos += 1 + end + + subsys_cache[].class_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) : fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) + subsys_cache[].saveobj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) : fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) + subsys_cache[].object_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) : fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) + subsys_cache[].obj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) : fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) + subsys_cache[].dynprop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) : fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) + + if region_offsets[6] != 0 + subsys_cache[]._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) + end + + if region_offsets[7] != 0 + subsys_cache[]._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) + end + + if version == 2 + subsys_cache[].prop_vals_saved = mcos_data[3:end-1, 1] + elseif version == 3 + subsys_cache[].prop_vals_saved = mcos_data[3:end-2, 1] + subsys_cache[]._c2 = mcos_data[end-1, 1] + else + subsys_cache[].prop_vals_saved = mcos_data[3:end-3, 1] + subsys_cache[]._c3 = mcos_data[end-2, 1] + end + + subsys_cache[].prop_vals_defaults = mcos_data[end, 1] +end + +function get_classname(class_id::UInt32) + namespace_idx = subsys_cache[].class_id_metadata[class_id*4+1] + classname_idx = subsys_cache[].class_id_metadata[class_id*4+2] + + namespace = if namespace_idx == 0 + "" + else + subsys_cache[].mcos_names[namespace_idx] * "." + end + + classname = namespace * subsys_cache[].mcos_names[classname_idx] + return classname +end + +function get_object_metadata(object_id::UInt32) + return subsys_cache[].object_id_metadata[object_id*6+1:object_id*6+6] +end + +function get_default_properties(class_id::UInt32) + # FIXME Should we use deepcopy here + return copy(subsys_cache[].prop_vals_defaults[class_id+1, 1]) +end + +function get_property_idxs(obj_type_id::UInt32, saveobj_ret_type::Bool) + prop_field_idxs = saveobj_ret_type ? subsys_cache[].saveobj_prop_metadata : subsys_cache[].obj_prop_metadata + nfields = 3 + offset = 1 + while obj_type_id > 0 + nprops = prop_field_idxs[offset] + offset += 1 + (nfields * nprops) + offset += (offset + 1) % 2 # Padding + obj_type_id -= 1 + end + nprops = prop_field_idxs[offset] + offset += 1 + return prop_field_idxs[offset:offset+nprops*nfields-1] +end + +function find_nested_prop(prop_value::Any) + # Hacky way to find a nested object + # Nested objects are stored as a uint32 Matrix with a unique signature + # MATLAB probably uses some kind of placeholders to decode + # But this should work here + if prop_value isa Dict + # Handle nested objects in a dictionary (struct) + for (key, value) in prop_value + prop_value[key] = find_nested_prop(value) + end + end + + if prop_value isa Matrix{Any} + # Handle nested objects in a Cell + for i in eachindex(prop_value) + prop_value[i] = find_nested_prop(prop_value[i]) + end + end + + if prop_value isa Matrix{UInt32} && prop_value[1,1] == 0xdd000000 + # MATLAB identifies any uint32 array with first value 0xdd000000 as an MCOS object + return load_mcos_object(prop_value, "MCOS") + end + + return prop_value +end + +function get_saved_properties(obj_type_id::UInt32, saveobj_ret_type::Bool) + save_prop_map = Dict{String,Any}() + prop_field_idxs = get_property_idxs(obj_type_id, saveobj_ret_type) + nprops = length(prop_field_idxs) ÷ 3 + for i in 0:nprops-1 + prop_name = subsys_cache[].mcos_names[prop_field_idxs[i*3+1]] + prop_type = prop_field_idxs[i*3+2] + if prop_type == 0 + prop_value = subsys_cache[].mcos_names[prop_field_idxs[i*3+3]] + elseif prop_type == 1 + prop_value = subsys_cache[].prop_vals_saved[prop_field_idxs[i*3+3]+1] + elseif prop_type == 2 + prop_value = prop_field_idxs[i*3+3] + else + error("Unknown property type ID: $prop_type encountered during deserialization") + end + save_prop_map[prop_name] = find_nested_prop(prop_value) + end + return save_prop_map +end + +function get_properties(object_id::UInt32) + if object_id == 0 + return Dict{String,Any}() + end + + class_id, _, _, saveobj_id, normobj_id, _ = get_object_metadata(object_id) + if saveobj_id != 0 + saveobj_ret_type = true + obj_type_id = saveobj_id + else + saveobj_ret_type = false + obj_type_id = normobj_id + end + + prop_map = get_default_properties(class_id) + merge!(prop_map, get_saved_properties(obj_type_id, saveobj_ret_type)) + # TODO: Add dynamic properties + return prop_map +end + +function load_mcos_object(metadata::Any, type_name::String) + if type_name != "MCOS" + @warn "Loading Type:$type_name is not implemented. Returning metadata." + return metadata + end + + if isa(metadata, Dict) + # TODO: Load Enumeration Instances + @warn "Loading enumeration instances are not supported. Returning Metadata" + return metadata + end + + if !(metadata isa Array{UInt32}) + @warn "Expected MCOS metadata to be an Array{UInt32}, got $(typeof(metadata)). Returning metadata." + return metadata + end + + if metadata[1, 1] != 0xDD000000 + @warn "MCOS object metadata is corrupted. Returning raw data." + return metadata + end + + ndims = metadata[2, 1] + dims = metadata[3:2+ndims, 1] + nobjects = prod(dims) + object_ids = metadata[3+ndims:2+ndims+nobjects, 1] + + class_id = metadata[end, 1] + classname = get_classname(class_id) + + object_arr = Array{Dict{String,Any}}(undef, convert(Vector{Int}, dims)...) + + for i = 1:length(object_arr) + oid = object_ids[i] + if haskey(object_cache[], oid) + prop_dict = object_cache[][oid] + else + prop_dict = Dict{String,Any}() + object_cache[][oid] = prop_dict + merge!(prop_dict, get_properties(oid)) + prop_dict["__class__"] = classname + end + object_arr[i] = prop_dict + end + + return object_arr + +end + +end \ No newline at end of file diff --git a/src/MAT_v5.jl b/src/MAT_v5.jl index 7370dd4..c412a71 100644 --- a/src/MAT_v5.jl +++ b/src/MAT_v5.jl @@ -29,6 +29,8 @@ module MAT_v5 using CodecZlib, BufferedStreams, HDF5, SparseArrays import Base: read, write, close +using ..MAT_subsys + round_uint8(data) = round.(UInt8, data) complex_array(a, b) = complex.(a, b) @@ -246,7 +248,7 @@ function read_sparse(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, flags:: end if length(ir) > length(pr) # Fix for Issue #169, xref https://github.com/JuliaLang/julia/pull/40523 - #= + #= # The following expression must be obeyed according to # https://github.com/JuliaLang/julia/blob/b3e4341d43da32f4ab6087230d98d00b89c8c004/stdlib/SparseArrays/src/sparsematrix.jl#L86-L90 @debug "SparseMatrixCSC" m n jc ir pr @@ -311,6 +313,18 @@ function read_string(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}) data end +function read_opaque(f::IO, swap_bytes::Bool) + type_name = String(read_element(f, swap_bytes, UInt8)) + classname = String(read_element(f, swap_bytes, UInt8)) + + if classname == "FileWrapper__" + return read_matrix(f, swap_bytes) + end + + _, metadata = read_matrix(f, swap_bytes) + return MAT_subsys.load_mcos_object(metadata, type_name) +end + # Read matrix data function read_matrix(f::IO, swap_bytes::Bool) (dtype, nbytes) = read_header(f, swap_bytes) @@ -332,15 +346,10 @@ function read_matrix(f::IO, swap_bytes::Bool) flags = read_element(f, swap_bytes, UInt32) class = flags[1] & 0xFF - if class == mxOPAQUE_CLASS - s0 = read_data(f, swap_bytes) - s1 = read_data(f, swap_bytes) - s2 = read_data(f, swap_bytes) - arr = read_matrix(f, swap_bytes) - return ("__opaque__", Dict("s0"=>s0, "s1"=>s1, "s2"=>s2, "arr"=>arr)) + if class != mxOPAQUE_CLASS + dimensions = read_element(f, swap_bytes, Int32) end - dimensions = read_element(f, swap_bytes, Int32) name = String(read_element(f, swap_bytes, UInt8)) local data @@ -354,6 +363,8 @@ function read_matrix(f::IO, swap_bytes::Bool) data = read_string(f, swap_bytes, dimensions) elseif class == mxFUNCTION_CLASS data = read_matrix(f, swap_bytes) + elseif class == mxOPAQUE_CLASS + data = read_opaque(f, swap_bytes) else if (flags[1] & (1 << 9)) != 0 # logical data = read_data(f, swap_bytes, Bool, dimensions) @@ -375,14 +386,41 @@ matopen(ios::IOStream, endian_indicator::UInt16) = # Read whole MAT file function read(matfile::Matlabv5File) - seek(matfile.ios, 128) vars = Dict{String, Any}() + + seek(matfile.ios, 116) + subsys_offset = read_bswap(matfile.ios, matfile.swap_bytes, UInt64) + if subsys_offset == 0x2020202020202020 + subsys_offset = 0 + end + if subsys_offset != 0 + read_subsystem(matfile.ios, matfile.swap_bytes, subsys_offset) + end + + seek(matfile.ios, 128) while !eof(matfile.ios) + offset = position(matfile.ios) + if offset == subsys_offset + # Skip reading subsystem again + (_, nbytes) = read_header(matfile.ios, matfile.swap_bytes) + skip(matfile.ios, nbytes) + continue + end (name, data) = read_matrix(matfile.ios, matfile.swap_bytes) vars[name] = data end vars end + +function read_subsystem(ios::IOStream, swap_bytes::Bool, offset::UInt64) + seek(ios, offset) + (_, subsystem_data) = read_matrix(ios, swap_bytes) + buf = IOBuffer(vec(subsystem_data)) + seek(buf, 8) # Skip subsystem header + _, subsys_data = read_matrix(buf, swap_bytes) + MAT_subsys.load_subsys!(subsys_data, swap_bytes) +end + # Read only variable names from an HDF5 file function getvarnames(matfile::Matlabv5File) if !isdefined(matfile, :varnames) diff --git a/test/read.jl b/test/read.jl index 4d8c9d4..64b2761 100644 --- a/test/read.jl +++ b/test/read.jl @@ -214,21 +214,46 @@ let objtestfile = "figure.fig" end # test reading file containing Matlab function handle, table, and datetime objects -# since we don't support these objects, just make sure that there are no errors -# reading the file and that the variables are there and replaced with `missing` let objtestfile = "function_handles.mat" vars = matread(joinpath(dirname(@__FILE__), "v7.3", objtestfile)) @test "sin" in keys(vars) - @test ismissing(vars["sin"]) + @test typeof(vars["sin"]) == Dict{String, Any} + @test Set(keys(vars["sin"])) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) @test "anonymous" in keys(vars) - @test ismissing(vars["anonymous"]) + @test typeof(vars["anonymous"]) == Dict{String, Any} + @test Set(keys(vars["anonymous"])) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) end -let objtestfile = "struct_table_datetime.mat" - vars = matread(joinpath(dirname(@__FILE__), "v7.3", objtestfile))["s"] - @test "testTable" in keys(vars) - @test ismissing(vars["testTable"]) - @test "testDatetime" in keys(vars) - @test ismissing(vars["testDatetime"]) + +for format in ["v7", "v7.3"] + let objtestfile = "struct_table_datetime.mat" + vars = matread(joinpath(dirname(@__FILE__), format, objtestfile))["s"] + @test "testTable" in keys(vars) + @test size(vars["testTable"]) == (1, 1) + @test Set(keys(vars["testTable"][1, 1])) == Set(["__class__", "props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) + @test vars["testTable"][1, 1]["__class__"] == "table" + @test vars["testTable"][1, 1]["ndims"] === 2.0 + @test vars["testTable"][1, 1]["nvars"] === 5.0 + @test vars["testTable"][1, 1]["nrows"] === 3.0 + @test vars["testTable"][1, 1]["data"][1, 1] == reshape([1261.0, 547.0, 3489.0], 3, 1) + @test vars["testTable"][1, 1]["data"][1, 2][1, 1]["__class__"] == "string" + @test vars["testTable"][1, 1]["data"][1, 3][1, 1]["__class__"] == "datetime" + @test vars["testTable"][1, 1]["data"][1, 4][1, 1]["__class__"] == "categorical" + @test vars["testTable"][1, 1]["data"][1, 5][1, 1]["__class__"] == "string" + + @test "testDatetime" in keys(vars) + @test size(vars["testDatetime"]) == (1, 1) + if format == "v7.3" + @test Set(keys(vars["testDatetime"][1, 1])) == Set(["__class__", "tz", "data", "fmt", "isDateOnly"]) + @test vars["testDatetime"][1, 1]["isDateOnly"] === false + else + # MATLAB removed property "isDateOnly" in later versions + @test Set(keys(vars["testDatetime"][1, 1])) == Set(["__class__", "tz", "data", "fmt"]) + end + @test vars["testDatetime"][1, 1]["__class__"] == "datetime" + @test vars["testDatetime"][1, 1]["tz"] === "" + @test vars["testDatetime"][1, 1]["fmt"] === "" + @test vars["testDatetime"][1, 1]["data"] === 1.575304969634e12 + 0.0im + end end # test reading of old-style Matlab object in v7.3 format diff --git a/test/v7/struct_table_datetime.mat b/test/v7/struct_table_datetime.mat new file mode 100644 index 0000000000000000000000000000000000000000..6a4885a927763e72aa91128ab0bdf467c376e9a4 GIT binary patch literal 1451 zcmV;c1yuS?K~zjZLLfCRFd$7qR4ry{Y-KDUP;6mzW^ZzBIv`L(S4mDbG%O%Pa%Ew3 zWn>_4ZaN@TXmub#b!Q+lF(5HHIy5jkGB6-AFfuhDARr(hARr(hARr(hARr(hARr(h zARr(hARr(hARr(hARr(hARr(H0RR91000000ZB~{0001f0001Zoa19)V8~!(U|<8% z96-#(00FE}J|i46G8BWwSr~YsVp2d{l3HBil30>jl9`(d=7c0B<)kve0Ur=IFd)nq zgo-o5X%+@wXa8VEsC)|4ERa4ABz?SS?qWl78v~Rt0b$-{LUA_)lj07L9VJk2P|F=4 zHVY5~0QR{S5)S|X0G$K?0C=3;SWk}|H57j{$)+3H-7ZxrD^7bsDiQ*Y9H<&)SD_%b zyXvZCZ`Cy8O)Os;_z1k`u@jFolWhLb z15fhU&wlUsp6%xs$J+p~`zAmG>rE_8$Z13K9+p4v7YCi)Y5Ie*LXgXDkr ztLJ^`7v+83SJzmV+PS+}USdgb?71!^7Y12CAC~cm!C90+_9X=30@6%!(St4y^44&QY6TO}A~|Lpy>_qB?=5}r!E#K(r?tGyR4PR%Md)pfCP>=+tk0M+Y~3;x=m|#Q@3BkvZLGAA&+Ix!Z5|x#n`IBBzE2PSeCY}`T$vT zfe1Kc(AWO-XUwlwGQVBbYJKtjH^VkIe)&t)FFv~W?XW!=FDJ715~}l!YmekVu;lBG zyxDUjzk^&S;BpzgkK;I8M(;R|r|h|T(Vdr%J$}to^E>rs$HUuDj0eh{^WtGo*YD{1 zeO-3|b0qX|&oh)6U}T)B7DdS(wm8NHgLknlu#ZN%I&MPj58*Qiu$N&!@db%d{$3ndS^j%BUyIk99K6N^?>FBX ztLeR77oLgXb>VGjAJ>Ive!~dwL#W=5%KV#CYc>5%rGFQyyAk#RK9iMYVYB)A@^_#_w0N4V(f0Y2c_#bMeT+o?o5U};zKBpZWF z`Q}WMbPCh7bt)DB>2b>B0p0bd$k?$wNsA+o1<%q_o&8u~SPXHpW@PGj*`K|?zHY6e zXUF4J;>o@5tW$rn{Ij!SzWnF@ygXpYK9B42ds(}S=J)3_SeQ@QQmfoUv{%iSJzoYhla3eqO+(7|n@pJos*SGI|HynL;y@|gK z)ui^j_G6cg=l<#`R0p-1BazEDx^;tq{T2t-Yn-vW zN?vmRU%T>tmEJ4({ZC-A^|asrLVYKiw?6K^y6;M5@6ql%(y0Ql-lJFWyYL>3^e>nB FO~fcN&4>U1 literal 0 HcmV?d00001 From 33c64553111c6a644361c54e4c8a26443f5c86d1 Mon Sep 17 00:00:00 2001 From: foreverallama Date: Thu, 21 Aug 2025 20:10:53 +0530 Subject: [PATCH 02/20] MAT_subsys.load_subsys!: Fix indexing for parsing metadata from region offests --- src/MAT_subsys.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index 57af12e..7d7eef2 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -118,11 +118,11 @@ function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) subsys_cache[].obj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) : fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) subsys_cache[].dynprop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) : fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) - if region_offsets[6] != 0 + if region_offsets[7] != 0 subsys_cache[]._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) end - if region_offsets[7] != 0 + if region_offsets[8] != 0 subsys_cache[]._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) end From 02444a657edf05ec475e3754f58e5de9c957f028 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 11:31:33 +0100 Subject: [PATCH 03/20] first step in subsystem refactoring --- src/MAT_subsys.jl | 100 +++++++++++++++++++++++++--------------------- src/MAT_types.jl | 26 ++++++++++++ test/read.jl | 18 ++++----- 3 files changed, 90 insertions(+), 54 deletions(-) diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index bda89aa..06d6b24 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -24,11 +24,12 @@ module MAT_subsys -import ..MAT_types: MatlabStructArray +import ..MAT_types: MatlabStructArray, MatlabOpaque const FWRAP_VERSION = 4 mutable struct Subsys + object_cache::Dict{UInt32, MatlabOpaque} num_names::UInt32 mcos_names::Vector{String} class_id_metadata::Vector{UInt32} @@ -46,6 +47,7 @@ mutable struct Subsys java_data::Any Subsys() = new( + Dict{UInt32, MatlabOpaque}(), UInt32(0), String[], UInt32[], @@ -64,17 +66,28 @@ mutable struct Subsys ) end +function get_object!(subsys::Subsys, oid::UInt32, classname::String) + if haskey(subsys.object_cache, oid) + # object is already cached, just retrieve it + obj = subsys.object_cache[oid] + else + prop_dict = Dict{String,Any}() + merge!(prop_dict, get_properties(oid)) + # cache it + obj = MatlabOpaque(prop_dict, classname) + subsys.object_cache[oid] = obj + end + return obj +end + const subsys_cache = Ref{Union{Nothing,Subsys}}(nothing) -const object_cache = Ref{Union{Nothing, Dict{UInt32, Dict{String,Any}}}}(nothing) function clear_subsys!() subsys_cache[] = nothing - object_cache[] = nothing end function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) subsys_cache[] = Subsys() - object_cache[] = Dict{UInt32, Dict{String,Any}}() subsys_cache[].handle_data = get(subsystem_data, "handle", nothing) subsys_cache[].java_data = get(subsystem_data, "java", nothing) mcos_data = get(subsystem_data, "MCOS", nothing) @@ -161,6 +174,7 @@ end function get_default_properties(class_id::UInt32) prop_vals_class = subsys_cache[].prop_vals_defaults[class_id+1, 1] + # is it always a MatlabStructArray? if prop_vals_class isa MatlabStructArray prop_vals_class = Dict{String,Any}(prop_vals_class) end @@ -184,31 +198,36 @@ function get_property_idxs(obj_type_id::UInt32, saveobj_ret_type::Bool) return prop_field_idxs[offset:offset+nprops*nfields-1] end -function find_nested_prop(prop_value::Any) - # Hacky way to find a nested object - # Nested objects are stored as a uint32 Matrix with a unique signature - # MATLAB probably uses some kind of placeholders to decode - # But this should work here - if prop_value isa Dict - # Handle nested objects in a dictionary (struct) - for (key, value) in prop_value - prop_value[key] = find_nested_prop(value) - end +update_nested_props!(prop_value) = prop_value + +function update_nested_props!(prop_value::Union{AbstractDict, MatlabStructArray}) + # Handle nested objects in structs + for (key, value) in prop_value + prop_value[key] = update_nested_props!(value) end + return prop_value +end - if prop_value isa Matrix{Any} - # Handle nested objects in a Cell - for i in eachindex(prop_value) - prop_value[i] = find_nested_prop(prop_value[i]) - end +function update_nested_props!(prop_value::Array{Any}) + # Handle nested objects in a Cell + for i in eachindex(prop_value) + prop_value[i] = update_nested_props!(prop_value[i]) end + return prop_value +end + +function update_nested_props!(prop_value::Array{UInt32}) + # Hacky way to find and update nested objects + # Nested objects are stored as a uint32 Matrix with a unique signature + # MATLAB probably uses some kind of placeholders to decode + # But this should work here - if prop_value isa Matrix{UInt32} && prop_value[1,1] == 0xdd000000 + if first(prop_value) == 0xdd000000 # MATLAB identifies any uint32 array with first value 0xdd000000 as an MCOS object return load_mcos_object(prop_value, "MCOS") + else + return prop_value end - - return prop_value end function get_saved_properties(obj_type_id::UInt32, saveobj_ret_type::Bool) @@ -227,7 +246,7 @@ function get_saved_properties(obj_type_id::UInt32, saveobj_ret_type::Bool) else error("Unknown property type ID: $prop_type encountered during deserialization") end - save_prop_map[prop_name] = find_nested_prop(prop_value) + save_prop_map[prop_name] = update_nested_props!(prop_value) end return save_prop_map end @@ -253,19 +272,18 @@ function get_properties(object_id::UInt32) end function load_mcos_object(metadata::Any, type_name::String) - if type_name != "MCOS" - @warn "Loading Type:$type_name is not implemented. Returning metadata." - return metadata - end + @warn "Expected MCOS metadata to be an Array{UInt32}, got $(typeof(metadata)). Returning metadata." + return metadata +end - if isa(metadata, Dict) - # TODO: Load Enumeration Instances - @warn "Loading enumeration instances are not supported. Returning Metadata" - return metadata - end +function load_mcos_object(metadata::Dict, type_name::String) + @warn "Loading enumeration instances are not supported. Returning Metadata" + return metadata +end - if !(metadata isa Array{UInt32}) - @warn "Expected MCOS metadata to be an Array{UInt32}, got $(typeof(metadata)). Returning metadata." +function load_mcos_object(metadata::Array{UInt32}, type_name::String) + if type_name != "MCOS" + @warn "Loading Type:$type_name is not implemented. Returning metadata." return metadata end @@ -282,23 +300,15 @@ function load_mcos_object(metadata::Any, type_name::String) class_id = metadata[end, 1] classname = get_classname(class_id) - object_arr = Array{Dict{String,Any}}(undef, convert(Vector{Int}, dims)...) + object_arr = Array{MatlabOpaque}(undef, convert(Vector{Int}, dims)...) for i = 1:length(object_arr) oid = object_ids[i] - if haskey(object_cache[], oid) - prop_dict = object_cache[][oid] - else - prop_dict = Dict{String,Any}() - object_cache[][oid] = prop_dict - merge!(prop_dict, get_properties(oid)) - prop_dict["__class__"] = classname - end - object_arr[i] = prop_dict + obj = get_object!(subsys_cache[], oid, classname) + object_arr[i] = obj end return object_arr - end end \ No newline at end of file diff --git a/src/MAT_types.jl b/src/MAT_types.jl index 6ae447f..b34d01e 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -277,4 +277,30 @@ module MAT_types d = Dict{String, Any}(string.(keys) .=> values) return MatlabClassObject(d, class) end + + """ + MatlabOpaque( + d::Dict{String, Any}, + class::String, + ) <: AbstractDict{String, Any} + + Type to store opaque class objects. + These are the 'modern' Matlab classes, different from the old `MatlabClassObject` types. + + """ + struct MatlabOpaque <: AbstractDict{String, Any} + d::Dict{String, Any} + class::String + end + + Base.eltype(::Type{MatlabOpaque}) = Pair{String, Any} + Base.length(m::MatlabOpaque) = length(m.d) + Base.keys(m::MatlabOpaque) = keys(m.d) + Base.values(m::MatlabOpaque) = values(m.d) + Base.getindex(m::MatlabOpaque, i) = getindex(m.d, i) + Base.setindex!(m::MatlabOpaque, v, k) = setindex!(m.d, v, k) + Base.iterate(m::MatlabOpaque, i) = iterate(m.d, i) + Base.iterate(m::MatlabOpaque) = iterate(m.d) + Base.haskey(m::MatlabOpaque, k) = haskey(m.d, k) + Base.get(m::MatlabOpaque, k, default) = get(m.d, k, default) end \ No newline at end of file diff --git a/test/read.jl b/test/read.jl index 1b56e66..b5b1519 100644 --- a/test/read.jl +++ b/test/read.jl @@ -233,27 +233,27 @@ for format in ["v7", "v7.3"] vars = matread(joinpath(dirname(@__FILE__), format, objtestfile))["s"] @test "testTable" in keys(vars) @test size(vars["testTable"]) == (1, 1) - @test Set(keys(vars["testTable"][1, 1])) == Set(["__class__", "props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) - @test vars["testTable"][1, 1]["__class__"] == "table" + @test Set(keys(vars["testTable"][1, 1])) == Set(["props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) + @test vars["testTable"][1, 1].class == "table" @test vars["testTable"][1, 1]["ndims"] === 2.0 @test vars["testTable"][1, 1]["nvars"] === 5.0 @test vars["testTable"][1, 1]["nrows"] === 3.0 @test vars["testTable"][1, 1]["data"][1, 1] == reshape([1261.0, 547.0, 3489.0], 3, 1) - @test vars["testTable"][1, 1]["data"][1, 2][1, 1]["__class__"] == "string" - @test vars["testTable"][1, 1]["data"][1, 3][1, 1]["__class__"] == "datetime" - @test vars["testTable"][1, 1]["data"][1, 4][1, 1]["__class__"] == "categorical" - @test vars["testTable"][1, 1]["data"][1, 5][1, 1]["__class__"] == "string" + @test vars["testTable"][1, 1]["data"][1, 2][1, 1].class == "string" + @test vars["testTable"][1, 1]["data"][1, 3][1, 1].class == "datetime" + @test vars["testTable"][1, 1]["data"][1, 4][1, 1].class == "categorical" + @test vars["testTable"][1, 1]["data"][1, 5][1, 1].class == "string" @test "testDatetime" in keys(vars) @test size(vars["testDatetime"]) == (1, 1) if format == "v7.3" - @test Set(keys(vars["testDatetime"][1, 1])) == Set(["__class__", "tz", "data", "fmt", "isDateOnly"]) + @test Set(keys(vars["testDatetime"][1, 1])) == Set(["tz", "data", "fmt", "isDateOnly"]) @test vars["testDatetime"][1, 1]["isDateOnly"] === false else # MATLAB removed property "isDateOnly" in later versions - @test Set(keys(vars["testDatetime"][1, 1])) == Set(["__class__", "tz", "data", "fmt"]) + @test Set(keys(vars["testDatetime"][1, 1])) == Set(["tz", "data", "fmt"]) end - @test vars["testDatetime"][1, 1]["__class__"] == "datetime" + @test vars["testDatetime"][1, 1].class == "datetime" @test vars["testDatetime"][1, 1]["tz"] === "" @test vars["testDatetime"][1, 1]["fmt"] === "" @test vars["testDatetime"][1, 1]["data"] === 1.575304969634e12 + 0.0im From c2b55cdb6696ed5dbfcc9039f6fe4cddd6f8c734 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 12:17:26 +0100 Subject: [PATCH 04/20] stateless MAT_subsys module --- src/MAT.jl | 1 - src/MAT_HDF5.jl | 31 ++++++------ src/MAT_subsys.jl | 126 +++++++++++++++++++++++----------------------- src/MAT_v5.jl | 51 ++++++++++--------- 4 files changed, 108 insertions(+), 101 deletions(-) diff --git a/src/MAT.jl b/src/MAT.jl index 9c5bc1c..6b95cf4 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -138,7 +138,6 @@ function matread(filename::AbstractString) try vars = read(file) finally - MAT_subsys.clear_subsys!() close(file) end vars diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 51482f8..a7443ff 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -44,9 +44,10 @@ mutable struct MatlabHDF5File <: HDF5.H5DataStore writeheader::Bool refcounter::Int compress::Bool + subsystem::Subsystem function MatlabHDF5File(plain, toclose::Bool=true, writeheader::Bool=false, refcounter::Int=0, compress::Bool=false) - f = new(plain, toclose, writeheader, refcounter, compress) + f = new(plain, toclose, writeheader, refcounter, compress, Subsystem()) if toclose finalizer(close, f) end @@ -118,8 +119,8 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo end subsys_refs = "#subsystem#" if haskey(fid.plain, subsys_refs) - subsys_data = m_read(fid.plain[subsys_refs]) - MAT_subsys.load_subsys!(subsys_data, endian_indicator) + subsys_data = m_read(fid.plain[subsys_refs], fid.subsystem) + MAT_subsys.load_subsys!(fid.subsystem, subsys_data, endian_indicator) end fid end @@ -142,14 +143,14 @@ function read_complex(dtype::HDF5.Datatype, dset::HDF5.Dataset, ::Type{T}) where return read(dset, Complex{T}) end -function read_cell(dset::HDF5.Dataset) +function read_cell(dset::HDF5.Dataset, subsys::Subsystem) refs = read(dset, Reference) out = Array{Any}(undef, size(refs)) f = HDF5.file(dset) for i = 1:length(refs) dset = f[refs[i]] try - out[i] = m_read(dset) + out[i] = m_read(dset, subsys) finally close(dset) end @@ -157,7 +158,7 @@ function read_cell(dset::HDF5.Dataset) return out end -function m_read(dset::HDF5.Dataset) +function m_read(dset::HDF5.Dataset, subsys::Subsystem) if haskey(dset, empty_attr_matlab) # Empty arrays encode the dimensions as the dataset dims = convert(Vector{Int}, read(dset)) @@ -184,14 +185,14 @@ function m_read(dset::HDF5.Dataset) if mattype == "cell" && objecttype === nothing # Cell arrays, represented as an array of refs - return read_cell(dset) + return read_cell(dset, subsys) elseif objecttype !== nothing if objecttype != 3 @warn "MATLAB Object Type $mattype is currently not supported." return missing end if mattype == "FileWrapper__" - return read_cell(dset) + return read_cell(dset, subsys) end if haskey(dset, "MATLAB_fields") @warn "Enumeration Instances are not supported currently." @@ -199,7 +200,7 @@ function m_read(dset::HDF5.Dataset) end elseif mattype == "struct_array_field" # This will be converted into MatlabStructArray in `m_read(g::HDF5.Group)` - return StructArrayField(read_cell(dset)) + return StructArrayField(read_cell(dset, subsys)) elseif !haskey(str2type_matlab,mattype) @warn "MATLAB $mattype values are currently not supported." return missing @@ -219,7 +220,7 @@ function m_read(dset::HDF5.Dataset) class_id = HDF5.API.h5t_get_class(dtype.id) d = class_id == HDF5.API.H5T_COMPOUND ? read_complex(dtype, dset, T) : read(dset, T) if objecttype !== nothing - return MAT_subsys.load_mcos_object(d, "MCOS") + return MAT_subsys.load_mcos_object(d, "MCOS", subsys) else return length(d) == 1 ? d[1] : d end @@ -265,7 +266,7 @@ function read_sparse_matrix(g::HDF5.Group, mattype::String) return SparseMatrixCSC(convert(Int, read_attribute(g, sparse_attr_matlab)), length(jc)-1, jc, ir, data) end -function read_struct_as_dict(g::HDF5.Group) +function read_struct_as_dict(g::HDF5.Group, subsys::Subsystem) if haskey(g, "MATLAB_fields") fn = [join(f) for f in read_attribute(g, "MATLAB_fields")] else @@ -275,7 +276,7 @@ function read_struct_as_dict(g::HDF5.Group) for i = 1:length(fn) dset = g[fn[i]] try - s[fn[i]] = m_read(dset) + s[fn[i]] = m_read(dset, subsys) finally close(dset) end @@ -284,7 +285,7 @@ function read_struct_as_dict(g::HDF5.Group) end # reading a struct, struct array, or sparse matrix -function m_read(g::HDF5.Group) +function m_read(g::HDF5.Group, subsys::Subsystem) if HDF5.name(g) == "/#subsystem#" mattype = "#subsystem#" else @@ -312,7 +313,7 @@ function m_read(g::HDF5.Group) else class = "" end - s = read_struct_as_dict(g) + s = read_struct_as_dict(g, subsys) out = convert_struct_array(s, class) return out end @@ -328,7 +329,7 @@ function read(f::MatlabHDF5File, name::String) local val obj = f.plain[name] try - val = m_read(obj) + val = m_read(obj, f.subsystem) finally close(obj) end diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index 06d6b24..8fadb67 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -26,9 +26,11 @@ module MAT_subsys import ..MAT_types: MatlabStructArray, MatlabOpaque +export Subsystem + const FWRAP_VERSION = 4 -mutable struct Subsys +mutable struct Subsystem object_cache::Dict{UInt32, MatlabOpaque} num_names::UInt32 mcos_names::Vector{String} @@ -46,7 +48,7 @@ mutable struct Subsys handle_data::Any java_data::Any - Subsys() = new( + Subsystem() = new( Dict{UInt32, MatlabOpaque}(), UInt32(0), String[], @@ -66,13 +68,13 @@ mutable struct Subsys ) end -function get_object!(subsys::Subsys, oid::UInt32, classname::String) +function get_object!(subsys::Subsystem, oid::UInt32, classname::String) if haskey(subsys.object_cache, oid) # object is already cached, just retrieve it obj = subsys.object_cache[oid] else prop_dict = Dict{String,Any}() - merge!(prop_dict, get_properties(oid)) + merge!(prop_dict, get_properties(subsys, oid)) # cache it obj = MatlabOpaque(prop_dict, classname) subsys.object_cache[oid] = obj @@ -80,16 +82,14 @@ function get_object!(subsys::Subsys, oid::UInt32, classname::String) return obj end -const subsys_cache = Ref{Union{Nothing,Subsys}}(nothing) - -function clear_subsys!() - subsys_cache[] = nothing +function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) + subsys = Subsystem() + load_subsys!(subsys, subsystem_data, swap_bytes) end -function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) - subsys_cache[] = Subsys() - subsys_cache[].handle_data = get(subsystem_data, "handle", nothing) - subsys_cache[].java_data = get(subsystem_data, "java", nothing) +function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_bytes::Bool) + subsys.handle_data = get(subsystem_data, "handle", nothing) + subsys.java_data = get(subsystem_data, "java", nothing) mcos_data = get(subsystem_data, "MCOS", nothing) if mcos_data === nothing return @@ -108,72 +108,74 @@ function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) error("Cannot read subsystem: Unsupported FileWrapper version: $version") end - subsys_cache[].num_names = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[5:8]) : fwrap_metadata[5:8])[1] + subsys.num_names = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[5:8]) : fwrap_metadata[5:8])[1] region_offsets = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[9:40]) : fwrap_metadata[9:40]) # Class and Property Names stored as list of null-terminated strings start = 41 pos = start name_count = 0 - while name_count < subsys_cache[].num_names + while name_count < subsys.num_names if fwrap_metadata[pos] == 0x00 - push!(subsys_cache[].mcos_names, String(fwrap_metadata[start:pos-1])) + push!(subsys.mcos_names, String(fwrap_metadata[start:pos-1])) name_count += 1 start = pos + 1 - if name_count == subsys_cache[].num_names + if name_count == subsys.num_names break end end pos += 1 end - subsys_cache[].class_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) : fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) - subsys_cache[].saveobj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) : fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) - subsys_cache[].object_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) : fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) - subsys_cache[].obj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) : fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) - subsys_cache[].dynprop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) : fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) + subsys.class_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) : fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) + subsys.saveobj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) : fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) + subsys.object_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) : fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) + subsys.obj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) : fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) + subsys.dynprop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) : fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) if region_offsets[7] != 0 - subsys_cache[]._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) + subsys._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) end if region_offsets[8] != 0 - subsys_cache[]._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) + subsys._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) end if version == 2 - subsys_cache[].prop_vals_saved = mcos_data[3:end-1, 1] + subsys.prop_vals_saved = mcos_data[3:end-1, 1] elseif version == 3 - subsys_cache[].prop_vals_saved = mcos_data[3:end-2, 1] - subsys_cache[]._c2 = mcos_data[end-1, 1] + subsys.prop_vals_saved = mcos_data[3:end-2, 1] + subsys._c2 = mcos_data[end-1, 1] else - subsys_cache[].prop_vals_saved = mcos_data[3:end-3, 1] - subsys_cache[]._c3 = mcos_data[end-2, 1] + subsys.prop_vals_saved = mcos_data[3:end-3, 1] + subsys._c3 = mcos_data[end-2, 1] end - subsys_cache[].prop_vals_defaults = mcos_data[end, 1] + subsys.prop_vals_defaults = mcos_data[end, 1] + + return subsys end -function get_classname(class_id::UInt32) - namespace_idx = subsys_cache[].class_id_metadata[class_id*4+1] - classname_idx = subsys_cache[].class_id_metadata[class_id*4+2] +function get_classname(subsys::Subsystem, class_id::UInt32) + namespace_idx = subsys.class_id_metadata[class_id*4+1] + classname_idx = subsys.class_id_metadata[class_id*4+2] namespace = if namespace_idx == 0 "" else - subsys_cache[].mcos_names[namespace_idx] * "." + subsys.mcos_names[namespace_idx] * "." end - classname = namespace * subsys_cache[].mcos_names[classname_idx] + classname = namespace * subsys.mcos_names[classname_idx] return classname end -function get_object_metadata(object_id::UInt32) - return subsys_cache[].object_id_metadata[object_id*6+1:object_id*6+6] +function get_object_metadata(subsys::Subsystem, object_id::UInt32) + return subsys.object_id_metadata[object_id*6+1:object_id*6+6] end -function get_default_properties(class_id::UInt32) - prop_vals_class = subsys_cache[].prop_vals_defaults[class_id+1, 1] +function get_default_properties(subsys::Subsystem, class_id::UInt32) + prop_vals_class = subsys.prop_vals_defaults[class_id+1, 1] # is it always a MatlabStructArray? if prop_vals_class isa MatlabStructArray prop_vals_class = Dict{String,Any}(prop_vals_class) @@ -183,8 +185,8 @@ function get_default_properties(class_id::UInt32) return r end -function get_property_idxs(obj_type_id::UInt32, saveobj_ret_type::Bool) - prop_field_idxs = saveobj_ret_type ? subsys_cache[].saveobj_prop_metadata : subsys_cache[].obj_prop_metadata +function get_property_idxs(subsys::Subsystem, obj_type_id::UInt32, saveobj_ret_type::Bool) + prop_field_idxs = saveobj_ret_type ? subsys.saveobj_prop_metadata : subsys.obj_prop_metadata nfields = 3 offset = 1 while obj_type_id > 0 @@ -198,25 +200,25 @@ function get_property_idxs(obj_type_id::UInt32, saveobj_ret_type::Bool) return prop_field_idxs[offset:offset+nprops*nfields-1] end -update_nested_props!(prop_value) = prop_value +update_nested_props!(prop_value, subsys::Subsystem) = prop_value -function update_nested_props!(prop_value::Union{AbstractDict, MatlabStructArray}) +function update_nested_props!(prop_value::Union{AbstractDict, MatlabStructArray}, subsys::Subsystem) # Handle nested objects in structs for (key, value) in prop_value - prop_value[key] = update_nested_props!(value) + prop_value[key] = update_nested_props!(value, subsys) end return prop_value end -function update_nested_props!(prop_value::Array{Any}) +function update_nested_props!(prop_value::Array{Any}, subsys::Subsystem) # Handle nested objects in a Cell for i in eachindex(prop_value) - prop_value[i] = update_nested_props!(prop_value[i]) + prop_value[i] = update_nested_props!(prop_value[i], subsys) end return prop_value end -function update_nested_props!(prop_value::Array{UInt32}) +function update_nested_props!(prop_value::Array{UInt32}, subsys::Subsystem) # Hacky way to find and update nested objects # Nested objects are stored as a uint32 Matrix with a unique signature # MATLAB probably uses some kind of placeholders to decode @@ -224,39 +226,39 @@ function update_nested_props!(prop_value::Array{UInt32}) if first(prop_value) == 0xdd000000 # MATLAB identifies any uint32 array with first value 0xdd000000 as an MCOS object - return load_mcos_object(prop_value, "MCOS") + return load_mcos_object(prop_value, "MCOS", subsys) else return prop_value end end -function get_saved_properties(obj_type_id::UInt32, saveobj_ret_type::Bool) +function get_saved_properties(subsys::Subsystem, obj_type_id::UInt32, saveobj_ret_type::Bool) save_prop_map = Dict{String,Any}() - prop_field_idxs = get_property_idxs(obj_type_id, saveobj_ret_type) + prop_field_idxs = get_property_idxs(subsys, obj_type_id, saveobj_ret_type) nprops = length(prop_field_idxs) ÷ 3 for i in 0:nprops-1 - prop_name = subsys_cache[].mcos_names[prop_field_idxs[i*3+1]] + prop_name = subsys.mcos_names[prop_field_idxs[i*3+1]] prop_type = prop_field_idxs[i*3+2] if prop_type == 0 - prop_value = subsys_cache[].mcos_names[prop_field_idxs[i*3+3]] + prop_value = subsys.mcos_names[prop_field_idxs[i*3+3]] elseif prop_type == 1 - prop_value = subsys_cache[].prop_vals_saved[prop_field_idxs[i*3+3]+1] + prop_value = subsys.prop_vals_saved[prop_field_idxs[i*3+3]+1] elseif prop_type == 2 prop_value = prop_field_idxs[i*3+3] else error("Unknown property type ID: $prop_type encountered during deserialization") end - save_prop_map[prop_name] = update_nested_props!(prop_value) + save_prop_map[prop_name] = update_nested_props!(prop_value, subsys) end return save_prop_map end -function get_properties(object_id::UInt32) +function get_properties(subsys::Subsystem, object_id::UInt32) if object_id == 0 return Dict{String,Any}() end - class_id, _, _, saveobj_id, normobj_id, _ = get_object_metadata(object_id) + class_id, _, _, saveobj_id, normobj_id, _ = get_object_metadata(subsys, object_id) if saveobj_id != 0 saveobj_ret_type = true obj_type_id = saveobj_id @@ -265,23 +267,23 @@ function get_properties(object_id::UInt32) obj_type_id = normobj_id end - prop_map = get_default_properties(class_id) - merge!(prop_map, get_saved_properties(obj_type_id, saveobj_ret_type)) + prop_map = get_default_properties(subsys, class_id) + merge!(prop_map, get_saved_properties(subsys, obj_type_id, saveobj_ret_type)) # TODO: Add dynamic properties return prop_map end -function load_mcos_object(metadata::Any, type_name::String) +function load_mcos_object(metadata::Any, type_name::String, subsys::Subsystem) @warn "Expected MCOS metadata to be an Array{UInt32}, got $(typeof(metadata)). Returning metadata." return metadata end -function load_mcos_object(metadata::Dict, type_name::String) +function load_mcos_object(metadata::Dict, type_name::String, subsys::Subsystem) @warn "Loading enumeration instances are not supported. Returning Metadata" return metadata end -function load_mcos_object(metadata::Array{UInt32}, type_name::String) +function load_mcos_object(metadata::Array{UInt32}, type_name::String, subsys::Subsystem) if type_name != "MCOS" @warn "Loading Type:$type_name is not implemented. Returning metadata." return metadata @@ -298,13 +300,13 @@ function load_mcos_object(metadata::Array{UInt32}, type_name::String) object_ids = metadata[3+ndims:2+ndims+nobjects, 1] class_id = metadata[end, 1] - classname = get_classname(class_id) + classname = get_classname(subsys, class_id) object_arr = Array{MatlabOpaque}(undef, convert(Vector{Int}, dims)...) for i = 1:length(object_arr) oid = object_ids[i] - obj = get_object!(subsys_cache[], oid, classname) + obj = get_object!(subsys, oid, classname) object_arr[i] = obj end diff --git a/src/MAT_v5.jl b/src/MAT_v5.jl index 28752d3..943ba68 100644 --- a/src/MAT_v5.jl +++ b/src/MAT_v5.jl @@ -38,9 +38,10 @@ complex_array(a, b) = complex.(a, b) mutable struct Matlabv5File <: HDF5.H5DataStore ios::IOStream swap_bytes::Bool + subsystem::Subsystem varnames::Dict{String, Int64} - Matlabv5File(ios, swap_bytes) = new(ios, swap_bytes) + Matlabv5File(ios, swap_bytes) = new(ios, swap_bytes, Subsystem()) end const miINT8 = 1 @@ -162,15 +163,15 @@ function read_data(f::IO, swap_bytes::Bool, ::Type{T}, dimensions::Vector{Int32} read_array ? convert(Array{T}, data) : convert(T, data) end -function read_cell(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}) +function read_cell(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, subsys::Subsystem) data = Array{Any}(undef, convert(Vector{Int}, dimensions)...) for i = 1:length(data) - (ignored_name, data[i]) = read_matrix(f, swap_bytes) + (ignored_name, data[i]) = read_matrix(f, swap_bytes, subsys) end data end -function read_struct(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, is_object::Bool) +function read_struct(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, is_object::Bool, subsys::Subsystem) if is_object class = String(read_element(f, swap_bytes, UInt8)) else @@ -195,7 +196,7 @@ function read_struct(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, is_obje data = Dict{String, Any}() sizehint!(data, n_fields+1) for field_name in field_name_strings - data[field_name] = read_matrix(f, swap_bytes)[2] + data[field_name] = read_matrix(f, swap_bytes, subsys)[2] end if is_object data = MatlabClassObject(data, class) @@ -207,7 +208,7 @@ function read_struct(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}, is_obje field_values = Array{Any, N}[Array{Any}(undef, dimensions...) for _ in 1:nfields] for i = 1:n_el for field in 1:nfields - field_values[field][i] = read_matrix(f, swap_bytes)[2] + field_values[field][i] = read_matrix(f, swap_bytes, subsys)[2] end end data = MatlabStructArray{N}(field_name_strings, field_values, class) @@ -317,23 +318,24 @@ function read_string(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}) data end -function read_opaque(f::IO, swap_bytes::Bool) +function read_opaque(f::IO, swap_bytes::Bool, subsys::Subsystem) type_name = String(read_element(f, swap_bytes, UInt8)) classname = String(read_element(f, swap_bytes, UInt8)) if classname == "FileWrapper__" - return read_matrix(f, swap_bytes) + return read_matrix(f, swap_bytes, subsys) end - _, metadata = read_matrix(f, swap_bytes) - return MAT_subsys.load_mcos_object(metadata, type_name) + _, metadata = read_matrix(f, swap_bytes, subsys) + return MAT_subsys.load_mcos_object(metadata, type_name, subsys) end # Read matrix data -function read_matrix(f::IO, swap_bytes::Bool) +function read_matrix(f::IO, swap_bytes::Bool, subsys::Subsystem) (dtype, nbytes) = read_header(f, swap_bytes) if dtype == miCOMPRESSED - return read_matrix(ZlibDecompressorStream(IOBuffer(read!(f, Vector{UInt8}(undef, nbytes)))), swap_bytes) + decompressed_ios = ZlibDecompressorStream(IOBuffer(read!(f, Vector{UInt8}(undef, nbytes)))) + return read_matrix(decompressed_ios, swap_bytes, subsys) elseif dtype != miMATRIX error("Unexpected data type") elseif nbytes == 0 @@ -358,17 +360,17 @@ function read_matrix(f::IO, swap_bytes::Bool) local data if class == mxCELL_CLASS - data = read_cell(f, swap_bytes, dimensions) + data = read_cell(f, swap_bytes, dimensions, subsys) elseif class == mxSTRUCT_CLASS || class == mxOBJECT_CLASS - data = read_struct(f, swap_bytes, dimensions, class == mxOBJECT_CLASS) + data = read_struct(f, swap_bytes, dimensions, class == mxOBJECT_CLASS, subsys) elseif class == mxSPARSE_CLASS data = read_sparse(f, swap_bytes, dimensions, flags) elseif class == mxCHAR_CLASS && length(dimensions) <= 2 data = read_string(f, swap_bytes, dimensions) elseif class == mxFUNCTION_CLASS - data = read_matrix(f, swap_bytes) + data = read_matrix(f, swap_bytes, subsys) elseif class == mxOPAQUE_CLASS - data = read_opaque(f, swap_bytes) + data = read_opaque(f, swap_bytes, subsys) else if (flags[1] & (1 << 9)) != 0 # logical data = read_data(f, swap_bytes, Bool, dimensions) @@ -384,6 +386,7 @@ function read_matrix(f::IO, swap_bytes::Bool) return (name, data) end +# FIXME: read subsystem here # Open MAT file for reading matopen(ios::IOStream, endian_indicator::UInt16) = Matlabv5File(ios, endian_indicator == 0x494D) @@ -398,7 +401,7 @@ function read(matfile::Matlabv5File) subsys_offset = 0 end if subsys_offset != 0 - read_subsystem(matfile.ios, matfile.swap_bytes, subsys_offset) + read_subsystem!(matfile, subsys_offset) end seek(matfile.ios, 128) @@ -410,19 +413,21 @@ function read(matfile::Matlabv5File) skip(matfile.ios, nbytes) continue end - (name, data) = read_matrix(matfile.ios, matfile.swap_bytes) + (name, data) = read_matrix(matfile.ios, matfile.swap_bytes, matfile.subsystem) vars[name] = data end vars end -function read_subsystem(ios::IOStream, swap_bytes::Bool, offset::UInt64) +function read_subsystem!(matfile::Matlabv5File, offset::UInt64) + ios = matfile.ios + swap_bytes = matfile.swap_bytes seek(ios, offset) - (_, subsystem_data) = read_matrix(ios, swap_bytes) + (_, subsystem_data) = read_matrix(ios, swap_bytes, matfile.subsystem) buf = IOBuffer(vec(subsystem_data)) seek(buf, 8) # Skip subsystem header - _, subsys_data = read_matrix(buf, swap_bytes) - MAT_subsys.load_subsys!(subsys_data, swap_bytes) + _, subsys_data = read_matrix(buf, swap_bytes, matfile.subsystem) + MAT_subsys.load_subsys!(matfile.subsystem, subsys_data, swap_bytes) end # Read only variable names from an HDF5 file @@ -464,7 +469,7 @@ function read(matfile::Matlabv5File, varname::String) error("no variable $varname in file") end seek(matfile.ios, varnames[varname]) - (name, data) = read_matrix(matfile.ios, matfile.swap_bytes) + (name, data) = read_matrix(matfile.ios, matfile.swap_bytes, matfile.subsystem) data end From 2e6b84807c274555b6d36d10b529fcb91d04df86 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 13:26:50 +0100 Subject: [PATCH 05/20] test read(matopen(filepath), ::String) for opaque subsystems --- src/MAT_HDF5.jl | 9 +++++++++ src/MAT_v5.jl | 38 ++++++++++++++++++++++++-------------- test/read.jl | 15 +++++++++++++-- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index a7443ff..852fa9d 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -55,6 +55,15 @@ mutable struct MatlabHDF5File <: HDF5.H5DataStore end end +function Base.show(io::IO, f::MatlabHDF5File) + print(io, "MatlabHDF5File(") + print(io, f.plain, ", ") + print(io, f.toclose, ", ") + print(io, f.writeheader, ", ") + print(io, f.refcounter, ", ") + print(io, f.compress, ")") +end + """ close(matfile_handle) diff --git a/src/MAT_v5.jl b/src/MAT_v5.jl index 943ba68..8293cae 100644 --- a/src/MAT_v5.jl +++ b/src/MAT_v5.jl @@ -39,9 +39,16 @@ mutable struct Matlabv5File <: HDF5.H5DataStore ios::IOStream swap_bytes::Bool subsystem::Subsystem + subsystem_position::UInt64 # nr of bytes taken by subsystem varnames::Dict{String, Int64} - Matlabv5File(ios, swap_bytes) = new(ios, swap_bytes, Subsystem()) + Matlabv5File(ios, swap_bytes) = new(ios, swap_bytes, Subsystem(), UInt64(0)) +end + +function Base.show(io::IO, f::Matlabv5File) + print(io, "Matlabv5File(") + print(io, f.ios, ", ") + print(io, f.swap_bytes, ")") end const miINT8 = 1 @@ -386,28 +393,31 @@ function read_matrix(f::IO, swap_bytes::Bool, subsys::Subsystem) return (name, data) end -# FIXME: read subsystem here # Open MAT file for reading -matopen(ios::IOStream, endian_indicator::UInt16) = - Matlabv5File(ios, endian_indicator == 0x494D) - -# Read whole MAT file -function read(matfile::Matlabv5File) - vars = Dict{String, Any}() +function matopen(ios::IOStream, endian_indicator::UInt16) + matfile = Matlabv5File(ios, endian_indicator == 0x494D) seek(matfile.ios, 116) subsys_offset = read_bswap(matfile.ios, matfile.swap_bytes, UInt64) if subsys_offset == 0x2020202020202020 - subsys_offset = 0 + subsys_offset = UInt64(0) end if subsys_offset != 0 - read_subsystem!(matfile, subsys_offset) + matfile.subsystem_position = subsys_offset + read_subsystem!(matfile) end + return matfile +end + +# Read whole MAT file +function read(matfile::Matlabv5File) + vars = Dict{String, Any}() + seek(matfile.ios, 128) while !eof(matfile.ios) - offset = position(matfile.ios) - if offset == subsys_offset + pos = position(matfile.ios) + if pos == matfile.subsystem_position # Skip reading subsystem again (_, nbytes) = read_header(matfile.ios, matfile.swap_bytes) skip(matfile.ios, nbytes) @@ -419,10 +429,10 @@ function read(matfile::Matlabv5File) vars end -function read_subsystem!(matfile::Matlabv5File, offset::UInt64) +function read_subsystem!(matfile::Matlabv5File) ios = matfile.ios swap_bytes = matfile.swap_bytes - seek(ios, offset) + seek(ios, matfile.subsystem_position) (_, subsystem_data) = read_matrix(ios, swap_bytes, matfile.subsystem) buf = IOBuffer(vec(subsystem_data)) seek(buf, 8) # Skip subsystem header diff --git a/test/read.jl b/test/read.jl index b5b1519..47e647a 100644 --- a/test/read.jl +++ b/test/read.jl @@ -230,8 +230,19 @@ end for format in ["v7", "v7.3"] let objtestfile = "struct_table_datetime.mat" - vars = matread(joinpath(dirname(@__FILE__), format, objtestfile))["s"] - @test "testTable" in keys(vars) + filepath = joinpath(dirname(@__FILE__), format, objtestfile) + + # make sure read(matopen(filepath), ::String) works + fid = matopen(filepath) + @test haskey(fid, "s") + vars = read(fid, "s") + @test haskey(vars, "testTable") + @test haskey(vars, "testDatetime") + close(fid) + + # matread interface + vars = matread(filepath)["s"] + @test haskey(vars, "testTable") @test size(vars["testTable"]) == (1, 1) @test Set(keys(vars["testTable"][1, 1])) == Set(["props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) @test vars["testTable"][1, 1].class == "table" From 3090b7f2c853795e4df252a780b5090a4e4cc1b6 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 14:30:54 +0100 Subject: [PATCH 06/20] MatlabOpaque to_string conversion --- Project.toml | 2 ++ src/MAT.jl | 3 ++- src/MAT_subsys.jl | 19 +++++++++++-------- src/MAT_types.jl | 35 +++++++++++++++++++++++++++++++++++ test/read.jl | 42 ++++++++++++++++++++---------------------- test/types.jl | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 31 deletions(-) diff --git a/Project.toml b/Project.toml index a500550..cde276f 100644 --- a/Project.toml +++ b/Project.toml @@ -7,11 +7,13 @@ BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68" [compat] BufferedStreams = "0.4.1, 1" CodecZlib = "0.5, 0.6, 0.7" HDF5 = "0.16, 0.17" +StringEncodings = "0.3.7" julia = "1.6" [extras] diff --git a/src/MAT.jl b/src/MAT.jl index 6b95cf4..4b6d934 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -25,6 +25,7 @@ module MAT using HDF5, SparseArrays +import StringEncodings include("MAT_types.jl") using .MAT_types @@ -37,7 +38,7 @@ include("MAT_v4.jl") using .MAT_HDF5, .MAT_v5, .MAT_v4, .MAT_subsys export matopen, matread, matwrite, @read, @write -export MatlabStructArray, MatlabClassObject +export MatlabStructArray, MatlabClassObject, MatlabOpaque # Open a MATLAB file const HDF5_HEADER = UInt8[0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a] diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index 8fadb67..dd4a2a1 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -302,15 +302,18 @@ function load_mcos_object(metadata::Array{UInt32}, type_name::String, subsys::Su class_id = metadata[end, 1] classname = get_classname(subsys, class_id) - object_arr = Array{MatlabOpaque}(undef, convert(Vector{Int}, dims)...) - - for i = 1:length(object_arr) - oid = object_ids[i] - obj = get_object!(subsys, oid, classname) - object_arr[i] = obj + if nobjects == 1 + oid = object_ids[1] + return get_object!(subsys, oid, classname) + else + object_arr = Array{MatlabOpaque}(undef, convert(Vector{Int}, dims)...) + for i = 1:length(object_arr) + oid = object_ids[i] + obj = get_object!(subsys, oid, classname) + object_arr[i] = obj + end + return object_arr end - - return object_arr end end \ No newline at end of file diff --git a/src/MAT_types.jl b/src/MAT_types.jl index b34d01e..ca87626 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -28,8 +28,11 @@ module MAT_types + import StringEncodings + export MatlabStructArray, StructArrayField, convert_struct_array export MatlabClassObject + export MatlabOpaque # struct arrays are stored as columns per field name """ @@ -303,4 +306,36 @@ module MAT_types Base.iterate(m::MatlabOpaque) = iterate(m.d) Base.haskey(m::MatlabOpaque, k) = haskey(m.d, k) Base.get(m::MatlabOpaque, k, default) = get(m.d, k, default) + + # for reference: https://github.com/foreverallama/matio/blob/main/matio/utils/converters/matstring.py + function to_string(obj::MatlabOpaque, encoding::String = "UTF-16LE") + data = obj["any"] + if data[1, 1] != 1 + @warn "String saved from a different MAT-file version. Returning empty string" + return "" + end + ndims = data[1, 2] + shape = data[1, 3 : (2 + ndims)] + num_strings = prod(shape) + char_counts = data[1, (3 + ndims) : (2 + ndims + num_strings)] + byte_data = data[1, (3 + ndims + num_strings) : end] + bytes = reinterpret(UInt8, byte_data) + + strings = String[] + pos = 1 + + for char_count in char_counts + byte_length = char_count * 2 # UTF-16 encoding + extracted_bytes = bytes[pos : pos + byte_length - 1] + str = StringEncodings.decode(extracted_bytes, encoding) + push!(strings, str) + pos += byte_length + end + + if num_strings==1 + return first(strings) + else + return reshape(strings, shape...) + end + end end \ No newline at end of file diff --git a/test/read.jl b/test/read.jl index 47e647a..857b348 100644 --- a/test/read.jl +++ b/test/read.jl @@ -235,39 +235,37 @@ for format in ["v7", "v7.3"] # make sure read(matopen(filepath), ::String) works fid = matopen(filepath) @test haskey(fid, "s") - vars = read(fid, "s") - @test haskey(vars, "testTable") - @test haskey(vars, "testDatetime") + var_s = read(fid, "s") + @test haskey(var_s, "testTable") + @test haskey(var_s, "testDatetime") close(fid) # matread interface vars = matread(filepath)["s"] @test haskey(vars, "testTable") - @test size(vars["testTable"]) == (1, 1) - @test Set(keys(vars["testTable"][1, 1])) == Set(["props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) - @test vars["testTable"][1, 1].class == "table" - @test vars["testTable"][1, 1]["ndims"] === 2.0 - @test vars["testTable"][1, 1]["nvars"] === 5.0 - @test vars["testTable"][1, 1]["nrows"] === 3.0 - @test vars["testTable"][1, 1]["data"][1, 1] == reshape([1261.0, 547.0, 3489.0], 3, 1) - @test vars["testTable"][1, 1]["data"][1, 2][1, 1].class == "string" - @test vars["testTable"][1, 1]["data"][1, 3][1, 1].class == "datetime" - @test vars["testTable"][1, 1]["data"][1, 4][1, 1].class == "categorical" - @test vars["testTable"][1, 1]["data"][1, 5][1, 1].class == "string" + @test Set(keys(vars["testTable"])) == Set(["props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) + @test vars["testTable"].class == "table" + @test vars["testTable"]["ndims"] === 2.0 + @test vars["testTable"]["nvars"] === 5.0 + @test vars["testTable"]["nrows"] === 3.0 + @test vars["testTable"]["data"][1, 1] == reshape([1261.0, 547.0, 3489.0], 3, 1) + @test vars["testTable"]["data"][1, 2].class == "string" + @test vars["testTable"]["data"][1, 3].class == "datetime" + @test vars["testTable"]["data"][1, 4].class == "categorical" + @test vars["testTable"]["data"][1, 5].class == "string" @test "testDatetime" in keys(vars) - @test size(vars["testDatetime"]) == (1, 1) if format == "v7.3" - @test Set(keys(vars["testDatetime"][1, 1])) == Set(["tz", "data", "fmt", "isDateOnly"]) - @test vars["testDatetime"][1, 1]["isDateOnly"] === false + @test Set(keys(vars["testDatetime"])) == Set(["tz", "data", "fmt", "isDateOnly"]) + @test vars["testDatetime"]["isDateOnly"] === false else # MATLAB removed property "isDateOnly" in later versions - @test Set(keys(vars["testDatetime"][1, 1])) == Set(["tz", "data", "fmt"]) + @test Set(keys(vars["testDatetime"])) == Set(["tz", "data", "fmt"]) end - @test vars["testDatetime"][1, 1].class == "datetime" - @test vars["testDatetime"][1, 1]["tz"] === "" - @test vars["testDatetime"][1, 1]["fmt"] === "" - @test vars["testDatetime"][1, 1]["data"] === 1.575304969634e12 + 0.0im + @test vars["testDatetime"].class == "datetime" + @test vars["testDatetime"]["tz"] === "" + @test vars["testDatetime"]["fmt"] === "" + @test vars["testDatetime"]["data"] === 1.575304969634e12 + 0.0im end end diff --git a/test/types.jl b/test/types.jl index aaa87da..bc5225f 100644 --- a/test/types.jl +++ b/test/types.jl @@ -85,4 +85,39 @@ end wrong_arr = [MatlabClassObject(d, "TestClassOld"), MatlabClassObject(d, "Bah")] @test_throws ErrorException MatlabStructArray(wrong_arr) +end + +@testset "MatlabOpaque to_string" begin + dat = UInt64[ + 0x0000000000000001 + 0x0000000000000002 + 0x0000000000000003 + 0x0000000000000001 + 0x0000000000000005 + 0x0000000000000005 + 0x0000000000000005 + 0x0065006e006f004a + 0x006f007200420073 + 0x006d0053006e0077 + 0x0000006800740069 + ] + dat = reshape(dat, 1, length(str)) + obj = MatlabOpaque(Dict{String, Any}("any" => str), "string") + str = MAT.MAT_types.to_string(obj) + @test size(str) == (3,1) + @test vec(str) == ["Jones", "Brown", "Smith"] + + dat = [ + 0x0000000000000001 + 0x0000000000000002 + 0x0000000000000001 + 0x0000000000000001 + 0x0000000000000005 + 0x0065006e006f004a + 0x0000000000000073 + ] + dat = reshape(dat, 1, length(str)) + obj = MatlabOpaque(Dict{String, Any}("any" => str), "string") + str = MAT.MAT_types.to_string(obj) + @test str == "Jones" end \ No newline at end of file From e0c90552877997b3926cbe063c138d17e04c1209 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 15:29:36 +0100 Subject: [PATCH 07/20] MatlabOpaque to_datetime conversion --- Project.toml | 4 ++++ src/MAT.jl | 1 - src/MAT_types.jl | 28 ++++++++++++++++++++++++++++ test/types.jl | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index cde276f..51bef76 100644 --- a/Project.toml +++ b/Project.toml @@ -5,14 +5,18 @@ version = "0.11.0" [deps] BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68" [compat] BufferedStreams = "0.4.1, 1" CodecZlib = "0.5, 0.6, 0.7" +Dates = "1.11.0" HDF5 = "0.16, 0.17" +Printf = "1.11.0" StringEncodings = "0.3.7" julia = "1.6" diff --git a/src/MAT.jl b/src/MAT.jl index 4b6d934..25988bf 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -25,7 +25,6 @@ module MAT using HDF5, SparseArrays -import StringEncodings include("MAT_types.jl") using .MAT_types diff --git a/src/MAT_types.jl b/src/MAT_types.jl index ca87626..85d2d65 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -29,6 +29,7 @@ module MAT_types import StringEncodings + import Dates: Date, DateTime, Second, Millisecond, Nanosecond export MatlabStructArray, StructArrayField, convert_struct_array export MatlabClassObject @@ -310,6 +311,9 @@ module MAT_types # for reference: https://github.com/foreverallama/matio/blob/main/matio/utils/converters/matstring.py function to_string(obj::MatlabOpaque, encoding::String = "UTF-16LE") data = obj["any"] + if isnothing(dat) || isempty(data) + return String[] + end if data[1, 1] != 1 @warn "String saved from a different MAT-file version. Returning empty string" return "" @@ -338,4 +342,28 @@ module MAT_types return reshape(strings, shape...) end end + + function to_datetime(obj::MatlabOpaque) + dat = obj["data"] + if isnothing(dat) || isempty(dat) + return DateTime[] + end + if !isempty(obj["tz"]) + @warn "no timezone conversion yet for datetime objects" + end + #isdate = obj["isDateOnly"] # optional: convert to Date instead of DateTime? + if dat isa AbstractArray + return map(ms_to_datetime, dat) + else + return ms_to_datetime(dat) + end + end + + # is the complex part the submilliseconds? + ms_to_datetime(ms::Complex) = ms_to_datetime(real(ms)) + function ms_to_datetime(ms::Real) + s, ms_rem = fldmod(ms, 1_000) # whole seconds and remainder milliseconds + return DateTime(1970,1,1) + Second(s) + Millisecond(ms_rem) + end + end \ No newline at end of file diff --git a/test/types.jl b/test/types.jl index bc5225f..8576817 100644 --- a/test/types.jl +++ b/test/types.jl @@ -1,4 +1,5 @@ using MAT, Test +using Dates @testset "MatlabStructArray" begin d_arr = Dict{String, Any}[ @@ -120,4 +121,35 @@ end obj = MatlabOpaque(Dict{String, Any}("any" => str), "string") str = MAT.MAT_types.to_string(obj) @test str == "Jones" +end + +@testset "MatlabOpaque to_datetime" begin + d = Dict{String, Any}( + "tz" => "", + "data" => ComplexF64[ + 1482192000000.0+0.0im; + 1482278400000.0+0.0im; + 1482364800000.0+0.0im;; + ], + "fmt" => "", + "isDateOnly" => true, + ) + obj = MatlabOpaque(d, "datetime") + expected_dates = [ + DateTime(2016, 12, 20) # 20-Dec-2016 + DateTime(2016, 12, 21) # 21-Dec-2016 + DateTime(2016, 12, 22) # 22-Dec-2016 + ] + @test all(MAT.MAT_types.to_datetime(obj) .== expected_dates) + + d = Dict{String, Any}( + "tz" => "", + "data" => 1575304969634.0+0.0im, + "fmt" => "", + "isDateOnly" => false, + ) + obj = MatlabOpaque(d, "datetime") + # "02-Dec-2019 16:42:49" + expected_dt = DateTime(2019, 12, 2, 16, 42, 49) + @test MAT.MAT_types.to_datetime(obj) - expected_dt < Second(1) end \ No newline at end of file From bac5677eaef942664956cf8a97572952a8c26516 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 15:34:14 +0100 Subject: [PATCH 08/20] fix to_string test --- Project.toml | 2 -- src/MAT_types.jl | 4 ++-- test/types.jl | 8 ++++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index 51bef76..6fccfce 100644 --- a/Project.toml +++ b/Project.toml @@ -7,7 +7,6 @@ BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" -Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68" @@ -16,7 +15,6 @@ BufferedStreams = "0.4.1, 1" CodecZlib = "0.5, 0.6, 0.7" Dates = "1.11.0" HDF5 = "0.16, 0.17" -Printf = "1.11.0" StringEncodings = "0.3.7" julia = "1.6" diff --git a/src/MAT_types.jl b/src/MAT_types.jl index 85d2d65..a767970 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -311,7 +311,7 @@ module MAT_types # for reference: https://github.com/foreverallama/matio/blob/main/matio/utils/converters/matstring.py function to_string(obj::MatlabOpaque, encoding::String = "UTF-16LE") data = obj["any"] - if isnothing(dat) || isempty(data) + if isnothing(data) || isempty(data) return String[] end if data[1, 1] != 1 @@ -349,7 +349,7 @@ module MAT_types return DateTime[] end if !isempty(obj["tz"]) - @warn "no timezone conversion yet for datetime objects" + @warn "no timezone conversion yet for datetime objects. timezone ignored" end #isdate = obj["isDateOnly"] # optional: convert to Date instead of DateTime? if dat isa AbstractArray diff --git a/test/types.jl b/test/types.jl index 8576817..b667d6d 100644 --- a/test/types.jl +++ b/test/types.jl @@ -102,8 +102,8 @@ end 0x006d0053006e0077 0x0000006800740069 ] - dat = reshape(dat, 1, length(str)) - obj = MatlabOpaque(Dict{String, Any}("any" => str), "string") + dat = reshape(dat, 1, length(dat)) + obj = MatlabOpaque(Dict{String, Any}("any" => dat), "string") str = MAT.MAT_types.to_string(obj) @test size(str) == (3,1) @test vec(str) == ["Jones", "Brown", "Smith"] @@ -117,8 +117,8 @@ end 0x0065006e006f004a 0x0000000000000073 ] - dat = reshape(dat, 1, length(str)) - obj = MatlabOpaque(Dict{String, Any}("any" => str), "string") + dat = reshape(dat, 1, length(dat)) + obj = MatlabOpaque(Dict{String, Any}("any" => dat), "string") str = MAT.MAT_types.to_string(obj) @test str == "Jones" end From fbde008f1e4d301e18643f38afae482951efe438 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 15:52:22 +0100 Subject: [PATCH 09/20] MatlabOpaque to_duration conversion --- src/MAT_types.jl | 32 ++++++++++++++++++++++++++------ test/types.jl | 25 +++++++++++++++++++++---- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/MAT_types.jl b/src/MAT_types.jl index a767970..8be1947 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -33,7 +33,7 @@ module MAT_types export MatlabStructArray, StructArrayField, convert_struct_array export MatlabClassObject - export MatlabOpaque + export MatlabOpaque, convert_opaque # struct arrays are stored as columns per field name """ @@ -308,6 +308,18 @@ module MAT_types Base.haskey(m::MatlabOpaque, k) = haskey(m.d, k) Base.get(m::MatlabOpaque, k, default) = get(m.d, k, default) + function convert_opaque(obj::MatlabOpaque) + if obj.class == "string" + return to_string(obj) + elseif obj.class == "datetime" + return to_datetime(obj) + elseif obj.class == "duration" + return to_duration(obj) + else + return obj + end + end + # for reference: https://github.com/foreverallama/matio/blob/main/matio/utils/converters/matstring.py function to_string(obj::MatlabOpaque, encoding::String = "UTF-16LE") data = obj["any"] @@ -352,11 +364,7 @@ module MAT_types @warn "no timezone conversion yet for datetime objects. timezone ignored" end #isdate = obj["isDateOnly"] # optional: convert to Date instead of DateTime? - if dat isa AbstractArray - return map(ms_to_datetime, dat) - else - return ms_to_datetime(dat) - end + return map_or_not(ms_to_datetime, dat) end # is the complex part the submilliseconds? @@ -366,4 +374,16 @@ module MAT_types return DateTime(1970,1,1) + Second(s) + Millisecond(ms_rem) end + function to_duration(obj::MatlabOpaque) + dat = obj["millis"] + #fmt = obj["fmt"] # TODO: format, e.g. 'd' to Day + if isnothing(dat) || isempty(dat) + return Millisecond[] + end + return map_or_not(Millisecond, dat) + end + + map_or_not(f, dat::AbstractArray) = map(f, dat) + map_or_not(f, dat) = f(dat) + end \ No newline at end of file diff --git a/test/types.jl b/test/types.jl index b667d6d..ff8b0d6 100644 --- a/test/types.jl +++ b/test/types.jl @@ -104,10 +104,11 @@ end ] dat = reshape(dat, 1, length(dat)) obj = MatlabOpaque(Dict{String, Any}("any" => dat), "string") - str = MAT.MAT_types.to_string(obj) + str = MAT.convert_opaque(obj) @test size(str) == (3,1) @test vec(str) == ["Jones", "Brown", "Smith"] + # single element string array is a single string in matlab, ofcourse dat = [ 0x0000000000000001 0x0000000000000002 @@ -119,7 +120,7 @@ end ] dat = reshape(dat, 1, length(dat)) obj = MatlabOpaque(Dict{String, Any}("any" => dat), "string") - str = MAT.MAT_types.to_string(obj) + str = MAT.convert_opaque(obj) @test str == "Jones" end @@ -140,7 +141,7 @@ end DateTime(2016, 12, 21) # 21-Dec-2016 DateTime(2016, 12, 22) # 22-Dec-2016 ] - @test all(MAT.MAT_types.to_datetime(obj) .== expected_dates) + @test all(MAT.convert_opaque(obj) .== expected_dates) d = Dict{String, Any}( "tz" => "", @@ -151,5 +152,21 @@ end obj = MatlabOpaque(d, "datetime") # "02-Dec-2019 16:42:49" expected_dt = DateTime(2019, 12, 2, 16, 42, 49) - @test MAT.MAT_types.to_datetime(obj) - expected_dt < Second(1) + @test MAT.convert_opaque(obj) - expected_dt < Second(1) +end + +@testset "MatlabOpaque to_duration" begin + d = Dict( + "millis" => [3.6e6 7.2e6], + "fmt" => 'h', + ) + obj = MatlabOpaque(d, "duration") + @test MAT.convert_opaque(obj) == map(Millisecond, d["millis"]) + + d = Dict( + "millis" => 12000.0, + "fmt" => 'h', + ) + obj = MatlabOpaque(d, "duration") + @test MAT.convert_opaque(obj) == Millisecond(d["millis"]) end \ No newline at end of file From 15e3d626a83a4084eb594007c4cae9fc60858043 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Mon, 17 Nov 2025 16:07:41 +0100 Subject: [PATCH 10/20] automatic MatlabOpaque conversion --- src/MAT_subsys.jl | 9 +++++---- test/read.jl | 26 ++++++++++---------------- test/types.jl | 3 ++- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index dd4a2a1..e59f158 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -24,7 +24,7 @@ module MAT_subsys -import ..MAT_types: MatlabStructArray, MatlabOpaque +import ..MAT_types: MatlabStructArray, MatlabOpaque, convert_opaque export Subsystem @@ -304,13 +304,14 @@ function load_mcos_object(metadata::Array{UInt32}, type_name::String, subsys::Su if nobjects == 1 oid = object_ids[1] - return get_object!(subsys, oid, classname) + obj = get_object!(subsys, oid, classname) + return convert_opaque(obj) else - object_arr = Array{MatlabOpaque}(undef, convert(Vector{Int}, dims)...) + object_arr = Array{Any}(undef, convert(Vector{Int}, dims)...) for i = 1:length(object_arr) oid = object_ids[i] obj = get_object!(subsys, oid, classname) - object_arr[i] = obj + object_arr[i] = convert_opaque(obj) end return object_arr end diff --git a/test/read.jl b/test/read.jl index 857b348..9b41e86 100644 --- a/test/read.jl +++ b/test/read.jl @@ -1,4 +1,5 @@ using MAT, Test +using Dates function check(filename, result) matfile = matopen(filename) @@ -248,24 +249,17 @@ for format in ["v7", "v7.3"] @test vars["testTable"]["ndims"] === 2.0 @test vars["testTable"]["nvars"] === 5.0 @test vars["testTable"]["nrows"] === 3.0 - @test vars["testTable"]["data"][1, 1] == reshape([1261.0, 547.0, 3489.0], 3, 1) - @test vars["testTable"]["data"][1, 2].class == "string" - @test vars["testTable"]["data"][1, 3].class == "datetime" - @test vars["testTable"]["data"][1, 4].class == "categorical" - @test vars["testTable"]["data"][1, 5].class == "string" + @test vars["testTable"]["data"][1] == reshape([1261.0, 547.0, 3489.0], 3, 1) + @test vars["testTable"]["data"][2] isa Matrix{String} + @test vars["testTable"]["data"][3] isa Matrix{DateTime} + @test vars["testTable"]["data"][4].class == "categorical" + @test vars["testTable"]["data"][5] isa Matrix{String} + @test all(x->length(x)==3, vars["testTable"]["data"][[1,2,3,5]]) @test "testDatetime" in keys(vars) - if format == "v7.3" - @test Set(keys(vars["testDatetime"])) == Set(["tz", "data", "fmt", "isDateOnly"]) - @test vars["testDatetime"]["isDateOnly"] === false - else - # MATLAB removed property "isDateOnly" in later versions - @test Set(keys(vars["testDatetime"])) == Set(["tz", "data", "fmt"]) - end - @test vars["testDatetime"].class == "datetime" - @test vars["testDatetime"]["tz"] === "" - @test vars["testDatetime"]["fmt"] === "" - @test vars["testDatetime"]["data"] === 1.575304969634e12 + 0.0im + dt = vars["testDatetime"] + @test dt isa DateTime + @test dt - DateTime(2019, 12, 2, 16, 42, 49) < Second(1) end end diff --git a/test/types.jl b/test/types.jl index ff8b0d6..a5107a0 100644 --- a/test/types.jl +++ b/test/types.jl @@ -133,7 +133,7 @@ end 1482364800000.0+0.0im;; ], "fmt" => "", - "isDateOnly" => true, + "isDateOnly" => true, # Note: "isDateOnly" not in all versions ) obj = MatlabOpaque(d, "datetime") expected_dates = [ @@ -152,6 +152,7 @@ end obj = MatlabOpaque(d, "datetime") # "02-Dec-2019 16:42:49" expected_dt = DateTime(2019, 12, 2, 16, 42, 49) + # still have some millisecond rounding issue? @test MAT.convert_opaque(obj) - expected_dt < Second(1) end From 113912045dcd95df88c666a2a32f9f72f2235b38 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Tue, 18 Nov 2025 09:42:30 +0100 Subject: [PATCH 11/20] attempt to fix ci: change Dates compat and more explicit type writing --- Project.toml | 2 +- src/MAT_HDF5.jl | 19 ++++++++++++++++--- test/write.jl | 7 +++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index 6fccfce..79bdb3d 100644 --- a/Project.toml +++ b/Project.toml @@ -13,7 +13,7 @@ StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68" [compat] BufferedStreams = "0.4.1, 1" CodecZlib = "0.5, 0.6, 0.7" -Dates = "1.11.0" +Dates = "1" HDF5 = "0.16, 0.17" StringEncodings = "0.3.7" julia = "1.6" diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 852fa9d..1a32715 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -33,7 +33,8 @@ using ..MAT_subsys import Base: names, read, write, close import HDF5: Reference -import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject +import Dates +import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque const HDF5Parent = Union{HDF5.File, HDF5.Group} const HDF5BitsOrBool = Union{HDF5.BitsType,Bool} @@ -601,8 +602,7 @@ end # Struct array: Array of Dict => MATLAB struct array -function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, - arr::AbstractArray{<:AbstractDict}) +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, arr::AbstractArray{<:AbstractDict}) m_write(mfile, parent, name, MatlabStructArray(arr)) end @@ -695,6 +695,11 @@ end m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s::AbstractDict) = m_write(mfile, parent, name, check_struct_keys(collect(keys(s))), collect(values(s))) +# Write named tuple as a struct +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, nt::NamedTuple) + m_write(mfile, parent, name, [string(x) for x in keys(nt)], collect(nt)) +end + # Write generic CompositeKind as a struct function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s) if isbits(s) @@ -704,6 +709,14 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s) m_write(mfile, parent, name, check_struct_keys([string(x) for x in fieldnames(T)]), [getfield(s, x) for x in fieldnames(T)]) end +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, dat::Dates.AbstractTime) + error("writing of Dates types is not yet supported") +end + +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::MatlabOpaque) + error("writing of MatlabOpaque types is not yet supported") +end + # Check whether a variable name is valid, then write it """ write(matfile_handle, varname, value) diff --git a/test/write.jl b/test/write.jl index 02acc62..21f32b3 100644 --- a/test/write.jl +++ b/test/write.jl @@ -152,6 +152,13 @@ test_write(Dict("reshape_arr"=>reshape([1 2 3;4 5 6;7 8 9]',1,9))) test_write(Dict("adjoint_arr"=>Any[1 2 3;4 5 6;7 8 9]')) test_write(Dict("reshape_arr"=>reshape(Any[1 2 3;4 5 6;7 8 9]',1,9))) +# named tuple +nt = (x = 5, y = Any[6, "string"]) +matwrite(tmpfile, Dict("nt" => nt)) +nt_read = matread(tmpfile)["nt"] +@test nt_read["x"] == 5 +@test nt_read["y"] == nt.y + # test nested struct array - interface via Dict array @testset "MatlabStructArray writing" begin sarr = Dict{String, Any}[ From 4b95172dd107f4975b4542cb7c5ad87a9390961d Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Tue, 18 Nov 2025 10:21:47 +0100 Subject: [PATCH 12/20] MatlabOpaque categorical support --- Project.toml | 2 ++ src/MAT_HDF5.jl | 4 ++++ src/MAT_types.jl | 39 +++++++++++++++++++++++++++++++-------- test/read.jl | 4 ++-- test/types.jl | 24 +++++++++++++++++++++--- 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/Project.toml b/Project.toml index 79bdb3d..d45e1e4 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" +PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68" @@ -15,6 +16,7 @@ BufferedStreams = "0.4.1, 1" CodecZlib = "0.5, 0.6, 0.7" Dates = "1" HDF5 = "0.16, 0.17" +PooledArrays = "1.4.3" StringEncodings = "0.3.7" julia = "1.6" diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 1a32715..5fa35f1 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -717,6 +717,10 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::M error("writing of MatlabOpaque types is not yet supported") end +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::AbstractArray{MatlabOpaque}) + error("writing of MatlabOpaque types is not yet supported") +end + # Check whether a variable name is valid, then write it """ write(matfile_handle, varname, value) diff --git a/src/MAT_types.jl b/src/MAT_types.jl index 8be1947..442916f 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -29,7 +29,8 @@ module MAT_types import StringEncodings - import Dates: Date, DateTime, Second, Millisecond, Nanosecond + import Dates: DateTime, Second, Millisecond + import PooledArrays: PooledArray, RefArray export MatlabStructArray, StructArrayField, convert_struct_array export MatlabClassObject @@ -310,18 +311,20 @@ module MAT_types function convert_opaque(obj::MatlabOpaque) if obj.class == "string" - return to_string(obj) + return from_string(obj) elseif obj.class == "datetime" - return to_datetime(obj) + return from_datetime(obj) elseif obj.class == "duration" - return to_duration(obj) + return from_duration(obj) + elseif obj.class == "categorical" + return from_categorical(obj) else return obj end end # for reference: https://github.com/foreverallama/matio/blob/main/matio/utils/converters/matstring.py - function to_string(obj::MatlabOpaque, encoding::String = "UTF-16LE") + function from_string(obj::MatlabOpaque, encoding::String = "UTF-16LE") data = obj["any"] if isnothing(data) || isempty(data) return String[] @@ -331,7 +334,7 @@ module MAT_types return "" end ndims = data[1, 2] - shape = data[1, 3 : (2 + ndims)] + shape = Int.(data[1, 3 : (2 + ndims)]) num_strings = prod(shape) char_counts = data[1, (3 + ndims) : (2 + ndims + num_strings)] byte_data = data[1, (3 + ndims + num_strings) : end] @@ -355,7 +358,7 @@ module MAT_types end end - function to_datetime(obj::MatlabOpaque) + function from_datetime(obj::MatlabOpaque) dat = obj["data"] if isnothing(dat) || isempty(dat) return DateTime[] @@ -374,7 +377,7 @@ module MAT_types return DateTime(1970,1,1) + Second(s) + Millisecond(ms_rem) end - function to_duration(obj::MatlabOpaque) + function from_duration(obj::MatlabOpaque) dat = obj["millis"] #fmt = obj["fmt"] # TODO: format, e.g. 'd' to Day if isnothing(dat) || isempty(dat) @@ -383,6 +386,26 @@ module MAT_types return map_or_not(Millisecond, dat) end + function from_categorical(obj::MatlabOpaque) + category_names = obj["categoryNames"] + codes = obj["codes"] + pool = vec(Array{element_type(category_names)}(category_names)) + code_type = eltype(codes) + invpool = Dict{eltype(pool), code_type}(pool .=> code_type.(1:length(pool))) + refs = RefArray(codes) + return PooledArray(refs, invpool, pool) + end + + function element_type(v::AbstractArray{T}) where T + isempty(v) && return T + first_el, remaining = Iterators.peel(v) + T_out = typeof(first_el) + for el in remaining + T_out = promote_type(T_out, typeof(el)) + end + return T_out + end + map_or_not(f, dat::AbstractArray) = map(f, dat) map_or_not(f, dat) = f(dat) diff --git a/test/read.jl b/test/read.jl index 9b41e86..483fbf8 100644 --- a/test/read.jl +++ b/test/read.jl @@ -252,9 +252,9 @@ for format in ["v7", "v7.3"] @test vars["testTable"]["data"][1] == reshape([1261.0, 547.0, 3489.0], 3, 1) @test vars["testTable"]["data"][2] isa Matrix{String} @test vars["testTable"]["data"][3] isa Matrix{DateTime} - @test vars["testTable"]["data"][4].class == "categorical" + @test vars["testTable"]["data"][4] isa AbstractMatrix{String} @test vars["testTable"]["data"][5] isa Matrix{String} - @test all(x->length(x)==3, vars["testTable"]["data"][[1,2,3,5]]) + @test all(x->size(x)==(3,1), vars["testTable"]["data"]) @test "testDatetime" in keys(vars) dt = vars["testDatetime"] diff --git a/test/types.jl b/test/types.jl index a5107a0..8be4e56 100644 --- a/test/types.jl +++ b/test/types.jl @@ -88,7 +88,7 @@ end @test_throws ErrorException MatlabStructArray(wrong_arr) end -@testset "MatlabOpaque to_string" begin +@testset "MatlabOpaque string" begin dat = UInt64[ 0x0000000000000001 0x0000000000000002 @@ -124,7 +124,7 @@ end @test str == "Jones" end -@testset "MatlabOpaque to_datetime" begin +@testset "MatlabOpaque datetime" begin d = Dict{String, Any}( "tz" => "", "data" => ComplexF64[ @@ -156,7 +156,7 @@ end @test MAT.convert_opaque(obj) - expected_dt < Second(1) end -@testset "MatlabOpaque to_duration" begin +@testset "MatlabOpaque duration" begin d = Dict( "millis" => [3.6e6 7.2e6], "fmt" => 'h', @@ -170,4 +170,22 @@ end ) obj = MatlabOpaque(d, "duration") @test MAT.convert_opaque(obj) == Millisecond(d["millis"]) +end + +@testset "MatlabOpaque categorical" begin + d = Dict( + "isProtected" => false, + "codes" => UInt8[0x02; 0x03; 0x01;; 0x01; 0x01; 0x02], + "categoryNames" => Any["Fair"; "Good"; "Poor";;], + "isOrdinal" => false, + ) + obj = MatlabOpaque(d, "categorical") + + c = MAT.convert_opaque(obj) + @test c == [ + "Good" "Fair" + "Poor" "Fair" + "Fair" "Good" + ] + end \ No newline at end of file From edefb99616a42a4b544585a5046ea1658a4bd359 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Tue, 18 Nov 2025 16:59:58 +0100 Subject: [PATCH 13/20] MatlabOpaque table support --- Project.toml | 2 ++ src/MAT.jl | 80 ++++++++++++++++++++++++++++++++++++++--------- src/MAT_HDF5.jl | 8 +++-- src/MAT_subsys.jl | 8 +++-- src/MAT_types.jl | 52 +++++++++++++++++++++++++++++- src/MAT_v5.jl | 6 ++-- test/read.jl | 34 +++++++++++++------- test/types.jl | 29 +++++++++++++++++ 8 files changed, 185 insertions(+), 34 deletions(-) diff --git a/Project.toml b/Project.toml index d45e1e4..5812ae9 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] BufferedStreams = "0.4.1, 1" @@ -18,6 +19,7 @@ Dates = "1" HDF5 = "0.16, 0.17" PooledArrays = "1.4.3" StringEncodings = "0.3.7" +Tables = "1.12.1" julia = "1.6" [extras] diff --git a/src/MAT.jl b/src/MAT.jl index 25988bf..ea95e5f 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -37,15 +37,15 @@ include("MAT_v4.jl") using .MAT_HDF5, .MAT_v5, .MAT_v4, .MAT_subsys export matopen, matread, matwrite, @read, @write -export MatlabStructArray, MatlabClassObject, MatlabOpaque +export MatlabStructArray, MatlabClassObject, MatlabOpaque, MatlabTable # Open a MATLAB file const HDF5_HEADER = UInt8[0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a] -function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool) +function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool; table::Type=MatlabTable) # When creating new files, create as HDF5 by default fs = filesize(filename) if cr && (tr || fs == 0) - return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, Base.ENDIAN_BOM == 0x04030201) + return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, Base.ENDIAN_BOM == 0x04030201; table=table) elseif fs == 0 error("File \"$filename\" does not exist and create was not specified") end @@ -73,7 +73,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo if wr || cr || tr || ff error("creating or appending to MATLAB v5 files is not supported") end - return MAT_v5.matopen(rawfid, endian_indicator) + return MAT_v5.matopen(rawfid, endian_indicator; table=table) end # Check for HDF5 file @@ -81,7 +81,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo seek(rawfid, offset) if read!(rawfid, Vector{UInt8}(undef, 8)) == HDF5_HEADER close(rawfid) - return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, endian_indicator == 0x494D) + return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, endian_indicator == 0x494D; table=table) end end @@ -89,10 +89,10 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo error("\"$filename\" is not a MAT file") end -function matopen(fname::AbstractString, mode::AbstractString; compress::Bool = false) - mode == "r" ? matopen(fname, true , false, false, false, false, false) : - mode == "r+" ? matopen(fname, true , true , false, false, false, compress) : - mode == "w" ? matopen(fname, false, true , true , true , false, compress) : +function matopen(fname::AbstractString, mode::AbstractString; compress::Bool = false, table::Type = MatlabTable) + mode == "r" ? matopen(fname, true , false, false, false, false, false; table=table) : + mode == "r+" ? matopen(fname, true , true , false, false, false, compress; table=table) : + mode == "w" ? matopen(fname, false, true , true , true , false, compress; table=table) : # mode == "w+" ? matopen(fname, true , true , true , true , false, compress) : # mode == "a" ? matopen(fname, false, true , true , false, true, compress) : # mode == "a+" ? matopen(fname, true , true , true , false, true, compress) : @@ -111,8 +111,8 @@ function matopen(f::Function, args...; kwargs...) end """ - matopen(filename [, mode]; compress = false) -> handle - matopen(f::Function, filename [, mode]; compress = false) -> f(handle) + matopen(filename [, mode]; compress = false, table = MatlabTable) -> handle + matopen(f::Function, filename [, mode]; compress = false, table = MatlabTable) -> f(handle) Mode defaults to `"r"` for read. It can also be `"w"` for write, @@ -122,18 +122,70 @@ Compression on reading is detected/handled automatically; the `compress` keyword argument only affects write operations. Use with `read`, `write`, `close`, `keys`, and `haskey`. + +Optional keyword argument is the `table` type, for automatic conversion of Matlab Tables. + +# Example + +```julia +using MAT, DataFrames +filepath = abspath(pkgdir(MAT), "./test/v7.3/struct_table_datetime.mat") +fid = matopen(filepath; table = DataFrame) +keys(fid) + +# outputs + +1-element Vector{String}: + "s" + +``` + +Now you can read any of the keys +``` +s = read(fid, "s") +close(fid) +s + +# outputs + +Dict{String, Any} with 2 entries: + "testDatetime" => DateTime("2019-12-02T16:42:49.634") + "testTable" => 3×5 DataFrame… + +``` """ matopen # Read all variables from a MATLAB file """ - matread(filename) -> Dict + matread(filename; table = MatlabTable) -> Dict Return a dictionary of all the variables and values in a Matlab file, opening and closing it automatically. + +Optionally provide the `table` type to convert Matlab tables into. Default uses a simple `MatlabTable` type. + +# Example + +```julia +using MAT, DataFrames +filepath = abspath(pkgdir(MAT), "./test/v7.3/struct_table_datetime.mat") +vars = matread(filepath; table = DataFrame) +vars["s"]["testTable"] + +# outputs + +3×5 DataFrame + Row │ FlightNum Customer Date Rating Comment + │ Float64 String DateTime String String +─────┼───────────────────────────────────────────────────────────────────────────────────── + 1 │ 1261.0 Jones 2016-12-20T00:00:00 Good Flight left on time, not crowded + 2 │ 547.0 Brown 2016-12-21T00:00:00 Poor Late departure, ran out of dinne… + 3 │ 3489.0 Smith 2016-12-22T00:00:00 Fair Late, but only by half an hour. … +``` """ -function matread(filename::AbstractString) - file = matopen(filename) +function matread(filename::AbstractString; table::Type=MatlabTable) + file = matopen(filename; table=table) local vars try vars = read(file) diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 5fa35f1..d6497fb 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -34,7 +34,8 @@ using ..MAT_subsys import Base: names, read, write, close import HDF5: Reference import Dates -import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque +import Tables +import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable const HDF5Parent = Union{HDF5.File, HDF5.Group} const HDF5BitsOrBool = Union{HDF5.BitsType,Bool} @@ -98,7 +99,7 @@ function close(f::MatlabHDF5File) nothing end -function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool, endian_indicator::Bool) +function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool, endian_indicator::Bool; table::Type=MatlabTable) local f if ff && !wr error("Cannot append to a read-only file") @@ -129,6 +130,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo end subsys_refs = "#subsystem#" if haskey(fid.plain, subsys_refs) + fid.subsystem.table_type = table subsys_data = m_read(fid.plain[subsys_refs], fid.subsystem) MAT_subsys.load_subsys!(fid.subsystem, subsys_data, endian_indicator) end @@ -704,6 +706,8 @@ end function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s) if isbits(s) error("This is the write function for CompositeKind, but the input doesn't fit") + elseif Tables.istable(s) + error("writing tables is not yet supported") end T = typeof(s) m_write(mfile, parent, name, check_struct_keys([string(x) for x in fieldnames(T)]), [getfield(s, x) for x in fieldnames(T)]) diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index e59f158..dbdab35 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -47,6 +47,7 @@ mutable struct Subsystem prop_vals_defaults::Any handle_data::Any java_data::Any + table_type::Type Subsystem() = new( Dict{UInt32, MatlabOpaque}(), @@ -64,7 +65,8 @@ mutable struct Subsystem nothing, nothing, nothing, - nothing + nothing, + Nothing ) end @@ -305,13 +307,13 @@ function load_mcos_object(metadata::Array{UInt32}, type_name::String, subsys::Su if nobjects == 1 oid = object_ids[1] obj = get_object!(subsys, oid, classname) - return convert_opaque(obj) + return convert_opaque(obj; table=subsys.table_type) else object_arr = Array{Any}(undef, convert(Vector{Int}, dims)...) for i = 1:length(object_arr) oid = object_ids[i] obj = get_object!(subsys, oid, classname) - object_arr[i] = convert_opaque(obj) + object_arr[i] = convert_opaque(obj; table=subsys.table_type) end return object_arr end diff --git a/src/MAT_types.jl b/src/MAT_types.jl index 442916f..ba4e718 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -31,10 +31,12 @@ module MAT_types import StringEncodings import Dates: DateTime, Second, Millisecond import PooledArrays: PooledArray, RefArray + import Tables export MatlabStructArray, StructArrayField, convert_struct_array export MatlabClassObject export MatlabOpaque, convert_opaque + export MatlabTable # struct arrays are stored as columns per field name """ @@ -212,6 +214,21 @@ module MAT_types return D(T.(keys) .=> values) end + # 1D MatlabStructArray also counts as table (mostly for testing purposes) + Tables.istable(::Type{MatlabStructArray{1}}) = true + Tables.columns(t::MatlabStructArray{1}) = Symbol.(t.values) + Tables.columnnames(t::MatlabStructArray{1}) = t.names + Tables.getcolumn(t::MatlabStructArray{1}, nm::String) = t[nm] + Tables.getcolumn(t::MatlabStructArray{1}, nm::Symbol) = Tables.getcolumn(t, string(nm)) + function MatlabStructArray{1}(t::Tables.CopiedColumns) + col_names = Tables.columnnames(t) + MatlabStructArray{1}( + string.(col_names), + [Vector{Any}(Tables.getcolumn(t, nm)) for nm in col_names] + ) + end + MatlabStructArray(t::Tables.CopiedColumns) = MatlabStructArray{1}(t) + struct StructArrayField{N} values::Array{Any,N} end @@ -309,7 +326,7 @@ module MAT_types Base.haskey(m::MatlabOpaque, k) = haskey(m.d, k) Base.get(m::MatlabOpaque, k, default) = get(m.d, k, default) - function convert_opaque(obj::MatlabOpaque) + function convert_opaque(obj::MatlabOpaque; table::Type=Nothing) if obj.class == "string" return from_string(obj) elseif obj.class == "datetime" @@ -318,6 +335,8 @@ module MAT_types return from_duration(obj) elseif obj.class == "categorical" return from_categorical(obj) + elseif obj.class == "table" + return from_table(obj, table) else return obj end @@ -409,4 +428,35 @@ module MAT_types map_or_not(f, dat::AbstractArray) = map(f, dat) map_or_not(f, dat) = f(dat) + struct MatlabTable + names::Vector{Symbol} + columns::Vector + end + Tables.istable(::Type{MatlabTable}) = true + Tables.columns(t::MatlabTable) = t.columns + Tables.columnnames(t::MatlabTable) = t.names + Tables.getcolumn(t::MatlabTable, nm::Symbol) = t[nm] + function find_index(m::MatlabTable, s::Symbol) + idx = findfirst(isequal(s), m.names) + if isnothing(idx) + error("column :$s not found in MatlabTable") + end + return idx + end + function Base.getindex(m::MatlabTable, s::Symbol) + idx = find_index(m, s) + return getindex(m.columns, idx) + end + Base.getindex(m::MatlabTable, s::AbstractString) = getindex(m, Symbol(s)) + MatlabTable(t::Tables.CopiedColumns{MatlabTable}) = Tables.source(t) + + function from_table(obj::MatlabOpaque, ::Type{T} = MatlabTable) where T + names = vec(Symbol.(obj["varnames"])) + cols = vec([vec(c) for c in obj["data"]]) + t = MatlabTable(names, cols) + return T(Tables.CopiedColumns(t)) + end + # option to not convert and get the MatlabOpaque as table + from_table(obj::MatlabOpaque, ::Type{Nothing}) = obj + end \ No newline at end of file diff --git a/src/MAT_v5.jl b/src/MAT_v5.jl index 8293cae..8f9eed7 100644 --- a/src/MAT_v5.jl +++ b/src/MAT_v5.jl @@ -28,7 +28,7 @@ module MAT_v5 using CodecZlib, BufferedStreams, HDF5, SparseArrays import Base: read, write, close -import ..MAT_types: MatlabStructArray, MatlabClassObject +import ..MAT_types: MatlabStructArray, MatlabClassObject, MatlabTable using ..MAT_subsys @@ -394,7 +394,7 @@ function read_matrix(f::IO, swap_bytes::Bool, subsys::Subsystem) end # Open MAT file for reading -function matopen(ios::IOStream, endian_indicator::UInt16) +function matopen(ios::IOStream, endian_indicator::UInt16; table::Type=MatlabTable) matfile = Matlabv5File(ios, endian_indicator == 0x494D) seek(matfile.ios, 116) @@ -404,6 +404,7 @@ function matopen(ios::IOStream, endian_indicator::UInt16) end if subsys_offset != 0 matfile.subsystem_position = subsys_offset + matfile.subsystem.table_type = table read_subsystem!(matfile) end @@ -413,7 +414,6 @@ end # Read whole MAT file function read(matfile::Matlabv5File) vars = Dict{String, Any}() - seek(matfile.ios, 128) while !eof(matfile.ios) pos = position(matfile.ios) diff --git a/test/read.jl b/test/read.jl index 483fbf8..892d870 100644 --- a/test/read.jl +++ b/test/read.jl @@ -230,6 +230,7 @@ let objtestfile = "function_handles.mat" end for format in ["v7", "v7.3"] + @testset "struct_table_datetime $format" begin let objtestfile = "struct_table_datetime.mat" filepath = joinpath(dirname(@__FILE__), format, objtestfile) @@ -244,23 +245,34 @@ for format in ["v7", "v7.3"] # matread interface vars = matread(filepath)["s"] @test haskey(vars, "testTable") - @test Set(keys(vars["testTable"])) == Set(["props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) - @test vars["testTable"].class == "table" - @test vars["testTable"]["ndims"] === 2.0 - @test vars["testTable"]["nvars"] === 5.0 - @test vars["testTable"]["nrows"] === 3.0 - @test vars["testTable"]["data"][1] == reshape([1261.0, 547.0, 3489.0], 3, 1) - @test vars["testTable"]["data"][2] isa Matrix{String} - @test vars["testTable"]["data"][3] isa Matrix{DateTime} - @test vars["testTable"]["data"][4] isa AbstractMatrix{String} - @test vars["testTable"]["data"][5] isa Matrix{String} - @test all(x->size(x)==(3,1), vars["testTable"]["data"]) + t = vars["testTable"] + @test t isa MatlabTable + @test t.names == [:FlightNum, :Customer, :Date, :Rating, :Comment] + @test t[:Date] isa Vector{DateTime} + @test t[:Rating] isa AbstractVector{String} + @test all(x->length(x)==3, t.columns) + + # using Nothing will keep the MatlabOpaque + vars = matread(filepath; table=Nothing)["s"] + t = vars["testTable"] + @test Set(keys(t)) == Set(["props", "varnames", "nrows", "data", "rownames", "ndims", "nvars"]) + @test t.class == "table" + @test t["ndims"] === 2.0 + @test t["nvars"] === 5.0 + @test t["nrows"] === 3.0 + @test t["data"][1] == reshape([1261.0, 547.0, 3489.0], 3, 1) + @test t["data"][2] isa Matrix{String} + @test t["data"][3] isa Matrix{DateTime} + @test t["data"][4] isa AbstractMatrix{String} + @test t["data"][5] isa Matrix{String} + @test all(x->size(x)==(3,1), t["data"]) @test "testDatetime" in keys(vars) dt = vars["testDatetime"] @test dt isa DateTime @test dt - DateTime(2019, 12, 2, 16, 42, 49) < Second(1) end + end end # test reading of old-style Matlab object in v7.3 format diff --git a/test/types.jl b/test/types.jl index 8be4e56..352ac0a 100644 --- a/test/types.jl +++ b/test/types.jl @@ -188,4 +188,33 @@ end "Fair" "Good" ] +end + +@testset "MatlabOpaque table" begin + # simplified table struct; there's some other properties as well + d = Dict{String,Any}( + "varnames" => Any["FlightNum" "Customer"], + "nrows" => 3.0, + "data" => reshape(Any[[1261.0; 547.0; 3489.0;;], ["Jones"; "Brown"; "Smith";;]], 1, 2), + "ndims" => 2.0, + "nvars" => 2.0, + ) + obj = MatlabOpaque(d, "table") + + t = MAT.convert_opaque(obj; table = MatlabTable) + @test t.names == [:FlightNum, :Customer] + @test t[:FlightNum] isa Vector{Float64} + @test t[:FlightNum] == [1261.0, 547.0, 3489.0] + @test t[:Customer] isa Vector{String} + @test t["Customer"] == ["Jones", "Brown", "Smith"] + + t = MAT.convert_opaque(obj; table = MatlabStructArray{1}) + @test t isa MatlabStructArray{1} + @test t["FlightNum"] == [1261.0, 547.0, 3489.0] + @test t["Customer"] == ["Jones", "Brown", "Smith"] + + t = MAT.convert_opaque(obj; table = Nothing) + @test t === obj + + # Note: this should all work with DataFrames.DataFrame, but that's a big dependency to add for testing end \ No newline at end of file From 386f67fa3169cc4b1536e20797a6d438cd495cd4 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Wed, 19 Nov 2025 09:28:03 +0100 Subject: [PATCH 14/20] Support for single row tables and ND columns --- src/MAT.jl | 3 ++- src/MAT_types.jl | 13 ++++++++++--- test/types.jl | 31 ++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/MAT.jl b/src/MAT.jl index ea95e5f..ae542d5 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -123,7 +123,8 @@ keyword argument only affects write operations. Use with `read`, `write`, `close`, `keys`, and `haskey`. -Optional keyword argument is the `table` type, for automatic conversion of Matlab Tables. +Optional keyword argument is the `table` type, for automatic conversion of Matlab tables. +Note that Matlab tables may contain non-vector colums which cannot always be converted to a Julia table, like `DataFrame`. # Example diff --git a/src/MAT_types.jl b/src/MAT_types.jl index ba4e718..ec01e84 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -408,14 +408,14 @@ module MAT_types function from_categorical(obj::MatlabOpaque) category_names = obj["categoryNames"] codes = obj["codes"] - pool = vec(Array{element_type(category_names)}(category_names)) + pool = vec(Array{promoted_eltype(category_names)}(category_names)) code_type = eltype(codes) invpool = Dict{eltype(pool), code_type}(pool .=> code_type.(1:length(pool))) refs = RefArray(codes) return PooledArray(refs, invpool, pool) end - function element_type(v::AbstractArray{T}) where T + function promoted_eltype(v::AbstractArray{Any}) isempty(v) && return T first_el, remaining = Iterators.peel(v) T_out = typeof(first_el) @@ -424,6 +424,7 @@ module MAT_types end return T_out end + promoted_eltype(::AbstractArray{T}) where T = T map_or_not(f, dat::AbstractArray) = map(f, dat) map_or_not(f, dat) = f(dat) @@ -452,11 +453,17 @@ module MAT_types function from_table(obj::MatlabOpaque, ::Type{T} = MatlabTable) where T names = vec(Symbol.(obj["varnames"])) - cols = vec([vec(c) for c in obj["data"]]) + cols = vec([try_vec(c) for c in obj["data"]]) t = MatlabTable(names, cols) return T(Tables.CopiedColumns(t)) end # option to not convert and get the MatlabOpaque as table from_table(obj::MatlabOpaque, ::Type{Nothing}) = obj + try_vec(c::Vector) = c + try_vec(c) = [c] + function try_vec(c::AbstractArray) + return (size(c, 2) == 1) ? vec(c) : c + end + end \ No newline at end of file diff --git a/test/types.jl b/test/types.jl index 352ac0a..55ae2c8 100644 --- a/test/types.jl +++ b/test/types.jl @@ -201,6 +201,7 @@ end ) obj = MatlabOpaque(d, "table") + # Note: this should work with DataFrames.DataFrame, but that's a big dependency to add for testing t = MAT.convert_opaque(obj; table = MatlabTable) @test t.names == [:FlightNum, :Customer] @test t[:FlightNum] isa Vector{Float64} @@ -216,5 +217,33 @@ end t = MAT.convert_opaque(obj; table = Nothing) @test t === obj - # Note: this should all work with DataFrames.DataFrame, but that's a big dependency to add for testing + nd_array = reshape(1:12, 2, 3, 2) + + # ND-arrays as columns + # Note: does not convert to DataFrame + d = Dict{String,Any}( + "varnames" => Any["Floats" "NDArray"], + "nrows" => 2.0, + "data" => reshape(Any[[1261.0; 547.0;;], nd_array], 1, 2), + "ndims" => 2.0, + "nvars" => 2.0, + ) + obj = MatlabOpaque(d, "table") + t = MAT.convert_opaque(obj; table = MatlabTable) + @test size(t[:Floats]) == (2,) + @test size(t[:NDArray]) == (2,3,2) + + # single row table + d = Dict{String,Any}( + "varnames" => Any["Age" "Name" "Matrix"], + "nrows" => 1.0, + "data" => reshape([25.0, "Smith", [1.0 2.0]], 1, 3), + "ndims" => 2.0, + "nvars" => 2.0, + ) + obj = MatlabOpaque(d, "table") + t = MAT.convert_opaque(obj; table = MatlabTable) + @test t[:Age] == [25.0] + @test t[:Name] == ["Smith"] + @test t[:Matrix] == [1.0 2.0] end \ No newline at end of file From 482d10af1fb4232931073606ae9bbdbcb25e706a Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Wed, 19 Nov 2025 09:46:00 +0100 Subject: [PATCH 15/20] implement PR comments --- src/MAT_subsys.jl | 23 ++++++++++------------- src/MAT_types.jl | 5 +++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index dbdab35..bab2c78 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -74,12 +74,13 @@ function get_object!(subsys::Subsystem, oid::UInt32, classname::String) if haskey(subsys.object_cache, oid) # object is already cached, just retrieve it obj = subsys.object_cache[oid] - else + else # it's a new object prop_dict = Dict{String,Any}() - merge!(prop_dict, get_properties(subsys, oid)) - # cache it obj = MatlabOpaque(prop_dict, classname) + # cache the new object subsys.object_cache[oid] = obj + # caching must be done before a next call to `get_properties` to avoid any infinite recursion + merge!(prop_dict, get_properties(subsys, oid)) end return obj end @@ -154,6 +155,9 @@ function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_ end subsys.prop_vals_defaults = mcos_data[end, 1] + for el in subsys.prop_vals_defaults + update_nested_props!(el, subsys) # just in case + end return subsys end @@ -177,14 +181,7 @@ function get_object_metadata(subsys::Subsystem, object_id::UInt32) end function get_default_properties(subsys::Subsystem, class_id::UInt32) - prop_vals_class = subsys.prop_vals_defaults[class_id+1, 1] - # is it always a MatlabStructArray? - if prop_vals_class isa MatlabStructArray - prop_vals_class = Dict{String,Any}(prop_vals_class) - end - # FIXME Should we use deepcopy here? - r = copy(prop_vals_class) - return r + return Dict{String,Any}(subsys.prop_vals_defaults[class_id+1, 1]) end function get_property_idxs(subsys::Subsystem, obj_type_id::UInt32, saveobj_ret_type::Bool) @@ -269,8 +266,8 @@ function get_properties(subsys::Subsystem, object_id::UInt32) obj_type_id = normobj_id end - prop_map = get_default_properties(subsys, class_id) - merge!(prop_map, get_saved_properties(subsys, obj_type_id, saveobj_ret_type)) + defaults = get_default_properties(subsys, class_id) + prop_map = merge(defaults, get_saved_properties(subsys, obj_type_id, saveobj_ret_type)) # TODO: Add dynamic properties return prop_map end diff --git a/src/MAT_types.jl b/src/MAT_types.jl index ec01e84..b856c04 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -382,8 +382,9 @@ module MAT_types if isnothing(dat) || isempty(dat) return DateTime[] end - if !isempty(obj["tz"]) - @warn "no timezone conversion yet for datetime objects. timezone ignored" + if haskey(obj, "tz") && !isempty(obj["tz"]) + tz = obj["tz"] + @warn "no timezone conversion yet for datetime objects. timezone of \"$tz\" ignored" end #isdate = obj["isDateOnly"] # optional: convert to Date instead of DateTime? return map_or_not(ms_to_datetime, dat) From 4469b4037d95aa17a02d51d367c414e1d7e71bdd Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Wed, 19 Nov 2025 10:21:15 +0100 Subject: [PATCH 16/20] document type conversions --- docs/make.jl | 1 + docs/src/types.md | 25 +++++++++++++++++++++++++ test/types.jl | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 docs/src/types.md diff --git a/docs/make.jl b/docs/make.jl index e48e71c..20eab35 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -25,6 +25,7 @@ makedocs(; pages = [ "Home" => "index.md", "Object Arrays" => "object_arrays.md", + "Types" => "types.md", "Methods" => "methods.md", ], warnonly = [:missing_docs,], diff --git a/docs/src/types.md b/docs/src/types.md new file mode 100644 index 0000000..4a14b6b --- /dev/null +++ b/docs/src/types.md @@ -0,0 +1,25 @@ +# Types and conversions + +MAT.jl uses the following type conversions from MATLAB types to Julia types: + +| MATLAB | Julia | +| -------- | ------- | +| numerical array | `Array{T}` | +| cell array | `Array{Any}` | +| char array | `String` | +| `struct` | `Dict{String,Any}` | +| `struct` array | `MAT.MatlabStructArray` | +| old class object | `MAT.MatlabClassObject` | +| new (opaque) class | `MAT.MatlabOpaque` | + +A few of the `MatlabOpaque` classes are automatically converted upon reading: + +| MATLAB | Julia | +| -------- | ------- | +| `string` | `String` | +| `datetime` | `Dates.DateTime` | +| `duration` | `Dates.Millisecond` | +| `category` | `PooledArrays.PooledArray` | +| `table` | `MAT.MatlabTable` (or any other table) | + +Note that single element arrays are typically converted to scalars in Julia, because MATLAB cannot distinguish between scalars and `1x1` sized arrays. \ No newline at end of file diff --git a/test/types.jl b/test/types.jl index 55ae2c8..b1cf01d 100644 --- a/test/types.jl +++ b/test/types.jl @@ -175,7 +175,7 @@ end @testset "MatlabOpaque categorical" begin d = Dict( "isProtected" => false, - "codes" => UInt8[0x02; 0x03; 0x01;; 0x01; 0x01; 0x02], + "codes" => reshape(UInt8[0x02, 0x03, 0x01, 0x01, 0x01, 0x02], 3, 2), "categoryNames" => Any["Fair"; "Good"; "Poor";;], "isOrdinal" => false, ) From c94581fe45f825c2e697bf2f6710b6927b073168 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Wed, 19 Nov 2025 10:36:50 +0100 Subject: [PATCH 17/20] small refactor --- src/MAT_subsys.jl | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index bab2c78..1e90382 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -104,16 +104,14 @@ function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_ end fwrap_metadata = vec(mcos_data[1, 1]) - # FIXME: Is this the best way to read? - # Integers are written as uint8 (with swap), interpret as uint32 - version = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[1:4]) : fwrap_metadata[1:4])[1] + version = swapped_reinterpret(fwrap_metadata[1:4], swap_bytes)[1] if version <= 1 || version > FWRAP_VERSION error("Cannot read subsystem: Unsupported FileWrapper version: $version") end - subsys.num_names = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[5:8]) : fwrap_metadata[5:8])[1] - region_offsets = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[9:40]) : fwrap_metadata[9:40]) - + subsys.num_names = swapped_reinterpret(fwrap_metadata[5:8], swap_bytes)[1] + region_offsets = swapped_reinterpret(fwrap_metadata[9:40], swap_bytes) + # Class and Property Names stored as list of null-terminated strings start = 41 pos = start @@ -130,18 +128,18 @@ function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_ pos += 1 end - subsys.class_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) : fwrap_metadata[region_offsets[1]+1:region_offsets[2]]) - subsys.saveobj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) : fwrap_metadata[region_offsets[2]+1:region_offsets[3]]) - subsys.object_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) : fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) - subsys.obj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) : fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) - subsys.dynprop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) : fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) + subsys.class_id_metadata = swapped_reinterpret(fwrap_metadata[region_offsets[1]+1:region_offsets[2]], swap_bytes) + subsys.saveobj_prop_metadata = swapped_reinterpret(fwrap_metadata[region_offsets[2]+1:region_offsets[3]], swap_bytes) + subsys.object_id_metadata = swapped_reinterpret(fwrap_metadata[region_offsets[3]+1:region_offsets[4]], swap_bytes) + subsys.obj_prop_metadata = swapped_reinterpret(fwrap_metadata[region_offsets[4]+1:region_offsets[5]], swap_bytes) + subsys.dynprop_metadata = swapped_reinterpret(fwrap_metadata[region_offsets[5]+1:region_offsets[6]], swap_bytes) if region_offsets[7] != 0 - subsys._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) + subsys._u6_metadata = swapped_reinterpret(fwrap_metadata[region_offsets[6]+1:region_offsets[7]], swap_bytes) end if region_offsets[8] != 0 - subsys._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) + subsys._u7_metadata = swapped_reinterpret(fwrap_metadata[region_offsets[7]+1:region_offsets[8]], swap_bytes) end if version == 2 @@ -162,6 +160,12 @@ function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_ return subsys end +function swapped_reinterpret(T::Type, A::AbstractArray, swap_bytes::Bool) + reinterpret(T, swap_bytes ? reverse(A) : A) +end +# integers are written as uint8 (with swap), interpret as uint32 +swapped_reinterpret(A::AbstractArray, swap_bytes::Bool) = swapped_reinterpret(UInt32, A, swap_bytes) + function get_classname(subsys::Subsystem, class_id::UInt32) namespace_idx = subsys.class_id_metadata[class_id*4+1] classname_idx = subsys.class_id_metadata[class_id*4+2] From 513be332405e0b714dacf914bef8044ddc57a538 Mon Sep 17 00:00:00 2001 From: Matthijs Cox Date: Wed, 19 Nov 2025 11:01:43 +0100 Subject: [PATCH 18/20] reduce SnoopCompile invalidations --- src/MAT.jl | 14 -------------- src/MAT_HDF5.jl | 5 +++++ src/MAT_types.jl | 11 +++++++---- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/MAT.jl b/src/MAT.jl index ae542d5..e0a4b6a 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -232,18 +232,4 @@ function _write_dict(fileio, dict::AbstractDict) end end -### -### v0.10.0 deprecations -### - -export exists -@noinline function exists(matfile::Union{MAT_v4.Matlabv4File,MAT_v5.Matlabv5File,MAT_HDF5.MatlabHDF5File}, varname::String) - Base.depwarn("`exists(matfile, varname)` is deprecated, use `haskey(matfile, varname)` instead.", :exists) - return haskey(matfile, varname) -end -@noinline function Base.names(matfile::Union{MAT_v4.Matlabv4File,MAT_v5.Matlabv5File,MAT_HDF5.MatlabHDF5File}) - Base.depwarn("`names(matfile)` is deprecated, use `keys(matfile)` instead.", :names) - return keys(matfile) -end - end diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index d6497fb..66f7361 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -35,6 +35,7 @@ import Base: names, read, write, close import HDF5: Reference import Dates import Tables +import PooledArrays: PooledArray import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable const HDF5Parent = Union{HDF5.File, HDF5.Group} @@ -725,6 +726,10 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::A error("writing of MatlabOpaque types is not yet supported") end +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, arr::PooledArray) + error("writing of PooledArray types as categorical is not yet supported") +end + # Check whether a variable name is valid, then write it """ write(matfile_handle, varname, value) diff --git a/src/MAT_types.jl b/src/MAT_types.jl index b856c04..0c0f97a 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -29,6 +29,7 @@ module MAT_types import StringEncodings + import StringEncodings: Encoding import Dates: DateTime, Second, Millisecond import PooledArrays: PooledArray, RefArray import Tables @@ -343,7 +344,7 @@ module MAT_types end # for reference: https://github.com/foreverallama/matio/blob/main/matio/utils/converters/matstring.py - function from_string(obj::MatlabOpaque, encoding::String = "UTF-16LE") + function from_string(obj::MatlabOpaque, encoding::Encoding = Encoding(Symbol("UTF-16LE"))) data = obj["any"] if isnothing(data) || isempty(data) return String[] @@ -411,9 +412,11 @@ module MAT_types codes = obj["codes"] pool = vec(Array{promoted_eltype(category_names)}(category_names)) code_type = eltype(codes) - invpool = Dict{eltype(pool), code_type}(pool .=> code_type.(1:length(pool))) - refs = RefArray(codes) - return PooledArray(refs, invpool, pool) + pool_type = eltype(pool) + invpool = Dict{pool_type, code_type}(pool .=> code_type.(1:length(pool))) + RA = typeof(codes) + N = ndims(codes) + return PooledArray{pool_type,code_type,N,RA}(RefArray(codes), invpool, pool) end function promoted_eltype(v::AbstractArray{Any}) From 871662ea614a0550dc8ba8a24a1326089985c217 Mon Sep 17 00:00:00 2001 From: foreverallama Date: Wed, 19 Nov 2025 22:02:17 +0530 Subject: [PATCH 19/20] Tests for nested objects, handle class objects --- src/MAT_subsys.jl | 14 +++---- test/read.jl | 60 ++++++++++++++++++++++++++- test/v7.3/user_defined_classdefs.mat | Bin 0 -> 34400 bytes test/v7/user_defined_classdefs.mat | Bin 0 -> 1593 bytes 4 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 test/v7.3/user_defined_classdefs.mat create mode 100644 test/v7/user_defined_classdefs.mat diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index 1e90382..374ebb4 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -111,7 +111,7 @@ function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_ subsys.num_names = swapped_reinterpret(fwrap_metadata[5:8], swap_bytes)[1] region_offsets = swapped_reinterpret(fwrap_metadata[9:40], swap_bytes) - + # Class and Property Names stored as list of null-terminated strings start = 41 pos = start @@ -153,10 +153,6 @@ function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_ end subsys.prop_vals_defaults = mcos_data[end, 1] - for el in subsys.prop_vals_defaults - update_nested_props!(el, subsys) # just in case - end - return subsys end @@ -185,7 +181,11 @@ function get_object_metadata(subsys::Subsystem, object_id::UInt32) end function get_default_properties(subsys::Subsystem, class_id::UInt32) - return Dict{String,Any}(subsys.prop_vals_defaults[class_id+1, 1]) + default_props = Dict{String,Any}(subsys.prop_vals_defaults[class_id+1, 1]) + for (key, value) in default_props + default_props[key] = update_nested_props!(value, subsys) + end + return default_props end function get_property_idxs(subsys::Subsystem, obj_type_id::UInt32, saveobj_ret_type::Bool) @@ -310,7 +310,7 @@ function load_mcos_object(metadata::Array{UInt32}, type_name::String, subsys::Su obj = get_object!(subsys, oid, classname) return convert_opaque(obj; table=subsys.table_type) else - object_arr = Array{Any}(undef, convert(Vector{Int}, dims)...) + object_arr = Array{MatlabOpaque}(undef, convert(Vector{Int}, dims)...) for i = 1:length(object_arr) oid = object_ids[i] obj = get_object!(subsys, oid, classname) diff --git a/test/read.jl b/test/read.jl index 892d870..4c318a1 100644 --- a/test/read.jl +++ b/test/read.jl @@ -268,11 +268,69 @@ for format in ["v7", "v7.3"] @test all(x->size(x)==(3,1), t["data"]) @test "testDatetime" in keys(vars) - dt = vars["testDatetime"] + dt = vars["testDatetime"] @test dt isa DateTime @test dt - DateTime(2019, 12, 2, 16, 42, 49) < Second(1) end end + + @testset "user defined classdef $format" begin + let objtestfile = "user_defined_classdefs.mat" + filepath = joinpath(dirname(@__FILE__), format, objtestfile) + + vars = matread(filepath) + @test haskey(vars, "obj_no_vals") + obj_no_vals = vars["obj_no_vals"] + @test obj_no_vals isa MatlabOpaque + @test obj_no_vals.class == "TestClasses.BasicClass" + @test obj_no_vals["a"] isa Matrix{Float64} + + @test haskey(vars, "obj_with_vals") + obj_with_vals = vars["obj_with_vals"] + @test obj_with_vals isa MatlabOpaque + @test obj_with_vals.class == "TestClasses.BasicClass" + @test obj_with_vals["a"] == 10.0 + + @test haskey(vars, "obj_with_default_val") + obj_with_default_val = vars["obj_with_default_val"] + @test obj_with_default_val isa MatlabOpaque + @test obj_with_default_val.class == "TestClasses.DefaultClass" + @test obj_with_default_val["a"] == "Default String" + @test obj_with_default_val["b"] == 10.0 + + @test haskey(vars, "obj_array") + obj_array = vars["obj_array"] + @test obj_array isa Array{MatlabOpaque} + @test size(obj_array) == (2, 2) + @test obj_array[1, 1] isa MatlabOpaque + @test obj_array[1, 1]["a"] == 1.0 + @test obj_array[1, 2]["a"] == 2.0 + + @test haskey(vars, "obj_with_nested_props") + obj_with_nested_props = vars["obj_with_nested_props"] + @test obj_with_nested_props isa MatlabOpaque + @test obj_with_nested_props.class == "TestClasses.BasicClass" + @test obj_with_nested_props["a"] isa MatlabOpaque + @test obj_with_nested_props["a"]["a"] == 1.0 + + @test obj_with_nested_props["b"] isa Matrix{Any} + @test obj_with_nested_props["b"][1] isa MatlabOpaque + @test obj_with_nested_props["b"][1]["b"] == "Obj1" + + @test obj_with_nested_props["c"] isa Dict{String, Any} + @test obj_with_nested_props["c"]["InnerProp"] isa MatlabOpaque + @test obj_with_nested_props["c"]["InnerProp"]["a"] == 2.0 + + @test haskey(vars, "obj_handle_1") + @test haskey(vars, "obj_handle_2") + obj_handle_1 = vars["obj_handle_1"] + obj_handle_2 = vars["obj_handle_2"] + @test obj_handle_1 === obj_handle_2 + @test obj_handle_1 isa MatlabOpaque + + end + end + end # test reading of old-style Matlab object in v7.3 format diff --git a/test/v7.3/user_defined_classdefs.mat b/test/v7.3/user_defined_classdefs.mat new file mode 100644 index 0000000000000000000000000000000000000000..6444c066ba01e62f9eec4ee7f8c24523ed5c6835 GIT binary patch literal 34400 zcmeHQO>k4m5gyriSIc^L4Vz7zO*VNINJ^EkV8F0mDyfw*29f1Q0SBB^5?i)qWFllo zwpo*^q`XO0irjo#jvP615BS#&%ZKkLDdv7$-ldUItWurGPsi%ATJo?(`Y&)X=FjUgpBZGv4xSA%8FDojg7?zn)C018RUP(!=0eNz?zS#u-y;S~3ZN4+TDv>l2Fplo}NJ zA-TWHH*sTpT&W&~-vgA(ROSHZ)`al#FlTUI@Wt@Wlt5;DG)paeD$RQg$eL@kte&1d zeUBNRf50$IA2Xu%1<@i@D`xqJOAA z-#6uiDK(K_UKyjp{POYhx#fjfgHZig02_VGmtmm4JwMkdbMxBu3klVy9tl&$AOB2n z<%{vZ5@z&nLqlCy)HLh zMU%7voGL9&-Om-xw5ACEYGI`SijU8+vJ;&y^ywu%@#W{HmdcAubO)m6%H`Yxb9crI zxzb!QKXnq<34V(3Bg?BZ%MbJ|?j12#94Y7Lmvt8XB<_R8J8yT%x|Qvd!yUypX# zb|hou!w$onWJmUsym+0I2wf*_(IlXgb`MN?8LyK>65Ro?PJ-VIyHt_&52SE)?D|d0 zW1MGD5D~Io9t_pX%tw6V*K+^I*Er*$0=F2?2%h;f<4b~X-VuHVs>0KpN(=t_IA^8> zuMBf$UGRZlbLO$&-bKzl7d!^LD*@=`rZ~6{rC(o9@$cIH4YRQ3_NyVoK=UfCZ!>4I zqZ3+LX*rfh+3)S+tqXzOABEn>6TREVvf}d zSyxw=?jJ0nB}lR!5P>HoqL)Z8LAupnzu5iBB1)Ua*2Xt=JM#nQrTuIS+>>emzTj~CL2&5UG1z)PXO zz8u0Jbpi#(b}(czusX-V=I-Lb6YZLWRXGelG1mKLPI*JtZHOG@wuG z74cA^p{Q)=O(&MBYIK|nOpX%RATaDq6F7G`zhlUk74R(45xWXpw`TiOUEds zGOvEYdG|CC{Rh~cDJc_{VRtT>cqjIXB{K0VKH zYmA{9XZYk2L+B!O4Y~wffi9%7%%A=p!?ha>pBhg=sjUgdpZ|s7#^(&_yU{oe8)xKq zqyH}Iji=k?+|p9MJVjFi;*kT=2O0VilNyabvQnHjDCildrX|q6bsf(a**>m9*f`G0 zLKS)d+@3$UjC;0r{+Kj(=vCPRlp&vBNq;u=hioy~kr#T-r~M=P(EW32M$IZK`fV+8 zl^rZoxzYo*yi#5$-BI@9SKqf@qNY+EkO;@V0BF!p!l|6<>n3V1xgP}G>up}h3^c;= z0gWJ;QhaPWxk6N5f=IY0(U3zEn8x%KBe_~Su5nR-IY?K}6!x0i=)1LhW2XNvjeJQa{(ra0Z7!BuBz@B= zYErxOElCDVr!YTAijCc4KEtl<=b)h-^1cw?Hm!!=Rm0zsVb!VDA7Xn!>mW`?(*&c~ zv388xd9H;1MtzvjZR7Bc$e~^O)Gc-!+=C47nWME?w>x-KMZ={KZT!jvh9^#wXM#s_?7oFhzF;Hf*6j^;ADmbM~ufQ_EGZj zC?N*KSnYRONQe>Da_kQZ?04tpwaOPHsZPyncl~avKd+6*Ab00`tVv-5T->%ZY&Uq^ z)!G@R(>#A&$20CK4jx942h{=fnJwhX)U+TDbl8rwYb#Xvz~$j#QWW=j%ve*!xS;YW0ul_0Fr>b~uVY+Sk*( zby~;6o7JOxrq=Yu zKRC)wg7o9_(EF~7TdFY5LnB@QyY(;F|q>*;C&C1zhUMz(eGy0Z|E-Q zZZ;3n#iH`I0>&qy2T&F^RDoZ3zt}s zX_2RUPJ`{>Tk46FvOfA+Pa={Jg#$gFPSE(v^rl$UB~?0?~?Aed!r(UcIi{M%opd|EVs^$!_LO@ z#^P~EFMc?2p>6uqUH%8AXZRnluCpN1A$rCBhmLNFBY|hse0C>;afxTxoMIn^zlUes z`~oL~buylTbBcY`lltKqIlo|!WZE@fK)&v|Cir<_u=RDv7 zZReD_#sA2N{_Ov8IGm=YbZQ)47CE}>)Y|w*%5ZjK-z;;RE4ePg-DiJbyz)nesXsBK z?kFUjpnEI0F9Mzi{M zHI`c>xm|0U+;)GRx|hbOEKB|OkZj01Ptqb!*Z;sgnw3W29VtG$lbIAg`yDC{i$SM_ z4`*@e@er*bd^oEU#AOz_BA*e!OI-mdus>-Mm9s*Wg_g!TjXpMTKH zJU(;g8NS-*A9#N6RrJ<`+7xA8raGWL^4LSu&pl-Qs&M^$G$fM~x*YY=??vG>Cs5Qv zKcC16>QN9J&j;dpC_KM|=UMO^2F}UjT=$Y(k8`;=M~U--cytfvm$3hf{m(VI9{X9? z9~c!n;=PDhB3?Hx38`5k$6l9~|w_X2MT zM1R9y7IF->kwCZk+t8mOw0lSOT#GVhO|&h$Rq9AeKNZfmi~2O#+Amb<2JX_CH(Q zk4fk#E5$+(SH|-MFaFIv{7^c2yM3FDe=#GXQ1QNv2(C+jyKjSdwfA+k01$^m{4G;m w+%$%9->vcATQ~sW?IY0L+Lkuwv*kAg&tXjoyC4hg@sug=LIJf zrWwsReBc24l+O-E)5Wv&w5ObB+uAIs#rMeDVMh7P2~QFv7!LgxSP!x*4ToKOz;?yY zi<~3~woQxz+j1BTo(WC_*_MN_jS<;rN(kHLPntC?Vpgya&{pP&P)k=%W_UE=$|IrY zbB;WEbS24Fyd%h9I={yA#uPjKD>Y#n^Lewxl|$z6X2}WMNp+f(%$6h|&mgBM8UeC1 z4~LzLknEfu6*Fm;P<+6Y_yC@1FiVA*9|xfy7}%$dm|9al&ER4T`~4^F>cRZ@G1lLzI1GaHGL{BSuaMwv~^WlJXRuIR^1F z=<}-nE#M_9od6yjop}9b$9sf zaxuMG6;yj_$*NrUto5_Edd1{U-n?_^n`u|p&o#Wi^+Ru3-Na)uy*H!o-!uKYr?t*X z>iwMUN=LPJYkp0w7ueU={LrB1`!|z%)*p{wr+(9tZa*}?>rdtT4$nQSYn_tYgZ6vc zPTr-~oGv<3eB#POtXurqzbJY9Vt?JN;Q1nSM$e`CL?`Koi3K(%e7F|*Pw9D7SK_i) zp~{@YxV&%0N6xhC633DY=j(ON3Xk1B=X~LvGMzeck)?}{M{oQRx$(>54fBr1Up@8L zbnTt^S$7lJZ~HSHl%p2L{*(@$uViI){BQNXI=jc8#l-e8)I~?#{w*&j zdNQ29!uOc*)@vW8?49!WZ{yap57G+%?TG(T%p#v3`O_#pbYsoM*59uu{j7>niH~n6 zpZVOr_}r>8i;ZsEayMFmHK}F>e%?bE3aHP`u_T=Tz>VjW4~2o-&d>epBE}Q zKmL4~{bd*XNA~kSZmctvnEyQbX7?2R%=&jZYwv%z|M0UiVcuy`b%mdge4fr_e)e}| x&+}`GDlPw>-t(Y*62mV|tN!x`w%5L|WUdLn{bMit^`$4i%c<^X=x#~i2LJ#+wSE8q literal 0 HcmV?d00001 From 8455021d0950d81d8821cd8904927e79a9e133b4 Mon Sep 17 00:00:00 2001 From: foreverallama Date: Wed, 19 Nov 2025 22:33:16 +0530 Subject: [PATCH 20/20] Add support for loading dynamic properties --- src/MAT_subsys.jl | 31 ++++++++++++++++++++++++++++++- test/read.jl | 15 +++++++++++++++ test/v7.3/dynamicprops.mat | Bin 0 -> 13576 bytes test/v7/dynamicprops.mat | Bin 0 -> 825 bytes 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 test/v7.3/dynamicprops.mat create mode 100644 test/v7/dynamicprops.mat diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index 374ebb4..4c16ae3 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -256,6 +256,34 @@ function get_saved_properties(subsys::Subsystem, obj_type_id::UInt32, saveobj_re return save_prop_map end +function get_dynamic_properties(subsys::Subsystem, dep_id::UInt32) + offset = 1 + while dep_id > 0 + nprops = subsys.dynprop_metadata[offset] + offset += 1 + nprops + offset += (offset + 1) % 2 # Padding + dep_id -= 1 + end + + ndynprops = subsys.dynprop_metadata[offset] + offset += 1 + dyn_prop_obj_ids = subsys.dynprop_metadata[offset:offset+ndynprops-1] + + if dyn_prop_obj_ids == UInt32[] + return Dict{String,Any}() + end + dyn_prop_map = Dict{String,Any}() + for (i, obj_id) in enumerate(dyn_prop_obj_ids) + dyn_class_id = get_object_metadata(subsys, obj_id)[1] + classname = get_classname(subsys, dyn_class_id) + dynobj_props = Dict{String,Any}() + dynobj = MatlabOpaque(dynobj_props, classname) + merge!(dynobj_props, get_properties(subsys, obj_id)) + dyn_prop_map["__dynamic_property_$(i)__"] = dynobj + end + return dyn_prop_map +end + function get_properties(subsys::Subsystem, object_id::UInt32) if object_id == 0 return Dict{String,Any}() @@ -272,7 +300,8 @@ function get_properties(subsys::Subsystem, object_id::UInt32) defaults = get_default_properties(subsys, class_id) prop_map = merge(defaults, get_saved_properties(subsys, obj_type_id, saveobj_ret_type)) - # TODO: Add dynamic properties + dyn_props = get_dynamic_properties(subsys, object_id) + merge!(prop_map, dyn_props) return prop_map end diff --git a/test/read.jl b/test/read.jl index 4c318a1..fe24fb3 100644 --- a/test/read.jl +++ b/test/read.jl @@ -331,6 +331,21 @@ for format in ["v7", "v7.3"] end end + @testset "dynamic property" begin + let objtestfile = "dynamicprops.mat" + filepath = joinpath(dirname(@__FILE__), format, objtestfile) + + vars = matread(filepath) + @test haskey(vars, "obj") + obj = vars["obj"] + @test obj isa MatlabOpaque + @test obj.class == "TestClasses.BasicDynamic" + @test haskey(obj, "__dynamic_property_1__") + @test obj["__dynamic_property_1__"]["Name"] == "DynamicData" + @test obj["__dynamic_property_1__"]["DynamicValue_"] == 42.0 + end + end + end # test reading of old-style Matlab object in v7.3 format diff --git a/test/v7.3/dynamicprops.mat b/test/v7.3/dynamicprops.mat new file mode 100644 index 0000000000000000000000000000000000000000..abee0cbf6f37240c2695974b3f842cc8d8d0e961 GIT binary patch literal 13576 zcmeHNy>nYd5MNm_Q4$B7Kp=eI5MWFQj-4+;8f0vPn8AsLXox%e>G>In=!^HB8M{jf zUFcG#OzD!+C1uJq%%p>o{{VJxccs&lo@2{~*on_0@5g=rZf|ezq`9fmH&b7jPscBs zxvA18%hg(N&J=52yxa`y*GzHx*4NjsTsmi_!@!G!CDUwNGo{Lkx#7pgd}b!ET$`M@ zHhINdn7DA+6lT7>Y$CrB)IBpfJ~3g&O}_-1hQ6Lt_JBSe1b)FaJAU+yqqg^44i3R6 z4`hhRKaz5nyJeOD7KOUBxDBY{_BBS^bbSlmpghd8hnA)1rFrhbo@LdX#LNzy4Vn zfxO7aF((J;#L=HG_%$zzDC1@P(B7l+luAJqPg7xpI^`0>J0;5U~7n(Zh8|_4$=oR*$M(lzbnxJoH%{P4n#%V}z45`$=I7#*7GKa2p?LUSquHqXUd_5l!yZo~(VP$iPvH{c79J#_af$@u?i+ z?R8Q)D|Y%C)}pl-M!TY>C@v*g*M;tdq<3{FqI`!4Mt-ZHYom2^4HV-=>IBmRS)`5E zSw8(H(>rf5z4tED$A;TpK_i1)}9P0^e@FhNc+zP!`D+m`B zP~VP+#oh|kiB0_tknNdvJ*N$7{2*-`f!&LL+5bM2fH>{m_D~$K+JVZhKju7M*uBXJ)7&V{D9u}F(lBG}{l|3Rn<osNM*ROQKSDF33vlfPCy zc_8IaIQ&V6r#Vm=>W_{>qBNFM_|dw^+C+Iton-a#s}oWu{k=i)qJk^cMobaN2xz^- zIR6|JE*Gat1%97UjHvwiGxnk8e3=BB`ukgrukti_Ov(LCcvF&WyFRY$?r)wu_tU$- z`LN2ui!h64oCkGiK@IFB92@LCcXXcV_^55sc>IbN67BfClWrV}=sA~uESGlg*Y(>j z&OUXHN%1VC;Hlx6;+d4Uj;FExWpPK(z#23xqE+xw@*u_!r~QuWlFUPKr89w2(oW@g zYTIeA+<8CT+)mkkv*>j^MAcqiu)qC|qwX`sF_(g)+GQH0cU8(eVLrG17a$Lpj=`mW z1n}N{uXysg4=|g@>I31aX9m?@YOc^}x@~{BOqFfY@vx-09PC?T0nzmz)E|D&u(ti- zyfm2Hw{(tD(*Bb2*8L%$7g;z9B|KFk^nu2DM%u-7AkS!^2q84_+zF7zc5m@?X-SpBwOS3U1v;+ z>zzNXHlx>h6G$t^C;;f2L0Y-IWayhhTDiThSHGSXH_(|vN&Qb!-n#!8we4il(psl76*J;^^ z$v8pxXK!534etE?)!2q8zFq%-+x+W8rbeAGii6s#{XrT1>G+S^#$-uQiDH@{a>n{`nb#D3g;hmVXa z_{g|z|I?5`>@<#f-f(Lk;5LxGwRRb|&b@}6aq@e)^m9SpeRAEu^GScFkRLk8_y^gq z!&q1MK0Z8Nynym{{r!*9f9DsUZABNO=RrlZ`rTevAxF@4(KF7;>S9+#(lf9t(-H0n z@1BuP_P$}&ukD5Sp!?OoSj=d&XHD%R^F?{)HjybmfSE*69MtKG0aT_?4_$4{_7K1YY) uXBoWyHpZQc%ik5j{q2mKulzk>)TgUHN$-n7eSR`uTdhR4_ENGBmI4T9A`n-u=q_L16_?pA8#Of$_;ao5C0! z7k-Z(lUw<9`#!$uzn;f@U;*oeBZcgu51qEHdAo35hOWEN{aX+AN>94$w)d{)k2==- zFE8DXIASx~_)TogE-U@Wxz4lSuRV55Y5JSmE9QYZ>!+MBTB#;{djD;`A4fk=`{uZk z|5W>ff0h3~T>o7AI{U#7nXe+u*TeQtOnS3B^pv%DG3WZ$zC(5MIx0mDMPyy>ZoL-N z{59~>5y{6k+CBVA`xL!j1-Q=ayu`9m(Bh@!_q|h>Nw)=M?}%Br?@Wr(CxxvpPpJXTQ2z~E(y9L({pZPp@#Q?b6ht}Yqw3@su|b%c52qnzn`=} z$^7S-R{gT(osZD+%)$eYrtCgjbXRimooPo!>QfAlmPhhe^Sn2{llG(G;c*^$qdzSV zmAhY7`oHzqD-*uP{i@W?=es|ObbVi#__t$Z?~!D{;fa%TlcT8|NP(S6St_;f(`&(eSZG{ literal 0 HcmV?d00001