Support for loading mxOPAQUE_CLASS objects in v7.3 HDF5 format

* MAT_HDF5.jl (matopen): New argument Endian indicator, Reads and parses subsystem on load
* MAT_HDF5.jl (close): Update to write endian header based on system endianness
* MAT_HDF5.jl (m_read::HDF5.Dataset): Update to handle MATLAB_object_decode (mxOPAQUE_CLASS) types
* MAT_HDF5.jl (m_read::HDF5.Group): Update to read subsystem data and function_handles
* MAT.jl (matopen): Update function calls

Subsystem Improvements

* MAT_subsys.jl: Add copyright notice
* MAT_subsys.jl: Add support for handle class objects
* MAT_subsys.jl (load_subsystem): Update to handle different FileWrapper versions
* MAT_subsys.jl: Add support to decode nested objects

Tests
* test/read.jl: Update tests for "function_handles.mat" and "struct_table_datetime.mat"

Author: foreverallama

src/MAT.jl

@@ -41,7 +41,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
     # When creating new files, create as HDF5 by default
     fs = filesize(filename)
     if cr && (tr || fs == 0)
-        return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress)
+        return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, Base.ENDIAN_BOM == 0x04030201)
     elseif fs == 0
         error("File \"$filename\" does not exist and create was not specified")
     end
@@ -77,7 +77,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
         seek(rawfid, offset)
         if read!(rawfid, Vector{UInt8}(undef, 8)) == HDF5_HEADER
             close(rawfid)
-            return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress)
+            return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, endian_indicator == 0x494D)
         end
     end
 

src/MAT_HDF5.jl

@@ -29,6 +29,7 @@
 module MAT_HDF5
 
 using HDF5, SparseArrays
+using ..MAT_subsys
 
 import Base: names, read, write, close
 import HDF5: Reference
@@ -69,8 +70,13 @@ function close(f::MatlabHDF5File)
                 unsafe_copyto!(magicptr, idptr, length(identifier))
             end
             magic[126] = 0x02
-            magic[127] = 0x49
-            magic[128] = 0x4d
+            if Base.ENDIAN_BOM == 0x04030201
+                magic[127] = 0x49
+                magic[128] = 0x4d
+            else
+                magic[127] = 0x4d
+                magic[128] = 0x49
+            end
             rawfid = open(f.plain.filename, "r+")
             write(rawfid, magic)
             close(rawfid)
@@ -80,7 +86,7 @@ function close(f::MatlabHDF5File)
     nothing
 end
 
-function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool)
+function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool, endian_indicator::Bool)
     local f
     if ff && !wr
         error("Cannot append to a read-only file")
@@ -109,6 +115,11 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
         fid.refcounter = length(g)-1
         close(g)
     end
+    subsys_refs = "#subsystem#"
+    if haskey(fid.plain, subsys_refs)
+        subsys_data = m_read(fid.plain[subsys_refs])
+        MAT_subsys.load_subsys!(subsys_data, endian_indicator)
+    end
     fid
 end
 
@@ -118,6 +129,7 @@ const name_type_attr_matlab = "MATLAB_class"
 const empty_attr_matlab = "MATLAB_empty"
 const sparse_attr_matlab = "MATLAB_sparse"
 const int_decode_attr_matlab = "MATLAB_int_decode"
+const object_type_attr_matlab = "MATLAB_object_decode"
 
 ### Reading
 function read_complex(dtype::HDF5.Datatype, dset::HDF5.Dataset, ::Type{T}) where T
@@ -128,6 +140,21 @@ function read_complex(dtype::HDF5.Datatype, dset::HDF5.Dataset, ::Type{T}) where
     return read(dset, Complex{T})
 end
 
+function read_cell(dset::HDF5.Dataset)
+    refs = read(dset, Reference)
+    out = Array{Any}(undef, size(refs))
+    f = HDF5.file(dset)
+    for i = 1:length(refs)
+        dset = f[refs[i]]
+        try
+            out[i] = m_read(dset)
+        finally
+            close(dset)
+        end
+    end
+    return out
+end
+
 function m_read(dset::HDF5.Dataset)
     if haskey(dset, empty_attr_matlab)
         # Empty arrays encode the dimensions as the dataset
@@ -150,36 +177,46 @@ function m_read(dset::HDF5.Dataset)
     end
 
     mattype = haskey(dset, name_type_attr_matlab) ? read_attribute(dset, name_type_attr_matlab) : "cell"
+    objecttype = haskey(dset, object_type_attr_matlab) ? read_attribute(dset, object_type_attr_matlab) : nothing
 
-    if mattype == "cell"
+    if mattype == "cell" && objecttype === nothing
         # Cell arrays, represented as an array of refs
-        refs = read(dset, Reference)
-        out = Array{Any}(undef, size(refs))
-        f = HDF5.file(dset)
-        for i = 1:length(refs)
-            dset = f[refs[i]]
-            try
-                out[i] = m_read(dset)
-            finally
-                close(dset)
-            end
+        return read_cell(dset)
+    elseif objecttype !== nothing
+        if objecttype != 3
+            @warn "MATLAB Object Type $mattype is currently not supported."
+            return missing
+        end
+        if mattype == "FileWrapper__"
+            return read_cell(dset)
+        end
+        if haskey(dset, "MATLAB_fields")
+            @warn "Enumeration Instances are not supported currently."
+            return missing
         end
-        return out
     elseif !haskey(str2type_matlab,mattype)
-        @warn "MATLAB $mattype values are currently not supported"
+        @warn "MATLAB $mattype values are currently not supported."
         return missing
     end
 
     # Regular arrays of values
     # Convert to Julia type
-    T = str2type_matlab[mattype]
+    if objecttype === nothing
+        T = str2type_matlab[mattype]
+    else
+        T = UInt32 # FIXME: Default for MATLAB objects?
+    end
 
     # Check for a COMPOUND data set, and if so handle complex numbers specially
     dtype = datatype(dset)
     try
         class_id = HDF5.API.h5t_get_class(dtype.id)
         d = class_id == HDF5.API.H5T_COMPOUND ? read_complex(dtype, dset, T) : read(dset, T)
-        length(d) == 1 ? d[1] : d
+        if objecttype !== nothing
+            return MAT_subsys.load_mcos_object(d, "MCOS")
+        else
+            return length(d) == 1 ? d[1] : d
+        end
     finally
         close(dtype)
     end
@@ -194,7 +231,11 @@ end
 
 # reading a struct, struct array, or sparse matrix
 function m_read(g::HDF5.Group)
-    mattype = read_attribute(g, name_type_attr_matlab)
+    if HDF5.name(g) == "/#subsystem#"
+        mattype = "#subsystem#"
+    else
+        mattype = read_attribute(g, name_type_attr_matlab)
+    end
     if mattype != "struct"
         # Check if this is a sparse matrix.
         fn = keys(g)
@@ -226,10 +267,11 @@ function m_read(g::HDF5.Group)
             end
             return SparseMatrixCSC(convert(Int, read_attribute(g, sparse_attr_matlab)), length(jc)-1, jc, ir, data)
         elseif mattype == "function_handle"
-            @warn "MATLAB $mattype values are currently not supported"
-            return missing
+            # Fall through
         else
-            @warn "Unknown non-struct group of type $mattype detected; attempting to read as struct"
+            if mattype != "#subsystem#"
+                @warn "Unknown non-struct group of type $mattype detected; attempting to read as struct"
+            end
         end
     end
     if haskey(g, "MATLAB_fields")

src/MAT_subsys.jl

@@ -1,3 +1,27 @@
+# MAT_subsys.jl
+# Tools for processing MAT-file subsystem data in Julia
+#
+# Copyright (C) 2025    Nithin Lakshmisha
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 module MAT_subsys
 
 const FWRAP_VERSION = 4
@@ -39,27 +63,34 @@ mutable struct Subsys
 end
 
 const subsys_cache = Ref{Union{Nothing,Subsys}}(nothing)
+const object_cache = Ref{Union{Nothing, Dict{UInt32, Dict{String,Any}}}}(nothing)
 
 function clear_subsys!()
     subsys_cache[] = nothing
+    object_cache[] = nothing
 end
 
 function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool)
     subsys_cache[] = Subsys()
+    object_cache[] = Dict{UInt32, Dict{String,Any}}()
     subsys_cache[].handle_data = get(subsystem_data, "handle", nothing)
     subsys_cache[].java_data = get(subsystem_data, "java", nothing)
     mcos_data = get(subsystem_data, "MCOS", nothing)
     if mcos_data === nothing
         return
     end
 
-    fwrap_metadata = vec(mcos_data[2][1, 1])
+    if mcos_data isa Tuple
+        # Backward compatibility with MAT_v5
+        mcos_data = mcos_data[2]
+    end
+    fwrap_metadata = vec(mcos_data[1, 1])
 
     # FIXME: Is this the best way to read?
     # Integers are written as uint8 (with swap), interpret as uint32
     version = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[1:4]) : fwrap_metadata[1:4])[1]
-    if version > FWRAP_VERSION
-        error("Unsupported FileWrapper version: $version")
+    if version <= 1 || version > FWRAP_VERSION
+        error("Cannot read subsystem: Unsupported FileWrapper version: $version")
     end
 
     subsys_cache[].num_names = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[5:8]) : fwrap_metadata[5:8])[1]
@@ -86,18 +117,26 @@ function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool)
     subsys_cache[].object_id_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[3]+1:region_offsets[4]]) : fwrap_metadata[region_offsets[3]+1:region_offsets[4]])
     subsys_cache[].obj_prop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[4]+1:region_offsets[5]]) : fwrap_metadata[region_offsets[4]+1:region_offsets[5]])
     subsys_cache[].dynprop_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[5]+1:region_offsets[6]]) : fwrap_metadata[region_offsets[5]+1:region_offsets[6]])
-    subsys_cache[]._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]])
-    subsys_cache[]._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]])
 
-    if version < 4
-        subsys_cache[].prop_vals_saved = mcos_data[2][3:end-2, 1]
+    if region_offsets[6] != 0
+        subsys_cache[]._u6_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[6]+1:region_offsets[7]]) : fwrap_metadata[region_offsets[6]+1:region_offsets[7]])
+    end
+
+    if region_offsets[7] != 0
+        subsys_cache[]._u7_metadata = reinterpret(UInt32, swap_bytes ? reverse(fwrap_metadata[region_offsets[7]+1:region_offsets[8]]) : fwrap_metadata[region_offsets[7]+1:region_offsets[8]])
+    end
+
+    if version == 2
+        subsys_cache[].prop_vals_saved = mcos_data[3:end-1, 1]
+    elseif version == 3
+        subsys_cache[].prop_vals_saved = mcos_data[3:end-2, 1]
+        subsys_cache[]._c2 = mcos_data[end-1, 1]
     else
-        subsys_cache[].prop_vals_saved = mcos_data[2][3:end-3, 1]
-        subsys_cache[]._c3 = mcos_data[2][end-2, 1]
+        subsys_cache[].prop_vals_saved = mcos_data[3:end-3, 1]
+        subsys_cache[]._c3 = mcos_data[end-2, 1]
     end
 
-    subsys_cache[]._c2 = mcos_data[2][end-1, 1]
-    subsys_cache[].prop_vals_defaults = mcos_data[2][end, 1]
+    subsys_cache[].prop_vals_defaults = mcos_data[end, 1]
 end
 
 function get_classname(class_id::UInt32)
@@ -107,10 +146,10 @@ function get_classname(class_id::UInt32)
     namespace = if namespace_idx == 0
         ""
     else
-        subsys_cache[].mcos_names[namespace_idx-1] * "."
+        subsys_cache[].mcos_names[namespace_idx] * "."
     end
 
-    classname = namespace * subsys_cache[].mcos_names[classname_idx-1]
+    classname = namespace * subsys_cache[].mcos_names[classname_idx]
     return classname
 end
 
@@ -138,6 +177,33 @@ function get_property_idxs(obj_type_id::UInt32, saveobj_ret_type::Bool)
     return prop_field_idxs[offset:offset+nprops*nfields-1]
 end
 
+function find_nested_prop(prop_value::Any)
+    # Hacky way to find a nested object
+    # Nested objects are stored as a uint32 Matrix with a unique signature
+    # MATLAB probably uses some kind of placeholders to decode
+    # But this should work here
+    if prop_value isa Dict
+        # Handle nested objects in a dictionary (struct)
+        for (key, value) in prop_value
+            prop_value[key] = find_nested_prop(value)
+        end
+    end
+
+    if prop_value isa Matrix{Any}
+        # Handle nested objects in a Cell
+        for i in eachindex(prop_value)
+            prop_value[i] = find_nested_prop(prop_value[i])
+        end
+    end
+
+    if prop_value isa Matrix{UInt32} && prop_value[1,1] == 0xdd000000
+        # MATLAB identifies any uint32 array with first value 0xdd000000 as an MCOS object
+        return load_mcos_object(prop_value, "MCOS")
+    end
+
+    return prop_value
+end
+
 function get_saved_properties(obj_type_id::UInt32, saveobj_ret_type::Bool)
     save_prop_map = Dict{String,Any}()
     prop_field_idxs = get_property_idxs(obj_type_id, saveobj_ret_type)
@@ -148,14 +214,13 @@ function get_saved_properties(obj_type_id::UInt32, saveobj_ret_type::Bool)
         if prop_type == 0
             prop_value = subsys_cache[].mcos_names[prop_field_idxs[i*3+3]]
         elseif prop_type == 1
-            # FIXME: Search for nested objects
             prop_value = subsys_cache[].prop_vals_saved[prop_field_idxs[i*3+3]+1]
         elseif prop_type == 2
             prop_value = prop_field_idxs[i*3+3]
         else
             error("Unknown property type ID: $prop_type encountered during deserialization")
         end
-        save_prop_map[prop_name] = prop_value
+        save_prop_map[prop_name] = find_nested_prop(prop_value)
     end
     return save_prop_map
 end
@@ -181,8 +246,6 @@ function get_properties(object_id::UInt32)
 end
 
 function load_mcos_object(metadata::Any, type_name::String)
-    # TODO: Add support for handle class objects
-
     if type_name != "MCOS"
         @warn "Loading Type:$type_name is not implemented. Returning metadata."
         return metadata
@@ -213,9 +276,17 @@ function load_mcos_object(metadata::Any, type_name::String)
     classname = get_classname(class_id)
 
     object_arr = Array{Dict{String,Any}}(undef, convert(Vector{Int}, dims)...)
+
     for i = 1:length(object_arr)
-        prop_dict = get_properties(object_ids[i])
-        prop_dict["__class__"] = classname
+        oid = object_ids[i]
+        if haskey(object_cache[], oid)
+            prop_dict = object_cache[][oid]
+        else
+            prop_dict = Dict{String,Any}()
+            object_cache[][oid] = prop_dict
+            merge!(prop_dict, get_properties(oid))
+            prop_dict["__class__"] = classname
+        end
         object_arr[i] = prop_dict
     end
 

test/read.jl

@@ -219,16 +219,18 @@ end
 let objtestfile = "function_handles.mat"
     vars = matread(joinpath(dirname(@__FILE__), "v7.3", objtestfile))
     @test "sin" in keys(vars)
-    @test ismissing(vars["sin"])
+    @test typeof(vars["sin"]) == Dict{String, Any}
+    @test Set(keys(vars["sin"])) == Set(["function_handle", "sentinel", "separator", "matlabroot"])
     @test "anonymous" in keys(vars)
-    @test ismissing(vars["anonymous"])
+    @test typeof(vars["anonymous"]) == Dict{String, Any}
+    @test Set(keys(vars["anonymous"])) == Set(["function_handle", "sentinel", "separator", "matlabroot"])
 end
 let objtestfile = "struct_table_datetime.mat"
     vars = matread(joinpath(dirname(@__FILE__), "v7.3", objtestfile))["s"]
     @test "testTable" in keys(vars)
-    @test ismissing(vars["testTable"])
+    @test vars["testTable"][1, 1]["__class__"] == "table"
     @test "testDatetime" in keys(vars)
-    @test ismissing(vars["testDatetime"])
+    @test vars["testDatetime"][1, 1]["__class__"] == "datetime"
 end
 
 # test reading of old-style Matlab object in v7.3 format