Loading MatlabOpaque types

Merge PR "Update handling of mxOPAQUE_CLASS objects and Subsystem in MAT_v5.jl (#205)"
* Add support for loading mxOPAQUE_CLASS types

* MAT_subsys.jl: New file MAT_subsys with methods to set, parse and retrieve subsystem data
* MAT_v5.jl: New method "read_opaque" to handle mxOPAQUE_CLASS
* MAT_v5.jl: New method "read_subsystem" to handle subsystem data
* MAT.jl (matread): Update to clear subsystem and object cache after load

Support for loading mxOPAQUE_CLASS objects in v7.3 HDF5 format

* MAT_HDF5.jl (matopen): New argument Endian indicator, Reads and parses subsystem on load
* MAT_HDF5.jl (close): Update to write endian header based on system endianness
* MAT_HDF5.jl (m_read::HDF5.Dataset): Update to handle MATLAB_object_decode (mxOPAQUE_CLASS) types
* MAT_HDF5.jl (m_read::HDF5.Group): Update to read subsystem data and function_handles
* MAT.jl (matopen): Update function calls

Updated test for struct_table_datetime.mat to ensure accurate deserialization (including nested properties) in both v7 and v7.3 formats

* test/read.jl: Update tests for "function_handles.mat" and "struct_table_datetime.mat"

* MAT_subsys.load_subsys!: Fix indexing for parsing metadata from region offests

* first step in subsystem refactoring

* stateless MAT_subsys module

* test read(matopen(filepath), ::String) for opaque subsystems

* MatlabOpaque to_string conversion

* MatlabOpaque to_datetime conversion

* fix to_string test

* MatlabOpaque to_duration conversion

* automatic MatlabOpaque conversion

* attempt to fix ci: change Dates compat and more explicit type writing

* MatlabOpaque categorical support

* MatlabOpaque table support

* Support for single row tables and ND columns

* implement PR comments

* document type conversions

* small refactor

* reduce SnoopCompile invalidations

* Tests for nested objects, handle class objects

* Add support for loading dynamic properties

* mcos refactor: avoid opaque array conversion and extracted load_mcos_names!

* bluestyle formatting

* avoid printing massive objects

* MAT_subsys.jl: Display warning instead of throwing error for unknown property types

---------

Co-authored-by: Matthijs Cox <[email protected]>

Author: foreverallama

.JuliaFormatter.toml

@@ -0,0 +1 @@
+style = "blue"

Project.toml

@@ -5,13 +5,21 @@ version = "0.11.0"
 [deps]
 BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
+PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+StringEncodings = "69024149-9ee7-55f6-a4c4-859efe599b68"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
 BufferedStreams = "0.4.1, 1"
 CodecZlib = "0.5, 0.6, 0.7"
+Dates = "1"
 HDF5 = "0.16, 0.17"
+PooledArrays = "1.4.3"
+StringEncodings = "0.3.7"
+Tables = "1.12.1"
 julia = "1.6"
 
 [extras]

docs/make.jl

@@ -25,6 +25,7 @@ makedocs(;
     pages = [
         "Home" => "index.md",
         "Object Arrays" => "object_arrays.md",
+        "Types" => "types.md",
         "Methods" => "methods.md",
     ],
     warnonly = [:missing_docs,],

docs/src/types.md

@@ -0,0 +1,25 @@
+# Types and conversions
+
+MAT.jl uses the following type conversions from MATLAB types to Julia types:
+
+| MATLAB    | Julia |
+| -------- | ------- |
+| numerical array | `Array{T}` |
+| cell array | `Array{Any}` |
+| char array | `String` |
+| `struct`  | `Dict{String,Any}`    |
+| `struct` array | `MAT.MatlabStructArray`     |
+| old class object    | `MAT.MatlabClassObject`    |
+| new (opaque) class    | `MAT.MatlabOpaque`    |
+
+A few of the `MatlabOpaque` classes are automatically converted upon reading:
+
+| MATLAB    | Julia |
+| -------- | ------- |
+| `string`    | `String`    |
+| `datetime`    | `Dates.DateTime`    |
+| `duration`    | `Dates.Millisecond`    |
+| `category`    | `PooledArrays.PooledArray`    |
+| `table`    | `MAT.MatlabTable` (or any other table) |
+
+Note that single element arrays are typically converted to scalars in Julia, because MATLAB cannot distinguish between scalars and `1x1` sized arrays.

src/MAT.jl

@@ -29,22 +29,23 @@ using HDF5, SparseArrays
 include("MAT_types.jl")
 using .MAT_types
 
+include("MAT_subsys.jl")
 include("MAT_HDF5.jl")
 include("MAT_v5.jl")
 include("MAT_v4.jl")
 
-using .MAT_HDF5, .MAT_v5, .MAT_v4
+using .MAT_HDF5, .MAT_v5, .MAT_v4, .MAT_subsys
 
 export matopen, matread, matwrite, @read, @write
-export MatlabStructArray, MatlabClassObject
+export MatlabStructArray, MatlabClassObject, MatlabOpaque, MatlabTable
 
 # Open a MATLAB file
 const HDF5_HEADER = UInt8[0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a]
-function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool)
+function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool; table::Type=MatlabTable)
     # When creating new files, create as HDF5 by default
     fs = filesize(filename)
     if cr && (tr || fs == 0)
-        return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress)
+        return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, Base.ENDIAN_BOM == 0x04030201; table=table)
     elseif fs == 0
         error("File \"$filename\" does not exist and create was not specified")
     end
@@ -72,26 +73,26 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
         if wr || cr || tr || ff
             error("creating or appending to MATLAB v5 files is not supported")
         end
-        return MAT_v5.matopen(rawfid, endian_indicator)
+        return MAT_v5.matopen(rawfid, endian_indicator; table=table)
     end
 
     # Check for HDF5 file
     for offset = 512:512:fs-8
         seek(rawfid, offset)
         if read!(rawfid, Vector{UInt8}(undef, 8)) == HDF5_HEADER
             close(rawfid)
-            return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress)
+            return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, endian_indicator == 0x494D; table=table)
         end
     end
 
     close(rawfid)
     error("\"$filename\" is not a MAT file")
 end
 
-function matopen(fname::AbstractString, mode::AbstractString; compress::Bool = false)
-    mode == "r"  ? matopen(fname, true , false, false, false, false, false)    :
-    mode == "r+" ? matopen(fname, true , true , false, false, false, compress) :
-    mode == "w"  ? matopen(fname, false, true , true , true , false, compress) :
+function matopen(fname::AbstractString, mode::AbstractString; compress::Bool = false, table::Type = MatlabTable)
+    mode == "r"  ? matopen(fname, true , false, false, false, false, false; table=table)    :
+    mode == "r+" ? matopen(fname, true , true , false, false, false, compress; table=table) :
+    mode == "w"  ? matopen(fname, false, true , true , true , false, compress; table=table) :
     # mode == "w+" ? matopen(fname, true , true , true , true , false, compress) :
     # mode == "a"  ? matopen(fname, false, true , true , false, true, compress)  :
     # mode == "a+" ? matopen(fname, true , true , true , false, true, compress)  :
@@ -110,8 +111,8 @@ function matopen(f::Function, args...; kwargs...)
 end
 
 """
-    matopen(filename [, mode]; compress = false) -> handle
-    matopen(f::Function, filename [, mode]; compress = false) -> f(handle)
+    matopen(filename [, mode]; compress = false, table = MatlabTable) -> handle
+    matopen(f::Function, filename [, mode]; compress = false, table = MatlabTable) -> f(handle)
 
 Mode defaults to `"r"` for read.
 It can also be `"w"` for write,
@@ -121,18 +122,71 @@ Compression on reading is detected/handled automatically; the `compress`
 keyword argument only affects write operations.
 
 Use with `read`, `write`, `close`, `keys`, and `haskey`.
+
+Optional keyword argument is the `table` type, for automatic conversion of Matlab tables.
+Note that Matlab tables may contain non-vector colums which cannot always be converted to a Julia table, like `DataFrame`.
+
+# Example
+
+```julia
+using MAT, DataFrames
+filepath = abspath(pkgdir(MAT), "./test/v7.3/struct_table_datetime.mat")
+fid = matopen(filepath; table = DataFrame)
+keys(fid)
+
+# outputs
+
+1-element Vector{String}:
+ "s"
+
+```
+
+Now you can read any of the keys
+```
+s = read(fid, "s")
+close(fid)
+s
+
+# outputs
+
+Dict{String, Any} with 2 entries:
+  "testDatetime" => DateTime("2019-12-02T16:42:49.634")
+  "testTable"    => 3×5 DataFrame…
+
+```
 """
 matopen
 
 # Read all variables from a MATLAB file
 """
-    matread(filename) -> Dict
+    matread(filename; table = MatlabTable) -> Dict
 
 Return a dictionary of all the variables and values in a Matlab file,
 opening and closing it automatically.
+
+Optionally provide the `table` type to convert Matlab tables into. Default uses a simple `MatlabTable` type.
+
+# Example
+
+```julia
+using MAT, DataFrames
+filepath = abspath(pkgdir(MAT), "./test/v7.3/struct_table_datetime.mat")
+vars = matread(filepath; table = DataFrame)
+vars["s"]["testTable"]
+
+# outputs
+
+3×5 DataFrame
+ Row │ FlightNum  Customer  Date                 Rating  Comment
+     │ Float64    String    DateTime             String  String
+─────┼─────────────────────────────────────────────────────────────────────────────────────
+   1 │    1261.0  Jones     2016-12-20T00:00:00  Good    Flight left on time, not crowded
+   2 │     547.0  Brown     2016-12-21T00:00:00  Poor    Late departure, ran out of dinne…
+   3 │    3489.0  Smith     2016-12-22T00:00:00  Fair    Late, but only by half an hour. …
+```
 """
-function matread(filename::AbstractString)
-    file = matopen(filename)
+function matread(filename::AbstractString; table::Type=MatlabTable)
+    file = matopen(filename; table=table)
     local vars
     try
         vars = read(file)
@@ -178,18 +232,4 @@ function _write_dict(fileio, dict::AbstractDict)
     end
 end
 
-###
-### v0.10.0 deprecations
-###
-
-export exists
-@noinline function exists(matfile::Union{MAT_v4.Matlabv4File,MAT_v5.Matlabv5File,MAT_HDF5.MatlabHDF5File}, varname::String)
-    Base.depwarn("`exists(matfile, varname)` is deprecated, use `haskey(matfile, varname)` instead.", :exists)
-    return haskey(matfile, varname)
-end
-@noinline function Base.names(matfile::Union{MAT_v4.Matlabv4File,MAT_v5.Matlabv5File,MAT_HDF5.MatlabHDF5File})
-    Base.depwarn("`names(matfile)` is deprecated, use `keys(matfile)` instead.", :names)
-    return keys(matfile)
-end
-
 end