diff --git a/docs/src/api/reference.md b/docs/src/api/reference.md index 3f6667d82..7ddb1c83c 100644 --- a/docs/src/api/reference.md +++ b/docs/src/api/reference.md @@ -69,6 +69,7 @@ For transforming DimensionalData objects: ```@docs groupby +combine DimensionalData.DimGroupByArray Bins ranges diff --git a/src/DimensionalData.jl b/src/DimensionalData.jl index 32247dba3..3cae1d603 100644 --- a/src/DimensionalData.jl +++ b/src/DimensionalData.jl @@ -36,11 +36,14 @@ include("Dimensions/Dimensions.jl") using .Dimensions using .Dimensions.Lookups + using .Dimensions: StandardIndices, DimOrDimType, DimTuple, DimTupleOrEmpty, DimType, AllDims -import .Lookups: metadata, set, _set, rebuild, basetypeof, +import .Dimensions: dims, refdims, name, lookup, kw2dims, hasdim, label, checkaxes, _astuple + +import .Lookups: Safety, Safe, Unsafe, SelectorOrInterval, Begin, End +import .Lookups: metadata, reorder, set, unsafe_set, _set, rebuild, basetypeof, order, span, sampling, locus, val, index, bounds, intervalbounds, - hasselection, units, SelectorOrInterval, Begin, End -import .Dimensions: dims, refdims, name, lookup, kw2dims, hasdim, label, _astuple + hasselection, units using OrderedCollections: OrderedDict @@ -51,6 +54,7 @@ export Lookups, Dimensions # Deprecated const LookupArrays = Lookups const LookupArray = Lookup + export LookupArrays, LookupArray # Dimension @@ -83,7 +87,7 @@ export dimnum, hasdim, hasselection, otherdims export set, rebuild, reorder, modify, broadcast_dims, broadcast_dims!, mergedims, unmergedims, maplayers -export groupby, seasons, months, hours, intervals, ranges +export groupby, combine, seasons, months, hours, intervals, ranges export @d @@ -116,6 +120,7 @@ include("tables.jl") include("plotrecipes.jl") include("utils.jl") include("set.jl") +include("opaque.jl") include("groupby.jl") include("precompile.jl") include("interface_tests.jl") diff --git a/src/Dimensions/Dimensions.jl b/src/Dimensions/Dimensions.jl index aec431d40..158f11634 100644 --- a/src/Dimensions/Dimensions.jl +++ b/src/Dimensions/Dimensions.jl @@ -23,13 +23,14 @@ using .Lookups const LU = Lookups const LookupArrays = Lookups -import .Lookups: rebuild, order, span, sampling, locus, val, index, set, _set, +import .Lookups: rebuild, order, span, sampling, locus, val, index, reorder, set, _set, metadata, bounds, intervalbounds, units, basetypeof, unwrap, selectindices, hasselection, shiftlocus, maybeshiftlocus, ordered_first, ordered_last, ordered_firstindex, ordered_lastindex, promote_first, _remove using .Lookups: StandardIndices, SelTuple, CategoricalEltypes, LookupTrait, AllMetadata, LookupSetters, AbstractBeginEndRange, SelectorOrInterval, Interval +import .Lookups: Safety, Safe, Unsafe using Base: tail, OneTo, @propagate_inbounds diff --git a/src/Dimensions/dimension.jl b/src/Dimensions/dimension.jl index 38d07a631..6e554448c 100644 --- a/src/Dimensions/dimension.jl +++ b/src/Dimensions/dimension.jl @@ -277,6 +277,7 @@ Base.axes(d::Dimension, i) = axes(d)[i] Base.eachindex(d::Dimension) = eachindex(val(d)) Base.length(d::Dimension) = length(val(d)) Base.ndims(d::Dimension) = 0 +Base.parentindices(d::Dimension{<:AbstractArray}) = parentindices(parent(d)) Base.ndims(d::Dimension{<:AbstractArray}) = ndims(val(d)) Base.iterate(d::Dimension{<:AbstractArray}, args...) = iterate(lookup(d), args...) Base.first(d::Dimension) = val(d) diff --git a/src/Dimensions/dimunitrange.jl b/src/Dimensions/dimunitrange.jl index 109ebe7ae..02a19514a 100644 --- a/src/Dimensions/dimunitrange.jl +++ b/src/Dimensions/dimunitrange.jl @@ -2,13 +2,14 @@ struct DimUnitRange{T,R<:AbstractUnitRange{T},D<:Dimension} <: AbstractUnitRange range::R dim::D end - DimUnitRange{T}(r::DimUnitRange{T}) where {T<:Integer} = r -function DimUnitRange{T}(r::DimUnitRange) where {T<:Integer} - return DimUnitRange(AbstractUnitRange{T}(parent(r)), dims(r)) -end +DimUnitRange{T}(r::DimUnitRange) where {T<:Integer} = + DimUnitRange(AbstractUnitRange{T}(parent(r)), dims(r)) + +@inline dims(r::DimUnitRange) = r.dim +@inline dims(rs::Tuple{DimUnitRange,Vararg{DimUnitRange}}) = map(dims, rs) -@inline Base.parent(r::DimUnitRange) = r.range +Base.parent(r::DimUnitRange) = r.range function Base.reduced_index(dur::DimUnitRange) r = Base.reduced_index(parent(dur)) @@ -21,9 +22,6 @@ function Base.reduced_index(dur::DimUnitRange) return DimUnitRange(r, d1) end -@inline dims(r::DimUnitRange) = r.dim -@inline dims(rs::Tuple{DimUnitRange,Vararg{DimUnitRange}}) = map(dims, rs) - # this is necessary to ensure that keyword syntax for DimArray works correctly Base.Slice(r::DimUnitRange) = Base.Slice(parent(r)) diff --git a/src/Dimensions/format.jl b/src/Dimensions/format.jl index ebeb6cb1a..ef774630e 100644 --- a/src/Dimensions/format.jl +++ b/src/Dimensions/format.jl @@ -56,8 +56,9 @@ _format(::Type{D}, axis::AbstractRange) where D<:Dimension = D(NoLookup(axes(axi _format(dim::Dimension{Colon}, axis::AbstractRange) = rebuild(dim, NoLookup(axes(axis, 1))) function _format(dim::Dimension, axis::AbstractRange) newlookup = format(val(dim), basetypeof(dim), axes(axis, 1)) - checkaxis(newlookup, axis) - return rebuild(dim, newlookup) + newdim = rebuild(dim, newlookup) + checkaxis(newdim, axis) + return newdim end format(val::AbstractArray, D::Type, axis::AbstractRange) = format(AutoLookup(), D, val, axis) @@ -153,10 +154,10 @@ _format(locus::AutoLocus, D::Type, values) = Center() _format(locus::AutoLocus, D::Type{<:TimeDim}, values) = Start() _format(locus::Locus, D::Type, values) = locus -_order(values) = first(values) <= last(values) ? ForwardOrdered() : ReverseOrdered() +checkaxis(lookup::Union{Dimension,Lookup}, axis::AbstractUnitRange) = + first(axes(lookup)) == axis || _checkaxiserror(lookup, axis) -checkaxis(lookup::Transformed, axis) = nothing -checkaxis(lookup, axis) = first(axes(lookup)) == axis || _checkaxiserror(lookup, axis) +checkaxes(lookups::Tuple, axes::Tuple) = all(map(checkaxis, lookups, axes)) @noinline _explicitpoints_error() = throw(ArgumentError("Cannot use Explicit span with Points sampling")) @@ -164,9 +165,9 @@ checkaxis(lookup, axis) = first(axes(lookup)) == axis || _checkaxiserror(lookup, throw(ArgumentError("lookup step $(step(span)) does not match lookup step $(step(values))")) @noinline _arraynosteperror() = throw(ArgumentError("`Regular` must specify `step` size with values other than `AbstractRange`")) -@noinline _checkaxiserror(lookup, axis) = +@noinline _checkaxiserror(dimorlookup::Union{Dimension,Lookup}, axis) = throw(DimensionMismatch( - "axes of $(basetypeof(lookup)) of $(first(axes(lookup))) do not match array axis of $axis" + "axes of $(basetypeof(dimorlookup)) of $(first(axes(dimorlookup))) do not match array axis of $axis" )) @noinline _valformaterror(v, D::Type) = throw(ArgumentError( diff --git a/src/Dimensions/set.jl b/src/Dimensions/set.jl index 2b3b78913..385db2265 100644 --- a/src/Dimensions/set.jl +++ b/src/Dimensions/set.jl @@ -1,52 +1,70 @@ -const DimSetters = Union{LookupSetters,Type,UnionAll,Dimension,Symbol} - -set(dim::Dimension, ::Type{T}) where T = set(dim, T()) -set(dims::DimTuple, ::Type{T}) where T = set(dims, T()) -set(dim::Dimension, x::DimSetters) = _set(dim, x) -set(dims_::DimTuple, args::Union{Dimension,DimTuple,Pair}...; kw...) = - - _set(dims_, args...; kw...) -set(dims::DimTuple, l::Lookup) = set(dims, map(d -> basedims(d) => l, dims)...) -set(dims::DimTuple, l::LookupTrait) = set(dims, map(d -> basedims(d) => l, dims)...) -# Convert args/kw to dims and set -_set(dims_::DimTuple, args::Dimension...; kw...) = _set(dims_, (args..., kw2dims(kw)...)) +const DimSetters = Union{Lookup,LookupSetters,Tuple,Dimension,Symbol} + +set(dim::Dimension, x::DimSetters) = _set(Safe(), dim, x) +set(dims::DimTuple, x::DimSetters) = _set(Safe(), dims, x) +set(dims::DimTuple, p::Pair) = _set(Safe(), dims, p) +set(dims::DimTuple, a1::Union{Dimension,Pair}, a2::Union{Dimension,Pair}, args::Union{Dimension,Pair}...) = + _set(Safe(), dims, a1, a2, args...) + +unsafe_set(dim::Dimension, x::DimSetters) = _set(Unsafe(), dim, x) +unsafe_set(dims::DimTuple, x::DimSetters) = _set(Unsafe(), dims, x) +unsafe_set(dims::DimTuple, p::Pair) = _set(Unsafe(), dims, p) +unsafe_set(dims::DimTuple, a1::Union{Dimension,Pair}, a2::Union{Dimension,Pair}, args::Union{Dimension,Pair}...) = + _set(Unsafe(), dims, a1, a2, args...) + +_set(s::Safety, dims::DimTuple, l::LookupSetters) = + _set(s, dims, map(d -> rebuild(d, l), dims)...) + # Convert pairs to wrapped dims and set -_set(dims_::DimTuple, p::Pair, ps::Vararg{Pair}) = _set(dims_, (p, ps...)) -_set(dims_::DimTuple, ps::Tuple{Vararg{Pair}}) = _set(dims_, pairs2dims(ps...)) -_set(dims_::DimTuple, ::Tuple{}) = dims_ +_set(s::Safety, dims::DimTuple, p::Pair, ps::Pair...) = + _set(s, dims, (p, ps...)) +_set(s::Safety, dims::DimTuple, ps::Tuple{Vararg{Pair}}) = + _set(s, dims, pairs2dims(ps...)) +_set(s::Safety, dims::DimTuple, ::Tuple{}) = dims +_set(s::Safety, dims::DimTuple, newdims::Dimension...) = + _set(s, dims, newdims) # Set dims with (possibly unsorted) wrapper vals -_set(dims::DimTuple, wrappers::DimTuple) = begin +_set(s::Safety, dims::DimTuple, wrappers::DimTuple) = begin # Check the dimension types match map(wrappers) do w hasdim(dims, w) || _wrongdimserr(dims, w) end # Missing dims return `nothing` from sortdims - newdims = map(_set, dims, sortdims(wrappers, dims)) + newdims = map(dims, sortdims(wrappers, dims)) do d, w + _set(s, d, w) + end # Swaps existing dims with non-nothing new dims swapdims(dims, newdims) end # Set things wrapped in dims -_set(dim::Dimension, wrapper::Dimension{<:DimSetters}) = - _set(_set(dim, basetypeof(wrapper)), val(wrapper)) +_set(s::Safety, dim::Dimension, wrapper::Dimension{<:DimSetters}) = begin + rewrapped = _set(s, dim, basetypeof(wrapper)) + _set(s, rewrapped, val(wrapper)) +end +_set(s::Safety, dim::Dimension, l::Union{Lookup,LookupSetters}) = + rebuild(dim, _set(s, val(dim), l)) # Set the dim, checking the lookup -_set(dim::Dimension, newdim::Dimension) = _set(newdim, _set(val(dim), val(newdim))) -# Construct types -_set(dim::Dimension, ::Type{T}) where T = _set(dim, T()) -_set(dim::Dimension, key::Symbol) = _set(dim, name2dim(key)) -_set(dim::Dimension, dt::DimType) = basetypeof(dt)(val(dim)) -_set(dim::Dimension, x) = rebuild(dim; val=_set(val(dim), x)) -# Set the lookup -# Otherwise pass this on to set fields on the lookup -_set(dim::Dimension, x::LookupTrait) = rebuild(dim, _set(lookup(dim), x)) +_set(s::Safety, dim::Dimension, newdim::Dimension) = + _set(s, newdim, _set(s, val(dim), val(newdim))) +_set(s::Safety, dim::Dimension, newdim::Dimension{<:Type}) = + _set(s, dim, val(newdim)()) +_set(s::Safety, dim::Dimension, key::Symbol) = _set(s, dim, name2dim(key)) +_set(s::Safety, dim::Dimension, x) = rebuild(dim, _set(s, val(dim), x)) +_set(s::Safety, dim::Dimension, ::Type{T}) where T = _set(s, dim, T()) # Metadata -_set(dim::Dimension, newmetadata::AllMetadata) = rebuild(dim, _set(lookup(dim), newmetadata)) +_set(s::Safety, dim::Dimension, newmetadata::AllMetadata) = + rebuild(dim, _set(s, lookup(dim), newmetadata)) -_set(x::Dimension, ::Nothing) = x -_set(::Nothing, x::Dimension) = x -_set(::Nothing, ::Nothing) = nothing -_set(x, ::Nothing) = x -_set(::Nothing, x) = x +_set(::Safety, x::Dimension, ::Nothing) = x +_set(::Safety, ::Nothing, x::Dimension) = x +_set(::Safety, ::Nothing, ::Nothing) = nothing +_set(::Safety, x, ::Nothing) = x +_set(::Safety, ::Nothing, x) = x +# For ambiguity +_set(::Safety, dims::DimTuple, ::Nothing) = dims +_set(::Safety, dims::Lookup, ::Nothing) = dims -@noinline _wrongdimserr(dims, w) = throw(ArgumentError("dim $(basetypeof(w))) not in $(map(basetypeof, dims))")) +@noinline _wrongdimserr(dims, w) = + throw(ArgumentError("dim $(basetypeof(w))) not in $(map(basetypeof, dims))")) diff --git a/src/Dimensions/utils.jl b/src/Dimensions/utils.jl index ec70d752d..7b804e938 100644 --- a/src/Dimensions/utils.jl +++ b/src/Dimensions/utils.jl @@ -11,3 +11,5 @@ for f in (:shiftlocus, :maybeshiftlocus) end end end + +reorder(dim::Dimension, o::Order) = rebuild(reorder(lookup(dim), o)) diff --git a/src/Lookups/Lookups.jl b/src/Lookups/Lookups.jl index fb7220d87..a1dee6e06 100644 --- a/src/Lookups/Lookups.jl +++ b/src/Lookups/Lookups.jl @@ -59,17 +59,15 @@ export Unaligned, Transformed, ArrayLookup # Deprecated export LookupArray -const StandardIndices = Union{AbstractArray{<:Integer},Colon,Integer,CartesianIndex,CartesianIndices} - # As much as possible keyword rebuild is automatic rebuild(x; kw...) = ConstructionBase.setproperties(x, (; kw...)) -include("metadata.jl") include("lookup_traits.jl") +include("metadata.jl") include("lookup_arrays.jl") +include("beginend.jl") include("predicates.jl") include("selector.jl") -include("beginend.jl") include("indexing.jl") include("methods.jl") include("utils.jl") diff --git a/src/Lookups/beginend.jl b/src/Lookups/beginend.jl index 86fdf190b..a84e0ac70 100644 --- a/src/Lookups/beginend.jl +++ b/src/Lookups/beginend.jl @@ -45,6 +45,8 @@ Base.to_indices(A, inds, (r, args...)::Tuple{<:Union{Begin,End,<:LazyMath},Varar _to_index(inds, a::Int) = a _to_index(inds, ::Begin) = first(inds) _to_index(inds, ::End) = last(inds) +_to_index(inds, ::Type{Begin}) = first(inds) +_to_index(inds, ::Type{End}) = last(inds) _to_index(inds, l::LazyMath{End}) = l.f(last(inds)) _to_index(inds, l::LazyMath{Begin}) = l.f(first(inds)) diff --git a/src/Lookups/lookup_arrays.jl b/src/Lookups/lookup_arrays.jl index 3be595a0c..44d99c3eb 100644 --- a/src/Lookups/lookup_arrays.jl +++ b/src/Lookups/lookup_arrays.jl @@ -32,6 +32,7 @@ Base.first(l::Lookup) = first(parent(l)) Base.last(l::Lookup) = last(parent(l)) Base.firstindex(l::Lookup) = firstindex(parent(l)) Base.lastindex(l::Lookup) = lastindex(parent(l)) +Base.parentindices(l::Lookup) = parentindices(parent(l)) function Base.:(==)(l1::Lookup, l2::Lookup) basetypeof(l1) == basetypeof(l2) && parent(l1) == parent(l2) end @@ -159,11 +160,10 @@ NoLookup() = NoLookup(AutoValues()) rebuild(l::NoLookup; data=parent(l), kw...) = NoLookup(data) # Used in @d broadcasts -struct Length1NoLookup <: AbstractNoLookup end -Length1NoLookup(::AbstractVector) = Length1NoLookup() - -rebuild(l::Length1NoLookup; kw...) = Length1NoLookup() -Base.parent(::Length1NoLookup) = Base.OneTo(1) +struct Length1NoLookup{A<:AbstractUnitRange} <: AbstractNoLookup + data::A +end +Length1NoLookup() = Length1NoLookup(Base.OneTo(1)) """ AbstractSampled <: Aligned @@ -468,7 +468,7 @@ abstract type AbstractCategorical{T,O} <: Aligned{T,O} end order(lookup::AbstractCategorical) = lookup.order metadata(lookup::AbstractCategorical) = lookup.metadata -const CategoricalEltypes = Union{AbstractChar,Symbol,AbstractString} +const CategoricalEltypes = Union{AbstractChar,Symbol,AbstractString,DataType} function Adapt.adapt_structure(to, l::AbstractCategorical) rebuild(l; data=Adapt.adapt(to, parent(l)), metadata=NoMetadata()) @@ -866,6 +866,7 @@ promote_first(x1, x2, xs...) = # Fallback NoLookup if not identical type promote_first(l1::Lookup) = l1 promote_first(l1::L, ls::L...) where L<:Lookup = rebuild(l1; metadata=NoMetadata) +promote_first(l1::L, ls::L...) where L<:AbstractNoLookup = l1 function promote_first(l1::Lookup, ls1::Lookup...) ls = _remove(Length1NoLookup, l1, ls1...) if length(ls) != length(ls1) + 1 diff --git a/src/Lookups/selector.jl b/src/Lookups/selector.jl index 01c6f6d7e..302120d62 100644 --- a/src/Lookups/selector.jl +++ b/src/Lookups/selector.jl @@ -1,3 +1,12 @@ +const StandardIndices = Union{ + AbstractArray{<:Integer}, + Colon, + Integer, + CartesianIndex, + CartesianIndices, + BeginEndRange, +} + struct SelectorError{L,S} <: Exception lookup::L selector::S diff --git a/src/Lookups/set.jl b/src/Lookups/set.jl index 82f73bca5..d0b617cd6 100644 --- a/src/Lookups/set.jl +++ b/src/Lookups/set.jl @@ -1,70 +1,139 @@ -const LookupSetters = Union{AllMetadata,Lookup,LookupTrait,Nothing,AbstractArray} +abstract type Safety end +struct Safe <: Safety end +struct Unsafe <: Safety end -set(lookup::Lookup, x::LookupSetters) = _set(lookup, x) -set(lookup::Lookup, ::Type{T}) where T = _set(lookup, T()) +const LookupSetters = Union{AllMetadata,Lookup,LookupTrait,AbstractArray} -# _set(lookup::Lookup, newlookup::Lookup) = lookup -_set(lookup::AbstractCategorical, newlookup::AutoLookup) = begin - lookup = _set(lookup, parent(newlookup)) - o = _set(order(lookup), order(newlookup)) - md = _set(metadata(lookup), metadata(newlookup)) - rebuild(lookup; order=o, metadata=md) +set(x, ::Type{T}) where T = set(x, T()) +set(lookup::Lookup, x::LookupSetters) =_set(Safe(), lookup, x) +set(a::LookupTrait, b::LookupTrait) =_set(Safe(), a, b) +set(lookup::Lookup, ::Type{T}) where T =_set(Safe(), lookup, T()) + +unsafe_set(lookup::Lookup, x::LookupSetters) =_set(Unsafe(), lookup, x) +unsafe_set(a::LookupTrait, b::LookupTrait) =_set(Unsafe(), a, b) +unsafe_set(lookup::Lookup, ::Type{T}) where T =_set(Unsafe(), lookup, T()) + +# Set with no keywords or arguments does nothing +_set(::Safety, x) = x +_set(s::Safety, lookup::Lookup, newlookup::Lookup) = _set_lookup(s, lookup, newlookup) +_set(s::Safety, lookup::Lookup, newlookup::AbstractArray) = _set_lookup_parent(s, lookup, newlookup) +_set(s::Safety, lookup::Lookup, prop::LookupSetters) = _set_lookup_property(s, lookup, prop) +# Lookup values +_set(::Safety, values::AbstractArray, newvalues::AbstractArray) = newvalues +_set(::Safety, values::AbstractArray, newvalues::Colon) = values +_set(::Safety, values::Colon, newvalues::AbstractArray) = newvalues +_set(::Safety, values::Colon, newvalues::Colon) = values + +_set_lookup(s::Safety, lookup::Lookup, newlookup::AutoLookup) = + _set_lookup_parent(s, lookup, parent(newlookup)) +_set_lookup(s::Safety, lookup::AbstractCategorical, newlookup::AutoLookup) = begin + # With autolookup we have to allow for missing fields and detect them + l1 = _set(Unsafe(), lookup, order(newlookup)) + l2 =_set(Unsafe(), l1, parent(newlookup)) + l3 = _set(s, l2, order(newlookup)) + _set(s, l3, metadata(newlookup)) +end +_set_lookup(s::Safety, lookup::AbstractSampled, newlookup::AutoLookup) = begin + # With autolookup we have to allow for missing fields and detect them + # First force the new order to avoid unnecessary reordering of arrays + # Then update lookup values + lookup1 =_set(s, lookup, parent(newlookup)) + # Then set the order + lookup2 =_set(s, lookup1, order(newlookup)) + # Then set the span + lookup3 =_set(s, lookup2, span(newlookup)) + # Then set traits that dont affect each other + sa =_set(s, sampling(lookup3), sampling(newlookup)) + md =_set(s, metadata(lookup3), metadata(newlookup)) + rebuild(lookup3; sampling=sa, metadata=md) end -_set(lookup::Lookup, newlookup::AbstractCategorical) = begin - lookup = _set(lookup, parent(newlookup)) - o = _set(order(lookup), order(newlookup)) - md = _set(metadata(lookup), metadata(newlookup)) +_set_lookup(s::Unsafe, lookup::Lookup, newlookup::AbstractCategorical) = begin + lookup =_set(s, lookup, parent(newlookup)) + o = _set(s, order(lookup), order(newlookup)) + md = _set(s, metadata(lookup), metadata(newlookup)) rebuild(newlookup; data=parent(lookup), order=o, metadata=md) end -_set(lookup::AbstractSampled, newlookup::AutoLookup) = begin - # Update lookup values - lookup = _set(lookup, parent(newlookup)) - o = _set(order(lookup), order(newlookup)) - sa = _set(sampling(lookup), sampling(newlookup)) - sp = _set(span(lookup), span(newlookup)) - md = _set(metadata(lookup), metadata(newlookup)) - rebuild(lookup; data=parent(lookup), order=o, span=sp, sampling=sa, metadata=md) +_set_lookup(s::Safe, lookup::Lookup, newlookup::AbstractCategorical) = begin + # We need to handle the new lookup having unfilled auto fields + if parent(newlookup) isa AutoValues + o = if order(newlookup) isa AutoLookup + _detect_order(parent(l)) + else + order(newlookup) + end + end + md = _set(s, metadata(lookup), metadata(newlookup)) + # Rebuild the new lookup with updated values + rebuild(newlookup; data=parent(lookup), order=o, metadata=md) end -_set(lookup::Lookup, newlookup::AbstractSampled) = begin +_set_lookup(s::Unsafe, lookup::Lookup, newlookup::AbstractSampled) = + _set_lookup_parent(s, lookup, parent(newlookup)) +_set_lookup(s::Safe, lookup::Lookup, newlookup::AbstractSampled) = begin # Update each field separately. The old lookup may not have these fields, or may have # a subset with the rest being traits. The new lookup may have some auto fields. - lookup = _set(lookup, parent(newlookup)) - o = _set(order(lookup), order(newlookup)) - sp = _set(span(lookup), span(newlookup)) - sa = _set(sampling(lookup), sampling(newlookup)) - md = _set(metadata(lookup), metadata(newlookup)) + data =_set(s, parent(lookup), parent(newlookup) ) + o = _set(order(newlookup), parent(newlookup)) + sp = _set(span(newlookup), parent(newlookup)) + sa = _set(s, sampling(lookup), sampling(newlookup)) + md = _set(s, metadata(lookup), metadata(newlookup)) # Rebuild the new lookup with the merged fields - rebuild(newlookup; data=parent(lookup), order=o, span=sp, sampling=sa, metadata=md) + rebuild(newlookup1; data, order=o, span=sp, sampling=sa, metadata=md) end -_set(lookup::AbstractArray, newlookup::NoLookup{<:AutoValues}) = NoLookup(axes(lookup, 1)) -_set(lookup::Lookup, newlookup::NoLookup{<:AutoValues}) = NoLookup(axes(lookup, 1)) -_set(lookup::Lookup, newlookup::NoLookup) = newlookup +_set_lookup(::Safety, lookup::Lookup, newlookup::NoLookup{<:AutoValues}) = NoLookup(axes(lookup, 1)) +_set_lookup(::Safety, lookup::Lookup, newlookup::AbstractNoLookup) = newlookup # Set the lookup values -_set(lookup::Lookup, values::Val) = rebuild(lookup; data=values) -_set(lookup::Lookup, values::Colon) = lookup -_set(lookup::Lookup, values::AutoLookup) = lookup -_set(lookup::Lookup, values::AbstractArray) = rebuild(lookup; data=values) - -_set(lookup::Lookup, values::AutoValues) = lookup -_set(lookup::Lookup, values::AbstractRange) = - rebuild(lookup; data=_set(parent(lookup), values), order=orderof(values)) -# Update the Sampling lookup of Sampled dims - it must match the range. -_set(lookup::AbstractSampled, values::AbstractRange) = begin - i = _set(parent(lookup), values) - o = orderof(values) - sp = Regular(step(values)) - rebuild(lookup; data=i, span=sp, order=o) +_set_lookup_parent(::Safety, lookup::AbstractCategorical, newlookup::AutoValues) = lookup +_set_lookup_parent(::Safety, lookup::Lookup, values::Colon) = lookup +_set_lookup_parent(::Safety, lookup::Lookup, values::AutoValues) = lookup +_set_lookup_parent(::Safety, lookup::Lookup, values::AbstractVector) = rebuild(lookup; data=values) +_set_lookup_parent(::Safe, lookup::AbstractCategorical, ::AutoValues) = lookup +_set_lookup_parent(s::Safe, lookup::AbstractCategorical, values::AbstractVector) = + rebuild(lookup; data=_set(s, parent(lookup), values), order=orderof(values)) +# Safe detects order and updates span +_set_lookup_parent(::Safe, lookup::AbstractSampled, ::AutoValues) = lookup +_set_lookup_parent(s::Safe, lookup::AbstractSampled, values::AbstractRange) = begin + data =_set(s, parent(lookup), values) + order = orderof(values) + span = Regular(step(values)) + rebuild(lookup; data, span, order) +end +_set_lookup_parent(s::Safe, lookup::AbstractSampled, values::AbstractVector) = begin + data =_set(s, parent(lookup), values) + order = orderof(values) + span = if sampling(lookup) isa Start + Irregular((first(values), nothing)) + elseif sampling(lookup) isa End + Irregular((nothing, last(values))) + else + Irregular((nothing, nothing)) + end + return rebuild(lookup; data, span, order) end # Order -_set(lookup::Lookup, neworder::Order) = rebuild(lookup; order=_set(order(lookup), neworder)) -_set(lookup::NoLookup, neworder::Order) = lookup -_set(order::Order, neworder::Order) = neworder -_set(order::Order, neworder::AutoOrder) = order - -# Span -_set(lookup::AbstractSampled, ::Irregular{AutoBounds}) = begin +_set_lookup_property(::Safe, lookup::Lookup, neworder::AutoOrder) = lookup +_set_lookup_property(::Unsafe, lookup::Lookup, neworder::AutoOrder) = lookup +_set_lookup_property(::Safe, lookup::AbstractNoLookup, neworder::AutoOrder) = lookup +_set_lookup_property(::Unsafe, lookup::AbstractNoLookup, neworder::AutoOrder) = lookup +_set_lookup_property(::Safe, lookup::AbstractNoLookup, neworder::Order) = lookup +_set_lookup_property(::Unsafe, lookup::AbstractNoLookup, neworder::Order) = lookup +# Unsafe leaves the lookup values as-is +_set_lookup_property(s::Unsafe, lookup::Lookup, neworder::Order) = + rebuild(lookup; order=_set(s, order(lookup), neworder)) +_set_lookup_property(s::Unsafe, lookup::Lookup, neworder::AutoOrder) = lookup +# For disambiguity +_set_lookup_property(::Unsafe, lookup::AbstractNoLookup, ::Order) = lookup +_set_lookup_property(::Unsafe, lookup::AbstractNoLookup, ::AutoOrder) = lookup +_set_lookup_property(::Safe, lookup::AbstractNoLookup, ::Order) = lookup +_set_lookup_property(::Safe, lookup::AbstractNoLookup, ::AutoOrder) = lookup +# Safe reorders them to match `neworder` +_set_lookup_property(::Safe, lookup::Lookup, neworder::Order) = + reorder(lookup, neworder) +_set_lookup_property(s::Safe, lookup::Lookup, neworder::AutoOrder) = + lookup +# Lookup Span +_set_lookup_property(::Safety, lookup::AbstractSampled, ::Irregular{AutoBounds}) = begin bnds = if parent(lookup) isa AutoValues || span(lookup) isa AutoSpan AutoBounds() else @@ -72,55 +141,120 @@ _set(lookup::AbstractSampled, ::Irregular{AutoBounds}) = begin end rebuild(lookup; span=Irregular(bnds)) end -_set(lookup::AbstractSampled, ::Regular{AutoStep}) = begin - stp = if span(lookup) isa AutoSpan || step(lookup) isa AutoStep - if parent(lookup) isa AbstractRange - step(parent(lookup)) - else - AutoStep() - end - else +_set_lookup_property(s::Safety, lookup::AbstractSampled, ::Regular{AutoStep}) = begin + stp = if span(lookup) isa Regular step(lookup) + else + stp = _detect_step(parent(lookup)) + isnothing(stp) && throw(ArgumentError("Can't set an irregular lookup values to Regular")) + stp end rebuild(lookup; span=Regular(stp)) end -_set(lookup::AbstractSampled, span::Span) = rebuild(lookup; span=span) -_set(lookup::AbstractSampled, span::AutoSpan) = lookup -_set(span::Span, newspan::Span) = newspan -_set(span::Span, newspan::AutoSpan) = span +_set_lookup_property(::Safety, lookup::AbstractSampled, newspan::AutoSpan) = lookup +_set_lookup_property(s::Safety, lookup::AbstractSampled, newspan::Span) = + _set_lookup_property(s, lookup, span(lookup), newspan) +function _set_lookup_property( + s::Safe, lookup::AbstractSampled, ::Span, ::Explicit{<:AutoBounds} +) + # Generate a new bounds matrix + span = Explicit(reinterpret(reshape, Float64, intervalbounds(lookup))) + # Explicit has to be Intervals + sampling = Intervals(locus(lookup)) + return rebuild(lookup; span, sampling) +end +_set_lookup_property(::Safe, lookup::AbstractSampled, ::Explicit, ::Explicit{<:AutoBounds}) = + lookup +function _set_lookup_property(::Safety, lookup::AbstractSampled, ::Span, span::Explicit) + rebuild(lookup; span, sampling=Intervals(locus(lookup))) +end +function _set_lookup_property( + ::Safe, lookup::AbstractSampled, ::Span, ::Regular{<:AutoStep} +) + rebuild(lookup; sampling=Intervals(bounds(lookup))) +end +function _set_lookup_property( + ::Safe, lookup::AbstractSampled, ::Span, ::Irregular{<:AutoBounds} +) + rebuild(lookup; sampling=Intervals(bounds(lookup))) +end +_set_lookup_property(::Safety, lookup::AbstractSampled, ::Span, newspan::Span) = + rebuild(lookup; span=newspan) +# Lookup Sampling +function _set_lookup_property(s::Safe, lookup::AbstractSampled, newsampling::Sampling) + s = _set(s, sampling(lookup), newsampling) + # If the locus is currently points, make it Center Intervals + if sampling(lookup) isa Points + if s isa Intervals # Points => Intervals + span1 = if span(lookup) isa Irregular + # We don't know the bounds + Irregular(nothing, nothing) + else + span(lookup) + end + lookup1 = rebuild(lookup; sampling=Intervals(Center())) + else # Points => Points, Nothing to do here + return lookup + end + else # Intervals => Points + span1 = if span(lookup) isa Union{Irregular,Explicit} + # We don't need bounds for Points + Irregular(nothing, nothing) + else # Regular stays the same + span(lookup) + end + lookup1 = lookup + end + # For Intervals this will shift the locus + # For Points always convert all loci to Center() + return rebuild(shiftlocus(locus(s), lookup1); sampling=s) +end +_set_lookup_property(s::Unsafe, lookup::AbstractSampled, sampling::Sampling) = + rebuild(lookup; sampling=_set(s, sampling(lookup), sampling)) +# Lookup Locus +_set_lookup_property(s::Unsafe, lookup::AbstractSampled, locus::Locus) = + rebuild(lookup; sampling=_set(s, sampling(lookup), locus)) +_set(::Safe, lookup::AbstractSampled, locus::Locus) = maybeshiftlocus(locus, lookup) +# Lookup Metadata +_set_lookup_property(::Safety, lookup::Lookup, newmetadata::AllMetadata) = + rebuild(lookup; metadata=newmetadata) +# Order +_set(::Safety, order::Order, neworder::Order) = neworder +_set(::Safety, order::Order, neworder::AutoOrder) = order +# Span +_set(::Safety, span::Span, newspan::Span) = newspan +_set(::Safety, span::Span, newspan::AutoSpan) = span # Sampling -_set(lookup::AbstractSampled, newsampling::Sampling) = - rebuild(lookup; sampling=_set(sampling(lookup), newsampling)) -_set(lookup::AbstractSampled, sampling::AutoSampling) = lookup -_set(sampling::Sampling, newsampling::Sampling) = newsampling -_set(sampling::Sampling, newsampling::AutoSampling) = sampling -_set(sampling::Sampling, newsampling::Intervals) = - _set(newsampling, _set(locus(sampling), locus(newsampling))) - +_set(::Safety, sampling::Sampling, newsampling::Sampling) = newsampling +_set(::Safety, sampling::Sampling, newsampling::AutoSampling) = sampling +_set(s::Safety, sampling::Sampling, newsampling::Intervals) = + _set(s, newsampling,_set(s, locus(sampling), locus(newsampling))) # Locus -_set(lookup::AbstractSampled, locus::Locus) = - rebuild(lookup; sampling=_set(sampling(lookup), locus)) -_set(sampling::Points, locus::Union{AutoPosition,Center}) = Points() -_set(sampling::Points, locus::Locus) = _locuserror() -_set(sampling::Intervals, locus::Locus) = Intervals(locus) -_set(sampling::Intervals, locus::AutoPosition) = sampling - -_set(locus::Locus, newlocus::Locus) = newlocus -_set(locus::Locus, newlocus::AutoPosition) = locus - +_set(::Safety, sampling::Points, locus::Union{AutoPosition,Center}) = Points() +_set(::Safety, sampling::Points, locus::Locus) = _locuserror() +_set(::Safety, sampling::Intervals, locus::Locus) = Intervals(locus) +_set(::Safety, sampling::Intervals, locus::AutoPosition) = sampling +_set(::Safety, locus::Locus, newlocus::Locus) = newlocus +_set(::Safety, locus::Locus, newlocus::AutoPosition) = locus # Metadata -_set(lookup::Lookup, newmetadata::AllMetadata) = rebuild(lookup; metadata=newmetadata) -_set(metadata::AllMetadata, newmetadata::AllMetadata) = newmetadata - -# Lookup values -_set(values::AbstractArray, newvalues::AbstractArray) = newvalues -_set(values::AbstractArray, newvalues::AutoLookup) = values -_set(values::AbstractArray, newvalues::Colon) = values -_set(values::Colon, newvalues::AbstractArray) = newvalues -_set(values::Colon, newvalues::Colon) = values +_set(::Safety, metadata::AllMetadata, newmetadata::AllMetadata) = newmetadata -_set(A, x) = _cantseterror(A, x) +# Other things error +_set(::Safety, A, x) = _cantseterror(A, x) @noinline _locuserror() = throw(ArgumentError("Can't set a locus for `Points` sampling other than `Center` - the lookup values are the exact points")) @noinline _cantseterror(a, b) = throw(ArgumentError("Can not set any fields of $(typeof(a)) to $(typeof(b))")) + +_detect_step(::AutoValues) = AutoStep() +_detect_step(A::AbstractRange) = step(A) +function _detect_step(A::AbstractVector) + step = (last(A) - first(A)) / (length(A) - 1) + for i in eachindex(A) + isapprox(A[i], first(A) + step * (i - 1)) || return nothing + end + return step +end + +_detect_order(values) = + first(values) <= last(values) ? ForwardOrdered() : ReverseOrdered() \ No newline at end of file diff --git a/src/Lookups/utils.jl b/src/Lookups/utils.jl index 93e54fe9b..ed7e2ed10 100644 --- a/src/Lookups/utils.jl +++ b/src/Lookups/utils.jl @@ -10,7 +10,7 @@ function shiftlocus(locus::Locus, lookup::Lookup) samp isa Intervals || error("Cannot shift locus of $(nameof(typeof(samp)))") newvalues = _shiftlocus(locus, lookup) newlookup = rebuild(lookup; data=newvalues) - return set(newlookup, locus) + return unsafe_set(newlookup, locus) end # Fallback - no shifting @@ -118,3 +118,26 @@ _order(A::AbstractArray{<:IntervalSets.Interval}) = first(A).left <= last(A).lef Base.@assume_effects :foldable _remove(::Type{T}, x, xs...) where T = (x, _remove(T, xs...)...) Base.@assume_effects :foldable _remove(::Type{T}, ::T, xs...) where T = _remove(T, xs...) Base.@assume_effects :foldable _remove(::Type) = () + + + +reorder(lookup::Lookup, neworder::Order) = _reorder(lookup, neworder) + +# Reorder +_reorder(lookup::Lookup, neworder::Order) = + _reorder(lookup, order(lookup), neworder) +# Same order, do nothing +_reorder(lookup::Lookup, ::O, ::O) where {O<:Ordered} = lookup +_reorder(lookup::Lookup, ::O, ::O) where {O<:Unordered} = lookup +# We can always convert to `Unordered` without changing the lookup +_reorder(lookup::Lookup, ::Ordered, neworder::Unordered) = + rebuild(lookup; order=neworder) +# Different order, reverse the lookup +_reorder(lookup::Lookup, ::Ordered, neworder::Ordered) = reverse(lookup) +# To set to `Ordered` we need to sort +function _reorder(l::Lookup, ::Unordered, o::Ordered) + # We use sortperm then index in case there is a bounds matrix + # or similar, rather than just sort on lookup values + idxs = sortperm(l; rev=isrev(o)) + return rebuild(l[idxs]; order=o) +end \ No newline at end of file diff --git a/src/array/array.jl b/src/array/array.jl index 0b776fa67..ec414e5ba 100644 --- a/src/array/array.jl +++ b/src/array/array.jl @@ -1,4 +1,5 @@ const IDim = Dimension{<:StandardIndices} +const MaybeDimTuple = Tuple{Vararg{Dimension}} """ AbstractBasicDimArray <: AbstractArray @@ -8,7 +9,7 @@ returns a `Tuple` of `Dimension` Only keyword `rebuild` is guaranteed to work with `AbstractBasicDimArray`. """ -abstract type AbstractBasicDimArray{T,N,D<:Tuple} <: AbstractArray{T,N} end +abstract type AbstractBasicDimArray{T,N,D<:MaybeDimTuple} <: AbstractArray{T,N} end const AbstractBasicDimVector = AbstractBasicDimArray{T,1} where T const AbstractBasicDimMatrix = AbstractBasicDimArray{T,2} where T @@ -94,10 +95,27 @@ metadata(A::AbstractDimArray) = A.metadata layerdims(A::AbstractDimArray) = basedims(A) -@inline rebuildsliced(A::AbstractBasicDimArray, args...) = rebuildsliced(getindex, A, args...) -@inline function rebuildsliced(f::Function, A::AbstractBasicDimArray, data::AbstractArray, I::Tuple, name=name(A)) +""" + rebuildsliced(f::Function, A::AbstractBasicDimArray, I) + +Rebuild `AbstractDimArray` where `f` is `getindex` , `view` or `dotview`. + +This does not need to be defined for `AbstractDimArray`, as `f` +is simply called on the parent array, dims and refdims are sliced with `slicedims`, +and `rebuild` is called. + +However for custom `AbstractBasicDimArray`, `rebuildsliced` methods are needed +to define slicing behavior, as there not be a parent array. +""" +@propagate_inbounds rebuildsliced(A::AbstractBasicDimArray, args...) = rebuildsliced(getindex, A, args...) +@propagate_inbounds function rebuildsliced(f::Function, A::AbstractDimArray, I::Tuple, name=name(A)) + I1 = to_indices(A, I) + data = f(parent(A), I1...) + return rebuildsliced(f, A, data, I1, name) +end +@propagate_inbounds function rebuildsliced(f::Function, A::AbstractDimArray, data::AbstractArray, I::Tuple, name=name(A)) I1 = to_indices(A, I) - rebuild(A, data, slicedims(f, A, I1)..., name) + return rebuild(A, data, slicedims(f, A, I1)..., name) end # Array interface methods ###################################################### @@ -107,6 +125,7 @@ Base.axes(A::AbstractDimArray) = map(Dimensions.DimUnitRange, axes(parent(A)), d Base.iterate(A::AbstractDimArray, args...) = iterate(parent(A), args...) Base.IndexStyle(A::AbstractDimArray) = Base.IndexStyle(parent(A)) Base.parent(A::AbstractDimArray) = data(A) +Base.parentindices(A::AbstractDimArray) = parentindices(parent(A)) Base.vec(A::AbstractDimArray) = vec(parent(A)) # Only compare data and dim - metadata and refdims can be different Base.:(==)(A1::AbstractDimArray, A2::AbstractDimArray) = @@ -170,14 +189,6 @@ end # An alternative would be to fill missing dims with `Anon`, and keep existing # dims but strip the Lookup? It just seems a little complicated when the methods # below using DimTuple work better anyway. -Base.similar(A::AbstractDimArray, i::Integer, I::Vararg{Integer}; kw...) = - similar(A, eltype(A), (i, I...); kw...) -Base.similar(A::AbstractDimArray, I::Tuple{Int,Vararg{Int}}; kw...) = - similar(A, eltype(A), I; kw...) -Base.similar(A::AbstractDimArray, ::Type{T}, i::Integer, I::Vararg{Integer}; kw...) where T = - similar(A, T, (i, I...); kw...) -Base.similar(A::AbstractDimArray, ::Type{T}, I::Tuple{Int,Vararg{Int}}; kw...) where T = - similar(parent(A), T, I) const MaybeDimUnitRange = Union{Integer,Base.OneTo,Dimensions.DimUnitRange} # when all axes are DimUnitRanges we can return an `AbstractDimArray` @@ -256,14 +267,27 @@ function _similar(::Type{T}, shape::Tuple; kw...) where {T<:AbstractArray} end # With Dimensions we can return an `AbstractDimArray` -Base.similar(A::AbstractBasicDimArray, D::DimTuple; kw...) = Base.similar(A, eltype(A), D; kw...) -Base.similar(A::AbstractBasicDimArray, D::Dimension...; kw...) = Base.similar(A, eltype(A), D; kw...) -Base.similar(A::AbstractBasicDimArray, ::Type{T}, D::Dimension...; kw...) where T = - Base.similar(A, T, D; kw...) +Base.similar(A::AbstractBasicDimArray, d1::Dimension, D::Dimension...; kw...) = + Base.similar(A, eltype(A), (d1, D...); kw...) +Base.similar(A::AbstractBasicDimArray, ::Type{T}, d1::Dimension, D::Dimension...; kw...) where T = + Base.similar(A, T, (d1, D...); kw...) +Base.similar(A::AbstractBasicDimArray, D::DimTuple; kw...) = + Base.similar(A, eltype(A), D; kw...) +function Base.similar(A::AbstractBasicDimArray, ::Type{T}, D::DimTuple; kw...) where T + data = _arraytype(T)(undef, _dimlength(D)) + dimconstructor(D)(data, D; kw...) +end +function Base.similar(A::AbstractBasicDimArray, ::Type{T}, D::Tuple{}; + refdims=(), name=_noname(A), metadata=NoMetadata(), kw... +) where T + data = _arraytype(T)(undef, _dimlength(D)) + dimconstructor(D)(data, (); refdims, name, metadata, kw...) +end + function Base.similar(A::AbstractDimArray, ::Type{T}, D::DimTuple; refdims=(), name=_noname(A), metadata=NoMetadata(), kw... ) where T - data = similar(parent(A), T, _dimlength(D)) + data = _arraytype(T)(undef, _dimlength(D)) dims = _maybestripval(D) return rebuild(A; data, dims, refdims, metadata, name, kw...) end @@ -274,6 +298,20 @@ function Base.similar(A::AbstractDimArray, ::Type{T}, D::Tuple{}; rebuild(A; data, dims=(), refdims, metadata, name, kw...) end +Base.similar(A::AbstractBasicDimArray, shape::Int...; kw...) = + similar(A, eltype(A), shape; kw...) +Base.similar(A::AbstractBasicDimArray, shape::Tuple{Vararg{Int}}; kw...) = + similar(A, eltype(A), shape; kw...) +Base.similar(A::AbstractBasicDimArray, ::Type{T}, shape::Int...; kw...) where T = + similar(A, T, shape; kw...) +Base.similar(A::AbstractBasicDimArray, ::Type{T}, shape::Tuple{Vararg{Int}}; kw...) where T = + _arraytype(T)(undef, shape) +Base.similar(A::AbstractDimArray, ::Type{T}, shape::Tuple{Vararg{Int}}; kw...) where T = + similar(parent(A), T, shape) + +_arraytype(::Type{T}) where T = Array{T} +_arraytype(::Type{Bool}) = BitArray + # Keep the same type in `similar` _noname(A::AbstractBasicDimArray) = _noname(name(A)) _noname(s::String) = "" diff --git a/src/array/broadcast.jl b/src/array/broadcast.jl index b6f6f27d9..30b86383f 100644 --- a/src/array/broadcast.jl +++ b/src/array/broadcast.jl @@ -35,7 +35,9 @@ strict_broadcast!(x::Bool) = STRICT_BROADCAST_CHECKS[] = x # It preserves the dimension names. # `S` should be the `BroadcastStyle` of the wrapped type. # Copied from NamedDims.jl (thanks @oxinabox). -struct DimensionalStyle{S <: BroadcastStyle} <: AbstractArrayStyle{Any} end +struct BasicDimensionalStyle{N} <: AbstractArrayStyle{Any} end + +struct DimensionalStyle{S<:BroadcastStyle} <: AbstractArrayStyle{Any} end DimensionalStyle(::S) where {S} = DimensionalStyle{S}() DimensionalStyle(::S, ::Val{N}) where {S,N} = DimensionalStyle(S(Val(N))) DimensionalStyle(::Val{N}) where N = DimensionalStyle{DefaultArrayStyle{N}}() @@ -53,6 +55,8 @@ function BroadcastStyle(::Type{<:AbstractDimArray{T,N,D,A}}) where {T,N,D,A} inner_style = typeof(BroadcastStyle(A)) return DimensionalStyle{inner_style}() end +BroadcastStyle(::Type{<:AbstractBasicDimArray{T,N}}) where {T,N} = + BasicDimensionalStyle{N}() BroadcastStyle(::DimensionalStyle, ::Base.Broadcast.Unknown) = Unknown() BroadcastStyle(::Base.Broadcast.Unknown, ::DimensionalStyle) = Unknown() @@ -79,12 +83,31 @@ function Broadcast.copy(bc::Broadcasted{DimensionalStyle{S}}) where S dims = format(Dimensions.promotedims(bdims...; skip_length_one=true), data) return rebuild(A; data, dims, refdims=refdims(A), name=Symbol("")) end +function Broadcast.copy(bc::Broadcasted{BasicDimensionalStyle{N}}) where N + A = _firstdimarray(bc) + data = collect(bc) + A isa Nothing && return data # No AbstractDimArray + + bdims = _broadcasted_dims(bc) + _comparedims_broadcast(A, bdims...) + + data isa AbstractArray || return data # result is a scalar + + # Return an AbstractDimArray + dims = format(Dimensions.promotedims(bdims...; skip_length_one=true), data) + return dimconstructor(dims)(data, dims; refdims=refdims(A), name=Symbol("")) +end function Base.copyto!(dest::AbstractArray, bc::Broadcasted{DimensionalStyle{S}}) where S fda = _firstdimarray(bc) isnothing(fda) || _comparedims_broadcast(fda, _broadcasted_dims(bc)...) copyto!(dest, _unwrap_broadcasted(bc)) end +function Base.copyto!(dest::AbstractArray, bc::Broadcasted{BasicDimensionalStyle{N}}) where N + fda = _firstdimarray(bc) + isnothing(fda) || _comparedims_broadcast(fda, _broadcasted_dims(bc)...) + copyto!(dest, bc) +end @inline function Base.Broadcast.materialize!(dest::AbstractDimArray, bc::Base.Broadcast.Broadcasted{<:Any}) # Need to check whether the dims are compatible in dest, @@ -97,7 +120,15 @@ end function Base.similar(bc::Broadcast.Broadcasted{DimensionalStyle{S}}, ::Type{T}) where {S,T} A = _firstdimarray(bc) - rebuildsliced(A, similar(_unwrap_broadcasted(bc), T, axes(bc)...), axes(bc), Symbol("")) + data = similar(_unwrap_broadcasted(bc), T, size(bc)) + dims, refdims = slicedims(A, axes(bc)) + return rebuild(A; data, dims, refdims, name=Symbol("")) +end +function Base.similar(bc::Broadcast.Broadcasted{BasicDimensionalStyle{N}}, ::Type{T}) where {N,T} + A = _firstdimarray(bc) + data = similar(A, T, size(bc)) + dims, refdims = slicedims(A, axes(bc)) + return dimconstructor(dims)(data, dims; refdims, name=Symbol("")) end @@ -383,9 +414,10 @@ _unwrap_broadcasted(boda::BroadcastOptionsDimArray) = parent(parent(boda)) # Get the first dimensional array in the broadcast _firstdimarray(x::Broadcasted) = _firstdimarray(x.args) -_firstdimarray(x::Tuple{<:AbstractDimArray,Vararg}) = x[1] +_firstdimarray(x::Tuple{<:AbstractBasicDimArray,Vararg}) = x[1] +_firstdimarray(x::AbstractBasicDimArray) = x _firstdimarray(ext::Base.Broadcast.Extruded) = _firstdimarray(ext.x) -function _firstdimarray(x::Tuple{<:Broadcasted,Vararg}) +function _firstdimarray(x::Tuple{<:Union{Broadcasted,Base.Broadcast.Extruded},Vararg}) found = _firstdimarray(x[1]) if found isa Nothing _firstdimarray(tail(x)) diff --git a/src/array/indexing.jl b/src/array/indexing.jl index ecce5fbef..68dc41c28 100644 --- a/src/array/indexing.jl +++ b/src/array/indexing.jl @@ -1,112 +1,87 @@ -# getindex/view/setindex! ====================================================== +const SelectorOrStandard = Union{SelectorOrInterval,StandardIndices} +const DimensionIndsArrays = Union{AbstractArray{<:Dimension},AbstractArray{<:DimTuple}} +const DimensionalIndices = Union{DimTuple,DimIndices,DimSelectors,Dimension,DimensionIndsArrays} +const _DimIndicesAmb = Union{AbstractArray{Union{}},DimIndices{<:Integer},DimSelectors{<:Integer}} +const IntegerOrCartesian = Union{Integer,CartesianIndex} -#### getindex/view #### +# getindex/view/setindex! ====================================================== for f in (:getindex, :view, :dotview) _dim_f = Symbol(:_dim_, f) + + # Integer indexing if f === :view - # No indices and we try to rebuild, for 0d - @eval @propagate_inbounds Base.view(A::AbstractDimArray) = rebuild(A, Base.view(parent(A)), ()) # With one Integer and 0d and 1d we try to rebuild - @eval @propagate_inbounds Base.$f(A::AbstractDimArray{<:Any,0}, i::Integer) = - rebuildsliced(Base.$f, A, Base.$f(parent(A), i), (i,)) - @eval @propagate_inbounds Base.$f(A::AbstractDimVector, i::Integer) = - rebuildsliced(Base.$f, A, Base.$f(parent(A), i), (i,)) + @eval @propagate_inbounds Base.$f(A::AbstractBasicDimArray{<:Any,0}, i::Integer) = + rebuildsliced(Base.$f, A, (i,)) + # One Integer on a vector and we also rebuild + @eval @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::Integer) = + rebuildsliced(Base.$f, A, (i,)) + # More Integers and we rebuild + @eval @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i1::Integer, i2::Integer, I::Integer...) = + rebuildsliced(Base.$f, A, (i1, i2, I...)) # Otherwise its linear indexing, don't rebuild - @eval @propagate_inbounds Base.$f(A::AbstractDimArray, i::Integer) = + @eval @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i::Integer) = Base.$f(parent(A), i) - # More Integer and we rebuild again - @eval @propagate_inbounds Base.$f(A::AbstractDimArray, i1::Integer, i2::Integer, I::Integer...) = - rebuildsliced(Base.$f, A, Base.$f(parent(A), i1, i2, I...), (i1, i2, I...)) - else - @eval @propagate_inbounds Base.$f(A::AbstractDimVector, i::Integer) = Base.$f(parent(A), i) - @eval @propagate_inbounds Base.$f(A::AbstractDimArray, i::Integer) = Base.$f(parent(A), i) - @eval @propagate_inbounds Base.$f(A::AbstractDimArray, i1::Integer, i2::Integer, I::Integer...) = - Base.$f(parent(A), i1, i2, I...) - @eval @propagate_inbounds Base.$f(A::AbstractDimArray) = Base.$f(parent(A)) end @eval begin - @propagate_inbounds Base.$f(A::AbstractDimVector, I::CartesianIndex) = + ### Standard indices + @propagate_inbounds Base.$f(A::AbstractBasicDimVector, I::CartesianIndex) = Base.$f(A, to_indices(A, (I,))...) - @propagate_inbounds Base.$f(A::AbstractDimArray, I::CartesianIndex) = + @propagate_inbounds Base.$f(A::AbstractBasicDimArray, I::CartesianIndex) = Base.$f(A, to_indices(A, (I,))...) - @propagate_inbounds Base.$f(A::AbstractDimVector, I::CartesianIndices) = - rebuildsliced(Base.$f, A, Base.$f(parent(A), I), (I,)) - @propagate_inbounds Base.$f(A::AbstractDimArray, I::CartesianIndices) = - rebuildsliced(Base.$f, A, Base.$f(parent(A), I), (I,)) - @propagate_inbounds function Base.$f(A::AbstractDimVector, i) - x = Base.$f(parent(A), Lookups._construct_types(i)) - if x isa AbstractArray - rebuildsliced(Base.$f, A, x, to_indices(A, (i,))) - else - x - end - end - @propagate_inbounds function Base.$f(A::AbstractDimArray, i1, i2, Is...) - I = Lookups._construct_types(i1, i2, Is...) - x = Base.$f(parent(A), I...) - if x isa AbstractArray - rebuildsliced(Base.$f, A, x, to_indices(A, I)) - else - x - end - end - # Linear indexing forwards to the parent array as it will break the dimensions - @propagate_inbounds Base.$f(A::AbstractDimArray, i::Union{Colon,AbstractArray{<:Integer}}) = - Base.$f(parent(A), i) - # Except 1D DimArrays - @propagate_inbounds Base.$f(A::AbstractDimVector, i::Union{Colon,AbstractArray{<:Integer}}) = - rebuildsliced(Base.$f, A, Base.$f(parent(A), i), (i,)) - @propagate_inbounds Base.$f(A::AbstractDimVector, i::SelectorOrInterval) = + @eval @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i1::IntegerOrCartesian, i2::IntegerOrCartesian, Is::IntegerOrCartesian...) = + Base.$f(A, to_indices(A, (i1, i2, Is...))...) + # 1D DimArrays dont need linear indexing + @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::Union{Colon,AbstractArray{<:Integer}}) = + rebuildsliced(Base.$f, A, (i,)) + @propagate_inbounds Base.$f(A::AbstractBasicDimVector, I::CartesianIndices) = rebuildsliced(Base.$f, A, (I,)) + @propagate_inbounds Base.$f(A::AbstractBasicDimArray, I::CartesianIndices) = rebuildsliced(Base.$f, A, (I,)) + @eval @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i1::StandardIndices, i2::StandardIndices, Is::StandardIndices...) = + rebuildsliced(Base.$f, A, to_indices(A, (i1, i2, Is...))) + + ### Selector/Interval indexing + @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::SelectorOrInterval) = Base.$f(A, dims2indices(A, (i,))...) - # Selector/Interval indexing - @propagate_inbounds Base.$f(A::AbstractDimArray, i1::SelectorOrStandard, i2::SelectorOrStandard, I::SelectorOrStandard...) = + @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i1::SelectorOrStandard, i2::SelectorOrStandard, I::SelectorOrStandard...) = Base.$f(A, dims2indices(A, (i1, i2, I...))...) + @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::Selector{<:Extents.Extent}) = + Base.$f(A, dims2indices(A, i)...) + @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i::Selector{<:Extents.Extent}) = + Base.$f(A, dims2indices(A, i)...) - @propagate_inbounds Base.$f(A::AbstractDimVector, extent::Union{Extents.Extent,Near{<:Extents.Extent},Touches{<:Extents.Extent}}) = - Base.$f(A, dims2indices(A, extent)...) - @propagate_inbounds Base.$f(A::AbstractDimArray, extent::Union{Extents.Extent,Near{<:Extents.Extent},Touches{<:Extents.Extent}}) = + # Extent indexing + @propagate_inbounds Base.$f(A::AbstractBasicDimVector, extent::Extents.Extent) = Base.$f(A, dims2indices(A, extent)...) - @propagate_inbounds Base.$f(A::AbstractBasicDimVector, extent::Union{Extents.Extent,Near{<:Extents.Extent},Touches{<:Extents.Extent}}) = + @propagate_inbounds Base.$f(A::AbstractBasicDimArray, extent::Extents.Extent) = Base.$f(A, dims2indices(A, extent)...) - @propagate_inbounds Base.$f(A::AbstractBasicDimArray, extent::Union{Extents.Extent,Near{<:Extents.Extent},Touches{<:Extents.Extent}}) = - Base.$f(A, dims2indices(A, extent)...) - # All Dimension indexing modes combined - @propagate_inbounds Base.$f(A::AbstractBasicDimArray; kw...) = - $_dim_f(A, _simplify_dim_indices(kw2dims(values(kw))...,)...) + + ### Dimension indexing + @propagate_inbounds function Base.$f(A::AbstractBasicDimArray; kw...) + # Need to use one method and check keywords to avoid method overwrites + if isempty(kw) + rebuildsliced(Base.$f, A, ()) + else + $_dim_f(A, _simplify_dim_indices(kw2dims(values(kw))...,)...) + end + end @propagate_inbounds Base.$f(A::AbstractBasicDimArray, d1::DimensionalIndices; kw...) = $_dim_f(A, _simplify_dim_indices(d1, kw2dims(values(kw))...)...) @propagate_inbounds Base.$f(A::AbstractBasicDimArray, d1::DimensionalIndices, d2::DimensionalIndices, D::DimensionalIndices...; kw...) = $_dim_f(A, _simplify_dim_indices(d1, d2, D..., kw2dims(values(kw))...)...) - @propagate_inbounds Base.$f(A::AbstractDimArray, i1::DimensionalIndices, i2::DimensionalIndices, I::DimensionalIndices...) = - $_dim_f(A, _simplify_dim_indices(i1, i2, I...)...) - @propagate_inbounds Base.$f(A::AbstractDimArray, i1::_DimIndicesAmb, i2::_DimIndicesAmb, I::_DimIndicesAmb...) = - $_dim_f(A, _simplify_dim_indices(i1, i2, I...)...) - @propagate_inbounds Base.$f(A::AbstractDimVector, i::DimensionalIndices) = - $_dim_f(A, _simplify_dim_indices(i)...) @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::DimensionalIndices) = $_dim_f(A, _simplify_dim_indices(i)...) - # For ambiguity - @propagate_inbounds Base.$f(A::AbstractDimArray, i::DimIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArray, i::DimSelectors) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArray, i::_DimIndicesAmb) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimVector, i::DimIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimVector, i::DimSelectors) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimVector, i::_DimIndicesAmb) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i::DimIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i::DimSelectors) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractBasicDimArray, i::_DimIndicesAmb) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::DimIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::DimSelectors) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractBasicDimVector, i::_DimIndicesAmb) = $_dim_f(A, i) - - # Use underscore methods to minimise ambiguities - @propagate_inbounds $_dim_f(A::AbstractBasicDimArray, ds::DimTuple) = - $_dim_f(A, ds...) + + # All dimension indexing is passed to these underscore methods to minimise ambiguities + @propagate_inbounds $_dim_f(A::AbstractBasicDimArray, ds::DimTuple) = $_dim_f(A, ds...) @propagate_inbounds $_dim_f(A::AbstractBasicDimArray, d1::Dimension, ds::Dimension...) = Base.$f(A, dims2indices(A, (d1, ds...))...) - @propagate_inbounds $_dim_f(A::AbstractBasicDimArray, ds::Dimension...) = - Base.$f(A, dims2indices(A, ds)...) + # Regular non-dimensional indexing + @propagate_inbounds $_dim_f(A::AbstractBasicDimArray, I...) = Base.$f(A, I...) + # Catch the edge case dims were passed but did not match - + # we want to index with all colons [:, :, ...], not [] + @propagate_inbounds $_dim_f(A::AbstractBasicDimArray{<:Any,N}) where N = + rebuildsliced(Base.$f, A, ntuple(i -> Colon(), Val(N))) @propagate_inbounds function $_dim_f( A::AbstractBasicDimArray, d1::Union{Dimension,DimensionIndsArrays}, @@ -114,35 +89,41 @@ for f in (:getindex, :view, :dotview) ) return merge_and_index(Base.$f, A, (d1, ds...)) end - end - # Standard indices - if f == :view - @eval @propagate_inbounds function Base.$f(A::AbstractDimArray, i1::StandardIndices, i2::StandardIndices, I::StandardIndices...) - I = to_indices(A, (i1, i2, I...)) - x = Base.$f(parent(A), I...) - rebuildsliced(Base.$f, A, x, I) - end - else - @eval @propagate_inbounds function Base.$f(A::AbstractDimArray, i1::StandardIndices, i2::StandardIndices, Is::StandardIndices...) - I = to_indices(A, (i1, i2, Is...)) - x = Base.$f(parent(A), I...) - all(i -> i isa Integer, I) ? x : rebuildsliced(Base.$f, A, x, I) + @propagate_inbounds function $_dim_f(A::AbstractBasicDimArray{<:Any,0}, d1::Dimension, ds::Dimension...) + Dimensions._extradimswarn((d1, ds...)) + return rebuildsliced(Base.$f, A, ()) end end + + ##### AbstractDimArray only methods + # Here we know we can just index into the parent object + # Linear indexing forwards to the parent array as it will break the dimensions + # AbstractBasicDimArray must defined their own methods + @eval @propagate_inbounds Base.$f(A::AbstractDimArray, i::Union{Colon,AbstractArray{<:Integer}}) = + Base.$f(parent(A), i) + # Except for AbstractDimVector + @eval @propagate_inbounds Base.$f(A::AbstractDimVector, i::Union{Colon,AbstractArray{<:Integer}}) = + rebuildsliced(Base.$f, A, (i,)) + if f in (:getindex, :dotview) + # We only define getindex with Integer on AbstractDimArray + # AbstractBasicDimArray must defined their own + @eval @propagate_inbounds Base.$f(A::AbstractDimVector, i::Integer) = Base.$f(parent(A), i) + @eval @propagate_inbounds Base.$f(A::AbstractDimArray, i::Integer) = Base.$f(parent(A), i) + @eval @propagate_inbounds Base.$f(A::AbstractDimArray, i1::Integer, i2::Integer, I::Integer...) = + Base.$f(parent(A), i1, i2, I...) + @eval @propagate_inbounds Base.$f(A::AbstractDimArray) = Base.$f(parent(A)) + end # Special case zero dimensional arrays being indexed with missing dims if f == :getindex # Catch this before the dimension is converted to () - @eval @propagate_inbounds function $_dim_f(A::AbstractDimArray{<:Any,0}) - return rebuild(A, fill(A[])) - end - @eval @propagate_inbounds function $_dim_f(A::AbstractDimArray{<:Any,0}, d1::Dimension, ds::Dimension...) + @eval $_dim_f(A::AbstractDimArray{<:Any,0}) = rebuild(A, fill(A[])) + @eval function $_dim_f(A::AbstractDimArray{<:Any,0}, d1::Dimension, ds::Dimension...) Dimensions._extradimswarn((d1, ds...)) return rebuild(A, fill(A[])) end end end - function merge_and_index(f, A, dims) dims, inds_arrays = _separate_dims_arrays(_simplify_dim_indices(dims...)...) # No arrays here, so abort (dispatch is tricky...) @@ -263,3 +244,7 @@ Base.@assume_effects :foldable @inline _simplify_dim_indices() = () view(A, args...; kw...) @propagate_inbounds Base.maybeview(A::AbstractDimArray, args::Vararg{Union{Number,Base.AbstractCartesianIndex}}; kw...) = view(A, args...; kw...) + +# We only own this to_indices dispatch for AbstractBasicDimArray +Base.to_indices(A::AbstractBasicDimArray, inds, (r, args...)::Tuple{<:Type,Vararg}) = + (Lookups._to_index(inds[1], r), to_indices(A, Base.tail(inds), args)...) \ No newline at end of file diff --git a/src/array/methods.jl b/src/array/methods.jl index c4a63ad8d..dec9f8c4f 100644 --- a/src/array/methods.jl +++ b/src/array/methods.jl @@ -90,7 +90,7 @@ end function Base.dropdims(A::AbstractDimArray; dims) dims = DD.dims(A, dims) data = Base.dropdims(parent(A); dims=dimnum(A, dims)) - rebuildsliced(A, data, _dropinds(A, dims)) + rebuildsliced(view, A, data, _dropinds(A, dims)) end @inline _dropinds(A, dims::Tuple) = dims2indices(A, map(d -> rebuild(d, 1), dims)) @@ -421,7 +421,7 @@ function _check_cat_lookups(D, ::Regular, lookups...) @warn _cat_warn_string(D, "step sizes $(step(span(l))) and $s do not match") return false end - if !(lastval + s ≈ first(l)) + if !(s isa Dates.AbstractTime) && !(lastval + s ≈ first(l)) @warn _cat_warn_string(D, "`Regular` lookups do not join with the correct step size: $(lastval) + $s ≈ $(first(l)) should hold") return false end @@ -582,7 +582,8 @@ end r = axes(A) # Copied from Base.diff r0 = ntuple(i -> i == dims ? UnitRange(1, last(r[i]) - 1) : UnitRange(r[i]), N) - rebuildsliced(A, diff(parent(A); dims=dimnum(A, dims)), r0) + data = diff(parent(A); dims=dimnum(A, dims)) + rebuildsliced(getindex, A, data, r0) end # Forward `replace` to parent objects diff --git a/src/dimindices.jl b/src/dimindices.jl index 5b0779b18..a168cad8a 100644 --- a/src/dimindices.jl +++ b/src/dimindices.jl @@ -1,16 +1,21 @@ +""" + AbstractDimArrayGenerator <: AbstractBasicDimArray +Abstract supertype for all AbstractBasicDimArrays that +generate their `data` on demand during `getindex`. +""" abstract type AbstractDimArrayGenerator{T,N,D} <: AbstractBasicDimArray{T,N,D} end dims(dg::AbstractDimArrayGenerator) = dg.dims +# Dims that contribute to the element type. +# May be larger than `dims` after slicing +eldims(di::AbstractDimArrayGenerator) = dims((dims(di)..., refdims(di)...), orderdims(di)) +eldims(di::AbstractDimArrayGenerator, d) = dims(eldims(di), d) + Base.size(dg::AbstractDimArrayGenerator) = map(length, dims(dg)) Base.axes(dg::AbstractDimArrayGenerator) = map(d -> axes(d, 1), dims(dg)) -Base.similar(A::AbstractDimArrayGenerator, ::Type{T}, D::DimTuple) where T = - dimconstructor(D)(A; data=similar(Array{T}, size(D)), dims=D, refdims=(), metadata=NoMetadata()) -Base.similar(A::AbstractDimArrayGenerator, ::Type{T}, D::Tuple{}) where T = - dimconstructor(D)(A; data=similar(Array{T}, ()), dims=(), refdims=(), metadata=NoMetadata()) - @inline Base.permutedims(A::AbstractDimArrayGenerator{<:Any,2}) = rebuild(A; dims=reverse(dims(A))) @inline Base.permutedims(A::AbstractDimArrayGenerator{<:Any,1}) = @@ -25,7 +30,34 @@ end rebuild(A; dims=dims(dims(A), Tuple(perm))) end -abstract type AbstractDimIndices{T,N,D} <: AbstractDimArrayGenerator{T,N,D} end +""" + AbstractRebuildableDimArrayGenerator <: AbstractDimArrayGenerator + +Abstract supertype for all AbstractDimArrayGenerator that +can be rebuilt when subsetted with `view` or `getindex`. + +These arrays must have `dims` and `refdims` fields that defined the data +They do not need to define `rebuildsliced` methods as this is defined +as simply doing `slicedims` on `dims` and `refdims` and rebuilding. +""" +abstract type AbstractRebuildableDimArrayGenerator{T,N,D,R<:MaybeDimTuple} <: AbstractDimArrayGenerator{T,N,D} end + +refdims(A::AbstractRebuildableDimArrayGenerator) = A.refdims + +_refdims_firsts(A::AbstractRebuildableDimArrayGenerator) = map(d -> rebuild(d, first(d)), refdims(A)) + +# Custom rebuildsliced where data is ignored, and just dims and refdims are slices +# This makes sense for AbstractRebuildableDimArrayGenerator because Arrays are +# generated in getindex from the dims/refdims combination. +# `f` is ignored, and views are always used +@propagate_inbounds function rebuildsliced(f::Function, A::AbstractRebuildableDimArrayGenerator, I) + dims, refdims = slicedims(view, A, I) + return rebuild(A; dims, refdims) +end + +abstract type AbstractDimIndices{T,N,D,R,O<:MaybeDimTuple} <: AbstractRebuildableDimArrayGenerator{T,N,D,R} end + +orderdims(di::AbstractDimIndices) = di.orderdims """ DimIndices <: AbstractArray @@ -85,48 +117,52 @@ julia> A[di] # Index A with these indices 0.6 0.745673 0.692209 ``` """ -struct DimIndices{T,N,D<:Tuple{Vararg{Dimension}}} <: AbstractDimIndices{T,N,D} +struct DimIndices{T<:MaybeDimTuple,N,D,R,O} <: AbstractDimIndices{T,N,D,R,O} dims::D + refdims::R + orderdims::O # Manual inner constructor for ambiguity only - function DimIndices{T,N,D}(dims::Tuple{Vararg{Dimension}}) where {T,N,D<:Tuple{Vararg{Dimension}}} - new{T,N,D}(dims) + function DimIndices(dims::D, refdims::R, orderdims::O) where {D<:MaybeDimTuple,R<:MaybeDimTuple,O<:MaybeDimTuple} + eldims = DD.dims((dims..., refdims...), orderdims) + T = typeof(map(d -> rebuild(d, 1), eldims)) + N = length(dims) + new{T,N,D,R,O}(dims, refdims, orderdims) end end -function DimIndices(dims::D) where {D<:Tuple{Vararg{Dimension}}} - T = typeof(map(d -> rebuild(d, 1), dims)) - N = length(dims) - dims = N > 0 ? _dimindices_format(dims) : dims - DimIndices{T,N,typeof(dims)}(dims) +function DimIndices(dims::MaybeDimTuple) + dims = length(dims) > 0 ? _dimindices_format(dims) : dims + return DimIndices(dims, (), basedims(dims)) end DimIndices(x) = DimIndices(dims(x)) DimIndices(dim::Dimension) = DimIndices((dim,)) DimIndices(::Nothing) = throw(ArgumentError("Object has no `dims` method")) # Forces multiple indices not linear -function Base.getindex(di::DimIndices, i1::Integer, i2::Integer, I::Integer...) - map(dims(di), (i1, i2, I...)) do d, i +function Base.getindex(A::DimIndices, i1::Integer, i2::Integer, I::Integer...) + dis = map(dims(A), (i1, i2, I...)) do d, i rebuild(d, d[i]) end + dims((dis..., _refdims_firsts(A)...), orderdims(A)) end # Dispatch to avoid linear indexing in multidimensional DimIndices -function Base.getindex(di::DimIndices{<:Any,1}, i::Integer) - d = dims(di, 1) - (rebuild(d, d[i]),) +function Base.getindex(A::DimIndices{<:Any,1}, i::Integer) + d = dims(A, 1) + di = rebuild(d, d[i]) + return dims((di, _refdims_firsts(A)...), orderdims(A)) end +Base.getindex(A::DimIndices{<:Any,0}) = dims(_refdims_firsts(A), orderdims(A)) _dimindices_format(dims::Tuple{}) = () _dimindices_format(dims::Tuple) = map(rebuild, dims, map(_dimindices_axis, dims)) -# Allow only CartesianIndices arguments _dimindices_axis(x::Integer) = Base.OneTo(x) _dimindices_axis(x::AbstractRange{<:Integer}) = x -# And Lookup, which we take the axes from _dimindices_axis(x::Dimension) = _dimindices_axis(val(x)) _dimindices_axis(x::Lookup) = axes(x, 1) _dimindices_axis(x) = throw(ArgumentError("`$x` is not a valid input for `DimIndices`. Use `Dimension`s wrapping `Integer`, `AbstractArange{<:Integer}`, or a `Lookup` (the `axes` will be used)")) -abstract type AbstractDimVals{T,N,D} <: AbstractDimIndices{T,N,D} end +abstract type AbstractDimVals{T,N,D,R,O} <: AbstractDimIndices{T,N,D,R,O} end (::Type{T})(::Nothing; kw...) where T<:AbstractDimVals = throw(ArgumentError("Object has no `dims` method")) (::Type{T})(x; kw...) where T<:AbstractDimVals = T(dims(x); kw...) @@ -150,34 +186,40 @@ that defines a `dims` method can be passed in. - `order`: determines the order of the points, the same as the order of `dims` by default. """ -struct DimPoints{T,N,D<:Tuple{Vararg{Dimension}},O} <: AbstractDimVals{T,N,D} +struct DimPoints{T<:Tuple,N,D,R,O} <: AbstractDimVals{T,N,D,R,O} dims::D - order::O + refdims::R + orderdims::O + function DimPoints(dims::D, refdims::R, orderdims::O) where {D<:MaybeDimTuple,R<:MaybeDimTuple,O<:MaybeDimTuple} + eldims = DD.dims((dims..., refdims...), orderdims) + T = Tuple{map(eltype, eldims)...} + N = length(dims) + new{T,N,D,R,O}(dims, refdims, orderdims) + end end DimPoints(dims::Tuple; order=dims) = DimPoints(dims, order) function DimPoints(dims::Tuple, order::Tuple) - order = map(d -> basetypeof(d)(), order) - T = Tuple{map(eltype, dims)...} - N = length(dims) - dims = N > 0 ? _format(dims) : dims - DimPoints{T,N,typeof(dims),typeof(order)}(dims, order) + dims = length(dims) > 0 ? format(dims) : dims + DimPoints(dims, (), basedims(order)) end -function Base.getindex(dp::DimPoints, i1::Integer, i2::Integer, I::Integer...) +function Base.getindex(A::DimPoints, i1::Integer, i2::Integer, I::Integer...) # Get dim-wrapped point values at i1, I... - pointdims = map(dims(dp), (i1, i2, I...)) do d, i + pointdims = map(dims(A), (i1, i2, I...)) do d, i rebuild(d, d[i]) end # Return the unwrapped point sorted by `order - return map(val, DD.dims(pointdims, dp.order)) + return map(val, DD.dims((pointdims..., _refdims_firsts(A)...), orderdims(A))) end -Base.getindex(di::DimPoints{<:Any,1}, i::Integer) = (dims(di, 1)[i],) - -_format(::Tuple{}) = () -function _format(dims::Tuple) - ax = map(d -> axes(val(d), 1), dims) - return format(dims, ax) +function Base.getindex(A::DimPoints{<:Any,1}, i::Integer) + # Get dim-wrapped point values at i1, I... + d1 = dims(A, 1) + pointdim = rebuild(d1, d1[i]) + # Return the unwrapped point sorted by `order + D = dims((pointdim, _refdims_firsts(A)...), orderdims(A)) + return map(val, D) end +Base.getindex(A::DimPoints{<:Any,0}) = map(val, dims(_refdims_firsts(A), orderdims(A))) """ DimSelectors <: AbstractArray @@ -226,19 +268,49 @@ Using `At` would make sure we only use exact interpolation, while `Contains` with sampling of `Intervals` would make sure that each values is taken only from an Interval that is present in the lookups. """ -struct DimSelectors{T,N,D<:Tuple{Vararg{Dimension}},S<:Tuple} <: AbstractDimVals{T,N,D} +struct DimSelectors{T<:MaybeDimTuple,N,D,R,O,S<:Tuple} <: AbstractDimVals{T,N,D,R,O} dims::D + refdims::R + orderdims::O selectors::S + function DimSelectors(dims::D, refdims::R, orderdims::O, selectors::S) where {D<:Tuple,R<:Tuple,O<:Tuple,S<:Tuple} + eldims = DD.dims((dims..., refdims...), orderdims) + T = _selector_eltype(eldims, selectors) + N = length(dims) + new{T,N,D,R,O,S}(dims, refdims, orderdims, selectors) + end end -function DimSelectors(dims::Tuple{Vararg{Dimension}}; atol=nothing, selectors=At()) +function DimSelectors(dims::MaybeDimTuple; atol=nothing, selectors=At()) s = _format_selectors(dims, selectors, atol) DimSelectors(dims, s) end -function DimSelectors(dims::Tuple{Vararg{Dimension}}, selectors::Tuple) - T = _selector_eltype(dims, selectors) - N = length(dims) - dims = N > 0 ? _format(dims) : dims - DimSelectors{T,N,typeof(dims),typeof(selectors)}(dims, selectors) +function DimSelectors(dims::MaybeDimTuple, selectors::Tuple) + dims = length(dims) > 0 ? format(dims) : dims + orderdims = basedims(dims) + refdims = () + length(dims) == length(selectors) || throw(ArgumentError("`length(dims) must match `length(selectors)`, got $(length(dims)) and $(length(selectors))")) + DimSelectors(dims, refdims, orderdims, selectors) +end + +@propagate_inbounds function Base.getindex(A::DimSelectors, i1::Integer, i2::Integer, I::Integer...) + D = map(dims(A), (i1, i2, I...)) do d, i + rebuild(d, d[i]) + end + return _rebuild_selectors(A, D) +end +@propagate_inbounds function Base.getindex(A::DimSelectors{<:Any,1}, i::Integer) + d1 = dims(A, 1) + d = rebuild(d1, d1[i]) + return _rebuild_selectors(A, (d,)) +end +@propagate_inbounds Base.getindex(A::DimSelectors{<:Any,0}) = + _rebuild_selectors(A, ()) + +function _rebuild_selectors(A, D) + sorteddims = dims((D..., _refdims_firsts(A)...), orderdims(A)) + map(sorteddims, A.selectors) do d, s + rebuild(d, rebuild(s; val=val(d))) + end end _selector_eltype(dims::Tuple, selectors::Tuple) = @@ -268,8 +340,7 @@ end _format_selectors(dims, selectors, map(_ -> atol, dims)) @inline _format_selectors(dims::Tuple, selectors::Tuple, atol::Tuple) = map(_format_selectors, dims, selectors, atol) - -_format_selectors(d::Dimension, T::Type, atol) = _format_selectors(d, T(), atol) +@inline _format_selectors(d::Dimension, T::Type, atol) = _format_selectors(d, T(), atol) @inline _format_selectors(d::Dimension, ::Near, atol) = Near(nothing) @inline _format_selectors(d::Dimension, ::Contains, atol) = Contains(nothing) @inline function _format_selectors(d::Dimension, at::At, atol) @@ -282,68 +353,129 @@ _atol(T::Type{<:AbstractFloat}, atol, ::Nothing) = atol _atol(T::Type{<:AbstractFloat}, ::Nothing, atol) = atol _atol(T::Type{<:AbstractFloat}, ::Nothing, ::Nothing) = eps(T) -@propagate_inbounds function Base.getindex(di::DimSelectors, i1::Integer, i2::Integer, I::Integer...) - map(dims(di), di.selectors, (i1, i2, I...)) do d, s, i - rebuild(d, rebuild(s; val=d[i])) # At selector with the value at i - end -end -@propagate_inbounds function Base.getindex(di::DimSelectors{<:Any,1}, i::Integer) - d = dims(di, 1) - (rebuild(d, rebuild(di.selectors[1]; val=d[i])),) -end - # Deprecated const DimKeys = DimSelectors -struct DimSlices{T,N,D<:Tuple{Vararg{Dimension}},P} <: AbstractDimArrayGenerator{T,N,D} +const SliceDim = Dimension{<:Union{<:AbstractVector{Int},<:AbstractVector{<:AbstractVector{Int}}}} + +""" + DimSlices <: AbstractRebuildableDimArrayGenerator + + DimSlices(x, dims; drop=true) + +A `Base.Slices` like object for returning view slices from a DimArray. + +This is used for `eachslice` on stacks. + +`dims` must be a `Tuple` of `Dimension` holding `AbstractVector{Int}` +or `AbstractVector{<:AbstractVector{Int}}`. + +# Keywords + +- `drop`: whether to drop dimensions from the outer array or keep the + same dimensions as the inner view, but with length 1. +""" +struct DimSlices{T,N,D,R,P,U} <: AbstractRebuildableDimArrayGenerator{T,N,D,R} _data::P dims::D + refdims::R + reduced::U end DimSlices(x; dims, drop=true) = DimSlices(x, dims; drop) -function DimSlices(x, dims; drop=true) - newdims = if length(dims) == 0 - map(d -> rebuild(d, :), DD.dims(x)) +DimSlices(x, dim; kw...) = DimSlices(x, (dim,); kw...) +function DimSlices(x, dims::Tuple; drop::Union{Bool,Nothing}=nothing) + dims = DD.dims(x, dims) + refdims = () + inds = if length(dims) == 0 + map(d -> rebuild(d, :), DD.dims(x)) + else + map(d -> rebuild(d, firstindex(d)), dims) + end + slicedims, reduced = if isnothing(drop) || drop + # We have to handle filling in colons for no dims because passing + # no dims at all is owned by base to mean A[] not A[D1(:), D2(:), D3(:)] + dims, () else - dims - end - inds = map(newdims) do d - rebuild(d, first(d)) - end - # `getindex` returns these views + # Get other dimensions as length 1 + reduced = map(otherdims(x, dims)) do o + reducedims(o) + end + # Re-sort to x dim order + slicedims = DD.dims((reduced..., dims...), DD.dims(x)) + sliceddims, basedims(reduced) + end T = typeof(view(x, inds...)) - N = length(newdims) - D = typeof(newdims) - P = typeof(x) - return DimSlices{T,N,D,P}(x, newdims) + N = length(slicedims) + D = typeof(slicedims) + R = typeof(refdims) + A = typeof(x) + U = typeof(reduced) + return DimSlices{T,N,D,R,A,U}(x, slicedims, refdims, reduced) end -rebuild(ds::A; dims) where {A<:DimSlices{T,N}} where {T,N} = - DimSlices{T,N,typeof(dims),typeof(ds._data)}(ds._data, dims) +function rebuild(ds::DimSlices{T,N}; + dims::D, refdims::R, reduced::U=ds.reduced +) where {T,N,D,R,U} + A = typeof(ds._data) + DimSlices{T,N,D,R,A,U}(ds._data, dims, refdims, reduced) +end +@propagate_inbounds function rebuildsliced(::Function, A::DimSlices, I) + @boundscheck checkbounds(A, I...) + # We use `unafe_view` to force always wrapping as a view, even for ranges + # Then in `_refdims_firsts` we can use `first(parentindices(d))` to get the offset + dims, refdims = slicedims(Base.unsafe_view, A, I) + return rebuild(A; dims, refdims) +end + +# We need to get the vist index from the view, so define this custom for DimSlices +_refdims_firsts(A::DimSlices) = map(d -> rebuild(d, first(parentindices(d))), refdims(A)) function Base.summary(io::IO, A::DimSlices{T,N}) where {T,N} print_ndims(io, size(A)) print(io, string(nameof(typeof(A)), "{$(nameof(T)),$N}")) end -@propagate_inbounds function Base.getindex(ds::DimSlices, i1::Integer, i2::Integer, Is::Integer...) +@propagate_inbounds function Base.getindex(A::DimSlices, i1::Integer, i2::Integer, Is::Integer...) I = (i1, i2, Is...) - @boundscheck checkbounds(ds, I...) - D = map(dims(ds), I) do d, i - rebuild(d, d[i]) + D = map(dims(A), I) do d, i + i1 = if hasdim(A.reduced, d) + @boundscheck checkbounds(d, i) + Colon() + else + eachindex(d)[i] + end + return rebuild(d, i1) end - return view(ds._data, D...) + R = _refdims_firsts(A) + return view(A._data, D..., R...) end # Dispatch to avoid linear indexing in multidimensional DimIndices -@propagate_inbounds function Base.getindex(ds::DimSlices{<:Any,1}, i::Integer) - d = dims(ds, 1) - return view(ds._data, rebuild(d, d[i])) +@propagate_inbounds function Base.getindex(A::DimSlices{<:Any,1}, i::Integer) + d1 = dims(A, 1) + d = if hasdim(A.reduced, d1) + @boundscheck checkbounds(d1, i) + rebuild(d1, :) + else + rebuild(d1, eachindex(d1)[i]) + end + return view(A._data, d, _refdims_firsts(A)...) +end +@propagate_inbounds function Base.getindex(A::DimSlices{<:Any,0}) + R = _refdims_firsts(A) + # Need to manually force the Colons in case there are no dims at all + D = map(otherdims(A._data, R)) do d + rebuild(d, :) + end + view(A._data, D..., R...) end # Extends the dimensions of any `AbstractBasicDimArray` # as if the array assigned into a larger array across all dimensions, # but without the copying. Theres is a cost for linear indexing these objects # as we need to convert to Cartesian. -struct DimExtensionArray{T,N,D<:Tuple{Vararg{Dimension}},R<:Tuple{Vararg{Dimension}},A<:AbstractBasicDimArray{T}} <: AbstractDimArrayGenerator{T,N,D} +struct DimExtensionArray{ + T,N,D<:MaybeDimTuple,R<:MaybeDimTuple,A<:AbstractBasicDimArray{T} +} <: AbstractDimArrayGenerator{T,N,D} _data::A dims::D refdims::R @@ -361,71 +493,41 @@ DimExtensionArray(A::AbstractBasicDimArray, dims::Tuple; refdims=refdims(A)) = name(A::DimExtensionArray) = name(A._data) metadata(A::DimExtensionArray) = metadata(A._data) -# Indexing that returns a new object with the same number of dims -for f in (:getindex, :dotview, :view) - __f = Symbol(:__, f) - T = Union{Colon,AbstractRange} - # For ambiguity - @eval @propagate_inbounds function Base.$f(de::DimExtensionArray{<:Any,1}, i::Integer) - if ndims(parent(de)) == 0 - $f(de._data) - else - $f(de._data, i) - end - end - @eval @propagate_inbounds function Base.$f(di::DimExtensionArray{<:Any,1}, i::Union{AbstractRange,Colon}) - rebuild(di; _data=di.data[i], dims=(dims(di, 1)[i],)) - end - # For ambiguity - @eval @propagate_inbounds function Base.$f(de::DimExtensionArray, i1::$T, i2::$T, Is::$T...) - $__f(de, i1, i2, Is...) - end - @eval @propagate_inbounds function Base.$f(de::DimExtensionArray, i1::StandardIndices, i2::StandardIndices, Is::StandardIndices...) - $__f(de, i1, i2, Is...) - end - @eval @propagate_inbounds function Base.$f( - de::DimensionalData.DimExtensionArray, - i1::Union{AbstractArray{Union{}}, DimensionalData.DimIndices{<:Integer}, DimensionalData.DimSelectors{<:Integer}}, - i2::Union{AbstractArray{Union{}}, DimensionalData.DimIndices{<:Integer}, DimensionalData.DimSelectors{<:Integer}}, - Is::Vararg{Union{AbstractArray{Union{}}, DimensionalData.DimIndices{<:Integer}, DimensionalData.DimSelectors{<:Integer}}} - ) - $__f(de, i1, i2, Is...) - end - @eval Base.@assume_effects :foldable @propagate_inbounds function $__f(de::DimExtensionArray, i1, i2, Is...) - I = (i1, i2, Is...) - newdims, newrefdims = slicedims(dims(de), refdims(de), I) - D = map(rebuild, dims(de), I) - A = de._data - realdims = dims(D, dims(A)) - if all(map(d -> val(d) isa Colon, realdims)) - rebuild(de; dims=newdims, refdims=newrefdims) - else - newrealparent = begin - x = parent(A)[dims2indices(A, realdims)...] - x isa AbstractArray ? x : fill(x) - end - newrealdims = dims(newdims, realdims) - newdata = rebuild(A; data=newrealparent, dims=newrealdims) - rebuild(de; _data=newdata, dims=newdims, refdims=newrefdims) +@propagate_inbounds function rebuildsliced(f::Function, de::DimExtensionArray, I) + newdims, newrefdims = slicedims(dims(de), refdims(de), I) + D = map(rebuild, dims(de), I) + A = de._data + realdims = dims(D, dims(A)) + if all(map(d -> val(d) isa Colon, realdims)) + rebuild(de; dims=newdims, refdims=newrefdims) + else + newrealparent = begin + x = f(parent(A), dims2indices(A, realdims)...) + x isa AbstractArray ? x : fill(x) end - end - @eval @propagate_inbounds function $__f(de::DimExtensionArray{<:Any,1}, i::$T) - newdims, _ = slicedims(dims(de), (i,)) - A = de._data - D = rebuild(only(dims(de)), i) - rebuild(de; dims=newdims, _data=A[D...]) + newrealdims = dims(newdims, realdims) + newdata = rebuild(A; data=newrealparent, dims=newrealdims) + rebuild(de; _data=newdata, dims=newdims, refdims=newrefdims) end end -for f in (:getindex, :dotview) - __f = Symbol(:__, f) - @eval function $__f(de::DimExtensionArray, i1::Int, i2::Int, Is::Int...) - D = map(rebuild, dims(de), (i1, i2, Is...)) - A = de._data - return $f(A, dims(D, dims(A))...) - end - @eval $__f(de::DimExtensionArray{<:Any,1}, i::Int) = $f(de._data, rebuild(dims(de, 1), i)) +@propagate_inbounds function rebuildsliced( + f::Function, de::DimExtensionArray{<:Any,1}, I::Tuple{<:Union{Colon,AbstractRange}} +) + newdims, _ = slicedims(dims(de), I) + A = de._data + D = rebuild(only(dims(de)), only(I)) + rebuild(de; dims=newdims, _data=A[D...]) end +# Integer indexing +function Base.getindex(de::DimExtensionArray, i1::Integer, i2::Integer, Is::Integer...) + D = map(rebuild, dims(de), (i1, i2, Is...)) + A = de._data + return getindex(A, dims(D, dims(A))...) +end +Base.getindex(de::DimExtensionArray{<:Any,1}, i::Integer) = getindex(de._data, rebuild(dims(de, 1), i)) +Base.getindex(de::DimExtensionArray{<:Any,0}) = de._data[] + function mergedims(A::DimExtensionArray, dim_pairs::Pair...) all_dims = dims(A) dims_new = mergedims(all_dims, dim_pairs...) @@ -434,47 +536,4 @@ function mergedims(A::DimExtensionArray, dim_pairs::Pair...) Aperm = PermutedDimsArray(A, dims_perm) data_merged = reshape(parent(Aperm), map(length, dims_new)) return DimArray(data_merged, dims_new) -end - -const SelectorOrStandard = Union{SelectorOrInterval,StandardIndices} -const DimensionIndsArrays = Union{AbstractArray{<:Dimension},AbstractArray{<:DimTuple}} -const DimensionalIndices = Union{DimTuple,DimIndices,DimSelectors,Dimension,DimensionIndsArrays} -const _DimIndicesAmb = Union{AbstractArray{Union{}},DimIndices{<:Integer},DimSelectors{<:Integer}} - -# Indexing that returns a new object with the same number of dims -for f in (:getindex, :dotview, :view) - T = Union{Colon,AbstractVector} - _dim_f = Symbol(:_dim_, f) - @eval begin - @propagate_inbounds function Base.$f(di::AbstractDimArrayGenerator, i1::$T, i2::$T, Is::$T...) - I = (i1, i2, Is...) - newdims, _ = slicedims(dims(di), I) - rebuild(di; dims=newdims) - end - @propagate_inbounds function Base.$f( - di::AbstractDimArrayGenerator, - i1::DimensionalIndices, - i2::DimensionalIndices, - Is::DimensionalIndices... - ) - $_dim_f(di, i1, i2, Is...) - end - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator, i::DimIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator, i::DimSelectors) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator, i::DimensionalIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator, i::_DimIndicesAmb) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator{<:Any,1}, i::DimIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator{<:Any,1}, i::DimSelectors) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator{<:Any,1}, i::DimensionalIndices) = $_dim_f(A, i) - @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator{<:Any,1}, i::_DimIndicesAmb) = $_dim_f(A, i) - @propagate_inbounds Base.$f(di::AbstractDimArrayGenerator{<:Any,1}, i::$T) = - rebuild(di; dims=(dims(di, 1)[i],)) - @propagate_inbounds Base.$f(dg::AbstractDimArrayGenerator, i::Integer) = - Base.$f(dg, Tuple(CartesianIndices(dg)[i])...) - end - if f == :view - @eval @propagate_inbounds Base.$f(A::AbstractDimArrayGenerator) = A - else - @eval @propagate_inbounds Base.$f(::AbstractDimArrayGenerator) = () - end -end +end \ No newline at end of file diff --git a/src/groupby.jl b/src/groupby.jl index 6ee94a2b4..a73e5a9ca 100644 --- a/src/groupby.jl +++ b/src/groupby.jl @@ -11,24 +11,47 @@ This wrapper allows for specialisations on later broadcast or reducing operations, e.g. for chunk reading with DiskArrays.jl, because we know the data originates from a single array. """ -struct DimGroupByArray{T,N,D<:Tuple,R<:Tuple,A<:AbstractArray{T,N},Na,Me} <: AbstractDimArray{T,N,D,A} - data::A +struct DimGroupByArray{T,N,D,R,A,Na,Me} <: AbstractRebuildableDimArrayGenerator{T,N,D,R} + _data::A dims::D refdims::R name::Na metadata::Me - function DimGroupByArray( - data::A, dims::D, refdims::R, name::Na, metadata::Me - ) where {D<:Tuple,R<:Tuple,A<:AbstractArray{T,N},Na,Me} where {T,N} - checkdims(data, dims) - new{T,N,D,R,A,Na,Me}(data, dims, refdims, name, metadata) - end +end +function DimGroupByArray(_data::A, dims::D, refdims::R, name::Na, metadata::Me) where {D,R,A,Na,Me} + T = typeof(view(_data, firstgroupinds(dims)...)) + N = length(dims) + DimGroupByArray{T,N,D,R,A,Na,Me}(_data, dims, refdims, name, metadata) end function DimGroupByArray(data::AbstractArray, dims::Union{Tuple,NamedTuple}; refdims=(), name=NoName(), metadata=NoMetadata() ) DimGroupByArray(data, format(dims, data), refdims, name, metadata) end + +name(A::DimGroupByArray) = A.name +metadata(A::DimGroupByArray) = A.metadata +refdims(A::DimGroupByArray) = A.refdims + +function groupinds(A::DimGroupByArray, I::Integer...) + # Get the group indices for each dimension + D = groupinds(dims(A), I...) + # And the indices in refdims + R = map(refdims(A)) do d + rebuild(d, only(hiddenparent(d))) + end + # And combine them, dim indexing will fix the order later + return (D..., R...) +end +function groupinds(dims::MaybeDimTuple, I::Integer...) + map(dims, I) do d, i + rebuild(d, hiddenparent(d)[i]) + end +end + +firstgroupinds(dims::MaybeDimTuple) = + groupinds(dims, ntuple(_ -> 1, length(dims))...) + @inline function rebuild( A::DimGroupByArray, data::AbstractArray, dims::Tuple, refdims::Tuple, name, metadata ) @@ -41,6 +64,18 @@ end rebuild(A, data, dims, refdims, name, metadata) # Rebuild as a regular DimArray end +Base.size(A::DimGroupByArray) = map(length, dims(A)) + +@propagate_inbounds function Base.getindex(A::DimGroupByArray, i1::Integer, i2::Integer, Is::Integer...) + I = (i1, i2, Is...) + return view(A._data, groupinds(A, I...)...) +end +# Dispatch to avoid linear indexing in multidimensional DimIndices +@propagate_inbounds Base.getindex(A::DimGroupByArray{<:Any,1}, i::Integer) = + view(A._data, groupinds(A, i)...) +@propagate_inbounds Base.getindex(A::DimGroupByArray{<:Any,0}) = + view(A._data, groupinds(A)...) + function Base.summary(io::IO, A::DimGroupByArray{T,N}) where {T<:AbstractArray{T1,N1},N} where {T1,N1} print_ndims(io, size(A)) print(io, string(nameof(typeof(A)), "{$(nameof(T)){$T1,$N1},$N}")) @@ -83,18 +118,6 @@ function Base.show(io::IO, s::DimSummariser) end Base.alignment(io::IO, s::DimSummariser) = (textwidth(sprint(show, s)), 0) -# An array that doesn't know what it holds, to simplify dispatch -# It can also hold something that is not an AbstractArray itself. -struct OpaqueArray{T,N,P} <: AbstractArray{T,N} - parent::P -end -OpaqueArray(A::P) where P<:AbstractArray{T,N} where {T,N} = OpaqueArray{T,N,P}(A) -OpaqueArray(st::P) where P<:AbstractDimStack{<:Any,T,N} where {T,N} = OpaqueArray{T,N,P}(st) - -Base.size(A::OpaqueArray) = size(A.parent) -Base.getindex(A::OpaqueArray, args...) = Base.getindex(A.parent, args...) -Base.setindex!(A::OpaqueArray, args...) = Base.setindex!(A.parent, args...) - abstract type AbstractBins <: Function end @@ -249,10 +272,10 @@ Group some data along the time dimension: ```jldoctest groupby; setup = :(using Random; Random.seed!(123)) julia> using DimensionalData, Dates - julia> A = rand(X(1:0.1:20), Y(1:20), Ti(DateTime(2000):Day(3):DateTime(2003))); julia> groups = groupby(A, Ti => month) # Group by month +metadata = Dict{Symbol, Any}(:groupby => (:Ti => Dates.month)) ┌ 12-element DimGroupByArray{DimArray{Float64,3},1} ┐ ├───────────────────────────────────────────────────┴───────────── dims ┐ ↓ Ti Sampled{Int64} [1, 2, …, 11, 12] ForwardOrdered Irregular Points @@ -277,15 +300,16 @@ julia> groupmeans = mean.(groups) # Take the monthly mean ┌ 12-element DimArray{Float64, 1} ┐ ├─────────────────────────────────┴─────────────────────────────── dims ┐ ↓ Ti Sampled{Int64} [1, 2, …, 11, 12] ForwardOrdered Irregular Points -├───────────────────────────────────────────────────────────── metadata ┤ - Dict{Symbol, Any} with 1 entry: - :groupby => :Ti=>month └───────────────────────────────────────────────────────────────────────┘ 1 0.500064 2 0.499762 3 0.500083 4 0.499985 - ⋮ + 5 0.500511 + 6 0.500042 + 7 0.500003 + 8 0.500257 + 9 0.500868 10 0.500874 11 0.498704 12 0.50047 @@ -297,24 +321,29 @@ match after application of `mean`. ```jldoctest groupby julia> map(.-, groupby(A, Ti=>month), mean.(groupby(A, Ti=>month), dims=Ti)); +metadata = Dict{Symbol, Any}(:groupby => (:Ti => Dates.month)) +metadata = Dict{Symbol, Any}(:groupby => (:Ti => Dates.month)) ``` +ups Or do something else with Y: ```jldoctest groupby julia> groupmeans = mean.(groupby(A, Ti=>month, Y=>isodd)) +metadata = Dict{Symbol, Any}(:groupby => (:Ti => Dates.month, :Y => isodd)) ┌ 12×2 DimArray{Float64, 2} ┐ ├───────────────────────────┴────────────────────────────────────── dims ┐ ↓ Ti Sampled{Int64} [1, 2, …, 11, 12] ForwardOrdered Irregular Points, → Y Sampled{Bool} [false, true] ForwardOrdered Irregular Points -├────────────────────────────────────────────────────────────── metadata ┤ - Dict{Symbol, Any} with 1 entry: - :groupby => (:Ti=>month, :Y=>isodd) └────────────────────────────────────────────────────────────────────────┘ ↓ → false true 1 0.499594 0.500533 2 0.498145 0.501379 + 3 0.499871 0.500296 + 4 0.500921 0.49905 ⋮ + 8 0.499599 0.500915 + 9 0.500715 0.501021 10 0.501105 0.500644 11 0.498606 0.498801 12 0.501643 0.499298 @@ -330,32 +359,48 @@ function DataAPI.groupby( end return groupby(A, dims) end -function DataAPI.groupby(A::DimArrayOrStack, dimfuncs::DimTuple) +function DataAPI.groupby(A::DimArrayOrStack, dimfuncs::DimTuple; name=:groupby) length(otherdims(dimfuncs, dims(A))) > 0 && Dimensions._extradimserror(otherdims(dimfuncs, dims(A))) # Get groups for each dimension - dim_groups_indices = map(dimfuncs) do d + group_dims = map(dimfuncs) do d _group_indices(dims(A, d), DD.val(d)) end - # Separate lookups dims from indices - group_dims = map(first, dim_groups_indices) - # Get indices for each group wrapped with dims for indexing - indices = map(rebuild, group_dims, map(last, dim_groups_indices)) - - # Hide that the parent is a DimSlices - views = OpaqueArray(DimSlices(A, indices)) # Put the groupby query in metadata - meta = map(d -> name(d) => val(d), dimfuncs) + meta = map(d -> DD.name(d) => val(d), dimfuncs) metadata = Dict{Symbol,Any}(:groupby => length(meta) == 1 ? only(meta) : meta) # Return a DimGroupByArray - return DimGroupByArray(views, format(group_dims, views), (), :groupby, metadata) + return DimGroupByArray(A, map(format, group_dims), (), name, metadata) +end + +# An array that holds another secret array that is indexed along with it. +# We use this to put groupings inside lookups so they are handled by `slicedims` +struct HiddenVector{T,A<:AbstractArray{T,1},H<:AbstractArray{<:Any,1}} <: AbstractArray{T,1} + data::A + hidden::H end +Base.getindex(A::HiddenVector, i::Int) = getindex(A.data, i) +Base.view(A::HiddenVector, i::Int) = + HiddenVector(view(A.data, i...), view(A.hidden, i)) +for f in (:view, :getindex) + @eval Base.$f(A::HiddenVector, i::Union{Colon,AbstractArray}) = + HiddenVector(Base.$f(A.data, i), Base.$f(A.hidden, i)) +end +Base.parent(A::HiddenVector) = A.hidden +Base.size(A::HiddenVector) = size(A.data) + +hiddenparent(A::HiddenVector) = A.hidden +hiddenparent(A::Dimension) = hiddenparent(parent(A)) +hiddenparent(A::AbstractArray) = hiddenparent(parent(A)) +hiddenparent(A::Array) = error("HiddenVector not found") + # Define the groups and find all the indices for values that fall in them function _group_indices(dim::Dimension, f::Base.Callable; labels=nothing) orig_lookup = lookup(dim) k1 = f(first(orig_lookup)) + # TODO: using a Dict here is a bit slow indices_dict = Dict{typeof(k1),Vector{Int}}() for (i, x) in enumerate(orig_lookup) k = f(x) @@ -363,9 +408,12 @@ function _group_indices(dim::Dimension, f::Base.Callable; labels=nothing) push!(inds, i) end ps = sort!(collect(pairs(indices_dict))) - group_dim = format(rebuild(dim, _maybe_label(labels, first.(ps)))) + lookupvals = _maybe_label(labels, first.(ps)) indices = last.(ps) - return group_dim, indices + # We combine lookup values and indices into on array + lookup_and_indices = HiddenVector(lookupvals, indices) + # And wrap and format them as a dimension + return format(rebuild(dim, lookup_and_indices)) end function _group_indices(dim::Dimension, group_lookup::Lookup; labels=nothing) orig_lookup = lookup(dim) @@ -374,13 +422,15 @@ function _group_indices(dim::Dimension, group_lookup::Lookup; labels=nothing) n = selectindices(group_lookup, Contains(v); err=Lookups._False()) isnothing(n) || push!(indices[n], i) end - group_dim = if isnothing(labels) - rebuild(dim, group_lookup) + lookupvals = if isnothing(labels) + group_lookup else - label_lookup = _maybe_label(labels, group_lookup) - rebuild(dim, label_lookup) + _maybe_label(labels, group_lookup) end - return group_dim, indices + # We combine lookup values and indices into on array + lookup_and_indices = HiddenVector(lookupvals, indices) + # And wrap and format them as a dimension + return format(rebuild(dim, lookup_and_indices)) end function _group_indices(dim::Dimension, bins::AbstractBins; labels=bins.labels) l = lookup(dim) @@ -447,7 +497,8 @@ end Generate a `Vector` of `UnitRange` with length `step(A)` """ -intervals(rng::AbstractRange) = IntervalSets.Interval{:closed,:open}.(rng, rng .+ step(rng)) +intervals(rng::AbstractRange) = + IntervalSets.Interval{:closed,:open}.(rng, rng .+ step(rng)) """ ranges(A::AbstractRange{<:Integer}) @@ -455,3 +506,47 @@ intervals(rng::AbstractRange) = IntervalSets.Interval{:closed,:open}.(rng, rng . Generate a `Vector` of `UnitRange` with length `step(A)` """ ranges(rng::AbstractRange{<:Integer}) = map(x -> x:x+step(rng)-1, rng) + +""" + combine(f::Function, gb::DimGroupByArray; dims=:) + +Combine the `DimGroupByArray` using function `f` over the group dimensions. +Unlike broadcasting a reducing function over a `DimGroupByArray`, this function +always returns a new flattened `AbstractDimArray` even where not all dimensions +are reduced. It will also work over grouped `AbstractDimStack`. + +If `dims` is given, it will combine only the dimensions in `dims`, the +others will be present in the final array. Note that all grouped dimensions +must be reduced and included in `dims`. + +The reducing function `f` must also accept a `dims` keyword. + +# Example + +```jldoctest groupby +```` +""" +function combine(f::Function, gb::DimGroupByArray{G}; dims=:) where G + targetdims = DD.commondims(first(gb), dims) + all(hasdim(first(gb), targetdims)) || throw(ArgumentError("dims must be a subset of the groupby dimensions")) + all(hasdim(targetdims, DD.dims(gb))) || throw(ArgumentError("grouped dimensions $(DD.basedims(gb)) must be included in dims")) + # This works for both arrays and stacks + # Combine the remaining dimensions after reduction and the group dimensions + destdims = (otherdims(DD.dims(first(gb)), dims)..., DD.dims(gb)...) + # Get the output eltype + T = Base.promote_op(f, G) + # Create a output array with the combined dimensions + dest = similar(first(gb), T, destdims) + for D in DimIndices(gb) + if all(hasdim(targetdims, DD.dims(first(gb)))) + # Assigned reduced scalar to dest + dest[D...] = f(gb[D]) + else + # Reduce with `f` and drop length 1 dimensions + xs = dropdims(f(gb[D]; dims); dims) + # Broadcast the reduced array to dest + broadcast_dims!(identity, view(dest, D...), xs) + end + end + return dest +end \ No newline at end of file diff --git a/src/opaque.jl b/src/opaque.jl new file mode 100644 index 000000000..87c234f30 --- /dev/null +++ b/src/opaque.jl @@ -0,0 +1,19 @@ +# OpaqueArray is an array that doesn't know what it holds, to simplify dispatch. +# One key property is that `parent(A::OpaqueArray)` returns the `OpaqueArray` `A` +# not the array it holds. +# +# It is often used here to hide dimensional arrays that may be generated lazily, +# To force them to act like simple Arrays without dimensional properties. +# +# OpaqueArray can also hold something that is not an AbstractArray itself. +struct OpaqueArray{T,N,P} <: AbstractArray{T,N} + parent::P +end +OpaqueArray(A::P) where P<:AbstractArray{T,N} where {T,N} = OpaqueArray{T,N,P}(A) +OpaqueArray(st::P) where P<:AbstractDimStack{<:Any,T,N} where {T,N} = OpaqueArray{T,N,P}(st) + +Base.size(A::OpaqueArray) = size(A.parent) +Base.getindex(A::OpaqueArray, I::Union{StandardIndices,Not{<:StandardIndices}}...) = + Base.getindex(A.parent, I...) +Base.setindex!(A::OpaqueArray, I::Union{StandardIndices,Not{<:StandardIndices}}...) = + Base.setindex!(A.parent, I...) \ No newline at end of file diff --git a/src/set.jl b/src/set.jl index 58df6acf7..0bcb647bc 100644 --- a/src/set.jl +++ b/src/set.jl @@ -2,39 +2,49 @@ const DimArrayOrStack = Union{AbstractDimArray,AbstractDimStack} """ set(x, val) - set(x, args::Pairs...) => x with updated field/s - set(x, args...; kw...) => x with updated field/s - set(x, args::Tuple{Vararg{Dimension}}; kw...) => x with updated field/s + set(x, dims::Pairs...) + set(x, dims::Tuple{Vararg{Dimension}}) + set(x; kw...) - set(dim::Dimension, index::AbstractArray) => Dimension - set(dim::Dimension, lookup::Lookup) => Dimension - set(dim::Dimension, lookupcomponent::LookupTrait) => Dimension - set(dim::Dimension, metadata::AbstractMetadata) => Dimension +Set the properties of an object, its internal data or the traits +of its dimensions and lookup index, returning a new, rebuild object. -Set the properties of an object, its internal data or the traits of its dimensions -and lookup index. +Related properties will be updated to match the change, for example, +changing the order of a lookup from ForwardOrdered to ReverseOrdered will reverse the +data as well. See [`unsafe_set`](@ref) for a version that makes the specified change. + +`x` can be a `AbstractDimArray`, `AbstractDimStack`, `Dimension`, `Lookup` or +`LookupTrait`. As DimensionalData is so strongly typed you do not need to specify what field -of a [`Lookup`](@ref) to `set` - there is no ambiguity. +of a [`Lookup`](@ref) to `set` - there is usually no ambiguity. + +## Updating object dimensions -To set fields of a `Lookup` you need to specify the dimension. This can be done -using `X => val` pairs, `X = val` keyword arguments, or `X(val)` wrapped arguments. +To set swap or alter the `Lookup` of an objects dimensions, you need to specify the dimension. +This can be done using `set(obj, X => val)` pairs or `set(obj,, X(val))` wrapped arguments. -You can also set the fields of all dimensions by simply passing a single [`Lookup`](@ref) -or lookup trait - it will be set for all dimensions. +You can also updata all dimensions by passing a lookup trait +e.g. `set(obj, ForwardOrdered)`. This will be set for all dimensions. When a `Dimension` or `Lookup` is passed to `set` to replace the -existing ones, fields that are not set will keep their original values. +existing ones, fields that were not set will keep their original values. + +## Updating object data -## Notes: +Passing an `AbstractArray` to `set` will update the data of the object. -Changing a lookup index range/vector will also update the step size and order where applicable. +## Updating object fields -Setting the [`Order`](@ref) like `ForwardOrdered` will *not* reverse the array or -dimension to match. Use `reverse` and [`reorder`](@ref) to do this. +Keywords can be passed to `set` to update the fields of an object, +working like keyword `rebuild` but updating related fields where needed. + +Fields are always the same as keywords for the objects constructor. ## Examples +Update the data in a `DimArray`: + ```jldoctest set julia> using DimensionalData; const DD = DimensionalData; @@ -67,13 +77,13 @@ julia> set(da, :Z => Ti, :custom => Z) 30.0 0.0 0.0 0.0 0.0 ``` -Change the lookup `Vector`: +Change the dimension lookup values: ```jldoctest set -julia> set(da, Z => [:a, :b, :c, :d], :custom => [4, 5, 6]) +julia> set(da, Z => [:a, :b, :c, :d], :custom => [6, 5, 4]) ┌ 3×4 DimArray{Float64, 2} ┐ ├──────────────────────────┴──────────────────────────────────────── dims ┐ - ↓ custom Sampled{Int64} [4, 5, 6] ForwardOrdered Regular Points, + ↓ custom Sampled{Int64} [6, 5, 4] ReverseOrdered Regular Points, → Z Sampled{Symbol} [:a, :b, :c, :d] ForwardOrdered Regular Points └─────────────────────────────────────────────────────────────────────────┘ ↓ → :a :b :c :d @@ -85,7 +95,7 @@ julia> set(da, Z => [:a, :b, :c, :d], :custom => [4, 5, 6]) Change the `Lookup` type: ```jldoctest set -julia> set(da, Z=DD.NoLookup(), custom=DD.Sampled()) +julia> set(da; Z => DD.NoLookup(), :custom => DD.Sampled()) ┌ 3×4 DimArray{Float64, 2} ┐ ├──────────────────────────┴──────────────────────────────────────── dims ┐ ↓ custom Sampled{Float64} 10.0:10.0:30.0 ForwardOrdered Regular Points, @@ -110,63 +120,146 @@ julia> set(da, :custom => DD.Irregular(10, 12), Z => DD.Regular(9.9)) 20.0 0.0 0.0 0.0 0.0 30.0 0.0 0.0 0.0 0.0 ``` + +Set the name of a `DimArray`: + +```jldoctest set +julia> set(da; name=:newname) +┌ 3×4 DimArray{Float64, 2} ┐ +├──────────────────────────┴────────────────────────────────────────── dims ┐ + ↓ custom Sampled{Float64} 10.0:10.0:30.0 ForwardOrdered Regular Points, + → Z Sampled{Float64} -20.0:10.0:10.0 ForwardOrdered Regular Points +└───────────────────────────────────────────────────────────────────────────┘ + ↓ → -20.0 -10.0 0.0 10.0 + 10.0 0.0 0.0 0.0 0.0 + 20.0 0.0 0.0 0.0 0.0 + 30.0 0.0 0.0 0.0 0.0 +``` +""" +set(x::DimArrayOrStack, args...; kw...) = + _set(Safe(), _set(Safe(), x, args...); kw...) +set(x::DimArrayOrStack, ::Type{T}) where T = set(x, T()) + """ -function set end + unsafe_set(x, val) + unsafe_set(x, dims::Pairs...) + unsafe_set(x, dims::Tuple{Vararg{Dimension}}) + unsafe_set(x; kw...) + +Set the properties of an object, its internal data or the traits +of its dimensions and lookup index, returning a new, rebuild object. + +Works the same as [`set`](@ref) but does not update other +properties to match any changes. + +It is usually type stable and can be faster than `set`, +but can produce broken objects if used incorrectly. + +`unsafe_set` with keywords is identical to `rebuild`. +""" +unsafe_set(x::DimArrayOrStack, args...; kw...) = + _set(Unsafe(), _set(Unsafe(), x, args...); kw...) +unsafe_set(x::DimArrayOrStack, ::Type{T}) where T = unsafe_set(x, T()) + +# Keywords are passed to rebuild, but with checks +function _set(s::Safety, A::AbstractDimArray; + data=nothing, dims=nothing, kw... +) + A1 = isnothing(data) ? A : _set_dimarray_data(s, A, data) + A2 = isnothing(dims) ? A1 : _set(s, A1, dims) + # Just `rebuild` everything else, it's assumed to have no interactions. + # Package developers note: if other fields do interact, implement this + # method for your own `AbstractDimArray` type. + return rebuild(A1; kw...) +end +function _set(s::Safety, st::AbstractDimStack; + data=nothing, dims=nothing, kw... +) + st1 = isnothing(data) ? st : _set_dimstack_data(s, st, data) + st2 = isnothing(dims) ? st1 : _set(s, st, dims) + # Just `rebuild` everything else, it's assumed to have no interactions. + # Package developers note: if other fields do interact, implement this + # method for your own `AbstractDimStack` type. + return rebuild(st2; kw...) +end -# Types are constructed -Base.@assume_effects :effect_free set(x::DimArrayOrStack, ::Type{T}) where T = - set(x, T()) # Dimensions and pairs are set for dimensions -Base.@assume_effects :effect_free function set( - A::AbstractDimArray, args::Union{Dimension,DimTuple,Pair}...; kw... +# Short circuit here to avoid multiple allocations +function _set( + s::Safety, A::AbstractDimArray, args::Union{Dimension,DimTuple,Pair}... ) - rebuild(A; dims=set(dims(A), args...; kw...)) + newdims = _set(s, dims(A), args...) + return rebuild(_rebuild_maybe_reorder(s, A, newdims); dims=newdims) end -Base.@assume_effects :effect_free function set( - st::AbstractDimStack, args::Union{Dimension,DimTuple,Pair}...; kw... +function _set( + s::Safety, st::AbstractDimStack, args::Union{Dimension,DimTuple,Pair}... ) - ds = set(dims(st), args...; kw...) - if dimsmatch(ds, dims(st)) - rebuild(st; dims=ds) + newdims = _set(s, dims(st), args...) + st = if dimsmatch(newdims, dims(st)) + _rebuild_maybe_reorder(s, st, newdims) else - dim_updates = map(rebuild, basedims(st), basedims(ds)) + dim_updates = map(rebuild, basedims(st), basedims(newdims)) lds = map(layerdims(st)) do lds # Swap out the dims with the updated dims # that match the dims of this layer map(val, dims(dim_updates, lds)) end - rebuild(st; dims=ds, layerdims=lds) + rebuild(_rebuild_maybe_reorder(s, st, newdims); layerdims=lds) end end # Single traits are set for all dimensions -Base.@assume_effects :effect_free set(A::DimArrayOrStack, x::LookupTrait) = - set(A, map(d -> basedims(d) => x, dims(A))...) -# Single lookups are set for all dimensions -# Need both for ambiguity -Base.@assume_effects :effect_free set(A::AbstractDimArray, x::Lookup) = - set(A, map(d -> basedims(d) => x, dims(A))...) -Base.@assume_effects :effect_free set(A::AbstractDimStack, x::Lookup) = - set(A, map(d -> basedims(d) => x, dims(A))...) +_set(s::Safety, A::DimArrayOrStack, x::LookupTrait) = + _set(s, A, map(d -> basedims(d) => x, dims(A))...) +# Single lookups are set for all dimensions. +_set(s::Safety, A::AbstractDimArray, x::Lookup) = + _set(s, A, map(d -> rebuild(d, x), dims(A))...) +_set(s::Safety, A::AbstractDimStack, x::Lookup) = + _set(s, A, map(d -> rebuild(d, x), dims(A))...) # Arrays are set as data for AbstractDimArray -Base.@assume_effects :effect_free function set( - A::AbstractDimArray, newdata::AbstractArray -) - axes(A) == axes(newdata) || _axiserr(A, newdata) - rebuild(A; data=newdata) +_set(s::Safety, A::AbstractDimArray, newdata::AbstractArray) = + _set_dimarray_data(s, A, newdata) +_set(s::Safety, A::AbstractDimStack, newdata::NamedTuple) = + _set_dimstack_data(s, A, newdata) + +# Check dimensions for Safe +function _set_dimarray_data(::Safe, A, data) + checkaxes(dims(A), axes(data)) + rebuild(A; data) end +# Just rebuild for Unsafe +_set_dimarray_data(::Unsafe, A, data) = rebuild(A; data) + # NamedTuples are set as data for AbstractDimStack -Base.@assume_effects :effect_free function set( - s::AbstractDimStack, newdata::NamedTuple -) - dat = data(s) - keys(dat) === keys(newdata) || _keyerr(keys(dat), keys(newdata)) - map(dat, newdata) do d, nd - axes(d) == axes(nd) || _axiserr(d, nd) +function _set_dimstack_data(::Safe, st, newdata) + # Allow updating subsets of data for NamedTuple + newdata1 = if parent(st) isa NamedTuple && newdata isa NamedTuple + ConstructionBase.setproperties(parent(st), newdata) + else + keys(st) === keys(newdata) || _keyerr(keys(dat), keys(newdata)) + newdata end - rebuild(s; data=newdata) + # Make sure the data matches the dimensions + map(layerdims(st), newdata1) do lds, nd + # TODO a message with the layer name could help here + checkaxes(dims(st, lds), axes(nd)) + end + return rebuild(st; data=newdata1) end -# Other things error -Base.@assume_effects :effect_free set(A, x) = Lookups._cantseterror(A, x) +# Just rebuild for Unsafe +_set_dimstack_data(::Unsafe, st, data) = rebuild(st; data) -@noinline _axiserr(a, b) = throw(ArgumentError("passed in axes $(axes(b)) do not match the currect axes $(axes(a))")) +@noinline _axiserr(a, b) = _axiserr(axes(a), axes(b)) +@noinline _axiserr(a::Tuple, b::Tuple) = + throw(ArgumentError("passed in axes $b do not match the currect axes $a")) @noinline _keyerr(ka, kb) = throw(ArgumentError("keys $ka and $kb do not match")) + +_rebuild_maybe_reorder(::Unsafe, A, newdims) = A + # Handle any changes to order +function _rebuild_maybe_reorder(::Safe, A, newdims) + if map(order, dims(A)) == map(order, newdims) + rebuild(A; dims=newdims) + else + A1 = reorder(A, map(rebuild, dims(A), order(newdims))) + rebuild(A; data=parent(A1), dims=newdims) + end +end \ No newline at end of file diff --git a/src/stack/indexing.jl b/src/stack/indexing.jl index dad1ebaaf..8843c99ad 100644 --- a/src/stack/indexing.jl +++ b/src/stack/indexing.jl @@ -150,6 +150,9 @@ for f in (:getindex, :view, :dotview) end end +@generated function _any_dimarray(v::Union{NamedTuple,Tuple}) + any(T -> T <: AbstractDimArray, v.types) +end #### setindex #### @propagate_inbounds Base.setindex!(s::AbstractDimStack, xs, I...; kw...) = @@ -160,22 +163,17 @@ end hassamedims(s) ? _map_setindex!(s, xs, i; kw...) : _setindex_mixed!(s, xs, i; kw...) @propagate_inbounds Base.setindex!(s::AbstractDimStack, xs::NamedTuple, i::AbstractArray; kw...) = hassamedims(s) ? _map_setindex!(s, xs, i; kw...) : _setindex_mixed!(s, xs, i; kw...) +@propagate_inbounds Base.setindex!(s::AbstractDimStack, xs::NamedTuple, i::DimensionIndsArrays; kw...) = + _map_setindex!(s, xs, i; kw...) +@propagate_inbounds Base.setindex!(s::AbstractDimStack, xs::NamedTuple, I...; kw...) = + _map_setindex!(s, xs, I...; kw...) -@propagate_inbounds function Base.setindex!( - s::AbstractDimStack, xs::NamedTuple, I...; kw... -) - map((A, x) -> setindex!(A, x, I...; kw...), layers(s), xs) -end - -_map_setindex!(s, xs, i; kw...) = map((A, x) -> setindex!(A, x, i...; kw...), layers(s), xs) +_map_setindex!(s, xs, i...; kw...) = map((A, x) -> setindex!(A, x, i...; kw...), layers(s), xs) -_setindex_mixed!(s::AbstractDimStack, x, i::AbstractArray) = - map(A -> setindex!(A, x, DimIndices(dims(s))[i]), layers(s)) -_setindex_mixed!(s::AbstractDimStack, i::Integer) = - map(A -> setindex!(A, x, DimIndices(dims(s))[i]), layers(s)) -function _setindex_mixed!(s::AbstractDimStack, x, i::Colon) - map(DimIndices(dims(s))) do D - map(A -> setindex!(A, D), x, layers(s)) +function _setindex_mixed!(s::AbstractDimStack, xs::NamedTuple, i) + D = DimIndices(dims(s))[i] + map(layers(s), xs) do A, x + A[D] = x end end diff --git a/src/stack/methods.jl b/src/stack/methods.jl index 7e684ba84..5c9659972 100644 --- a/src/stack/methods.jl +++ b/src/stack/methods.jl @@ -68,8 +68,8 @@ julia> size(slices) (4, 2) julia> map(dims, axes(slices)) -(↓ Z Base.OneTo(4), -→ X Base.OneTo(2)) +(↓ Z, +→ X Categorical{Symbol} [:x1, :x2] ForwardOrdered) julia> first(slices) ┌ 3×5 DimStack ┐ diff --git a/src/stack/stack.jl b/src/stack/stack.jl index 92bb07963..0e212f1c8 100644 --- a/src/stack/stack.jl +++ b/src/stack/stack.jl @@ -153,7 +153,6 @@ Base.length(s::AbstractDimStack) = prod(size(s)) Base.axes(s::AbstractDimStack) = map(first ∘ axes, dims(s)) Base.axes(s::AbstractDimStack, dims::DimOrDimType) = axes(s, dimnum(s, dims)) Base.axes(s::AbstractDimStack, dims::Integer) = axes(s)[dims] -Base.similar(s::AbstractDimStack, args...) = maplayers(A -> similar(A, args...), s) Base.eltype(::AbstractDimStack{<:Any,T}) where T = T Base.ndims(::AbstractDimStack{<:Any,<:Any,N}) where N = N Base.CartesianIndices(s::AbstractDimStack) = CartesianIndices(dims(s)) @@ -197,6 +196,36 @@ Base.get(f::Base.Callable, st::AbstractDimStack, k::Symbol) = @propagate_inbounds Base.iterate(st::AbstractDimStack, i) = i > length(st) ? nothing : (st[DimIndices(st)[i]], i + 1) +Base.similar(s::AbstractDimStack) = similar(s, eltype(s)) +Base.similar(s::AbstractDimStack, dims::Dimension...) = similar(s, dims) +Base.similar(s::AbstractDimStack, ::Type{T},dims::Dimension...) where T = + similar(s, T, dims) +Base.similar(s::AbstractDimStack, dims::Tuple{Vararg{Dimension}}) = + similar(s, eltype(s), dims) +Base.similar(s::AbstractDimStack, ::Type{T}) where T = + similar(s, T, dims(s)) +function Base.similar(s::AbstractDimStack, ::Type{T}, dims::Tuple) where T + # Any dims not in the stack are added to all layers + ods = otherdims(dims, DD.dims(s)) + maplayers(s) do A + # Original layer dims are maintained, other dims are added + D = DD.commondims(dims, (DD.dims(A)..., ods...)) + similar(A, T, D) + end +end +function Base.similar(s::AbstractDimStack, ::Type{T}, dims::Tuple) where T<:NamedTuple + ods = otherdims(dims, DD.dims(s)) + maplayers(s, _nt_types(T)) do A, Tx + D = DD.commondims(dims, (DD.dims(A)..., ods...)) + similar(A, Tx, D) + end +end + +@generated function _nt_types(::Type{NamedTuple{K,T}}) where {K,T} + expr = Expr(:tuple, T.parameters...) + return :(NamedTuple{K}($expr)) +end + # `merge` for AbstractDimStack and NamedTuple. # One of the first three arguments must be an AbstractDimStack for dispatch to work. Base.merge(s::AbstractDimStack) = s diff --git a/src/tables.jl b/src/tables.jl index 9773a8495..ef0c560d2 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -1,3 +1,22 @@ +# This lets use switch array of NamedTupleto NamedTuple of Array +struct LayerArray{K,T,N,A} <: AbstractArray{T,N} + data::A +end +function LayerArray{K}(a::A) where {A<:AbstractArray{<:NamedTuple,N}} where {K,N} + T = typeof(a[1][K]) + LayerArray{K,T,N,A}(a) +end +Base.parent(A::LayerArray) = parent(A.data) +Base.size(A::LayerArray) = size(parent(A)) +@propagate_inbounds Base.getindex(A::LayerArray{K}, I::Integer...) where K = + getproperty(getindex(parent(A), I...), K) + +function layerarrays(A::AbstractDimArray{<:NamedTuple{K}}) where K + map(K) do k + rebuild(A; data=LayerArray{k}(A)) + end |> NamedTuple{K} +end + """ AbstractDimTable <: Tables.AbstractColumns @@ -5,6 +24,9 @@ Abstract supertype for dim tables """ abstract type AbstractDimTable <: Tables.AbstractColumns end +struct Columns end +struct Rows end + # Tables.jl interface for AbstractDimStack and AbstractDimArray DimTableSources = Union{AbstractDimStack,AbstractDimArray} @@ -12,12 +34,8 @@ DimTableSources = Union{AbstractDimStack,AbstractDimArray} Tables.istable(::Type{<:DimTableSources}) = true Tables.columnaccess(::Type{<:DimTableSources}) = true Tables.columns(x::DimTableSources) = DimTable(x) - -Tables.columnnames(A::AbstractDimArray) = _colnames(DimStack(A)) -Tables.columnnames(s::AbstractDimStack) = _colnames(s) - -Tables.schema(A::AbstractDimArray) = Tables.schema(DimStack(A)) -Tables.schema(s::AbstractDimStack) = Tables.schema(DimTable(s)) +Tables.columnnames(x::DimTableSources) = _colnames(x) +Tables.schema(x::DimTableSources) = Tables.schema(DimTable(x)) @inline Tables.getcolumn(x::DimTableSources, i::Int) = Tables.getcolumn(DimTable(x), i) @inline Tables.getcolumn(x::DimTableSources, key::Symbol) = @@ -27,11 +45,14 @@ Tables.schema(s::AbstractDimStack) = Tables.schema(DimTable(s)) @inline Tables.getcolumn(t::DimTableSources, dim::DimOrDimType) = Tables.getcolumn(t, dimnum(t, dim)) -function _colnames(s::AbstractDimStack) - dimkeys = map(name, dims(s)) - # The data is always the last column/s - (dimkeys..., keys(s)...) +_colnames(s::AbstractDimStack) = (map(name, dims(s))..., keys(s)...) +function _colnames(A::AbstractDimArray) + n = Symbol(name(A)) == Symbol("") ? :value : Symbol(name(A)) + (map(name, dims(A))..., n) end +_colnames(A::AbstractDimArray{T}) where T<:NamedTuple = + (map(name, dims(A))..., _colnames(T)...) +_colnames(::Type{<:NamedTuple{Keys}}) where Keys = Keys # DimTable @@ -57,49 +78,67 @@ To get dimension columns, you can index with `Dimension` (`X()`) or # Keywords - `mergedims`: Combine two or more dimensions into a new dimension. -- `layersfrom`: Treat a dimension of an `AbstractDimArray` as layers of an `AbstractDimStack`. +- `preservedims`: Preserve one or more dimensions from flattening into the table. + `DimArray`s of views with these dimensions will be present in the layer column, + rather than scalar values. +- `layersfrom`: Treat a dimension of an `AbstractDimArray` as layers of an `AbstractDimStack` + by specifying a dimension to use as layers. # Example -```jldoctest +Here we generate a GeoInterface.jl compatible table with `:geometry` +column made of `(X, Y)` points, and data columns from `:band` slices. + +```julia julia> using DimensionalData, Tables -julia> a = DimArray(ones(16, 16, 3), (X, Y, Dim{:band})) -┌ 16×16×3 DimArray{Float64, 3} ┐ -├──────────────────────── dims ┤ - ↓ X, → Y, ↗ band -└──────────────────────────────┘ -[:, :, 1] - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 … 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 … 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 … 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 … 1.0 1.0 1.0 1.0 1.0 1.0 1.0 - -julia> +julia> A = ones(X(4), Y(3), Dim{:band}('a':'d'); name=:data); +julia> DimTable(A; layersfrom=:band, mergedims=(X, Y)=>:geometry) +DimTable with 12 rows, 5 columns, and schema: + :geometry Tuple{Int64, Int64} + :band_a Float64 + :band_b Float64 + :band_c Float64 + :band_d Float64 ``` + +And here bands for each X/Y position are kept as vectors, using `preservedims`. +This may be useful if e.g. bands are color components of spectral images. + +```julia +julia> DimTable(A; preservedims=:band) +DimTable with 12 rows, 3 columns, and schema: + :X … Int64 + :Y Int64 + :data DimVector{Float64, Tuple{Dim{:band, Categorical{Char, StepRange{Char, Int64}, ForwardOrdered, NoMetadata}}}, Tuple{X{NoLookup{UnitRange{Int64}}}, Y{NoLookup{UnitRange{Int64}}}}, SubArray{Float64, 1, Array{Float64, 3}, Tuple{Int64, Int64, Slice{OneTo{Int64}}}, true}, Symbol, NoMetadata} (alias for DimArray{Float64, 1, Tuple{Dim{:band, DimensionalData.Dimensions.Lookups.Categorical{Char, StepRange{Char, Int64}, DimensionalData.Dimensions.Lookups.ForwardOrdered, DimensionalData.Dimensions.Lookups.NoMetadata}}}, Tuple{X{DimensionalData.Dimensions.Lookups.NoLookup{UnitRange{Int64}}}, Y{DimensionalData.Dimensions.Lookups.NoLookup{UnitRange{Int64}}}}, SubArray{Float64, 1, Array{Float64, 3}, Tuple{Int64, Int64, Base.Slice{Base.OneTo{Int64}}}, true}, Symbol, DimensionalData.Dimensions.Lookups.NoMetadata}) + +```julia +julia> DimTable(A) +DimTable with 48 rows, 4 columns, and schema: + :X Int64 + :Y Int64 + :band Char + :data Float64 """ -struct DimTable <: AbstractDimTable +struct DimTable{Mode} <: AbstractDimTable parent::Union{AbstractDimArray,AbstractDimStack} colnames::Vector{Symbol} dimcolumns::Vector{AbstractVector} - dimarraycolumns::Vector{AbstractVector} + dimarraycolumns::Vector end - -function DimTable(s::AbstractDimStack; mergedims=nothing) +function DimTable(s::AbstractDimStack; + mergedims=nothing, + preservedims=nothing, +) s = isnothing(mergedims) ? s : DD.mergedims(s, mergedims) + s = if isnothing(preservedims) + s + else + maplayers(s) do A + _maybe_presevedims(A, preservedims) + end + end dimcolumns = collect(_dimcolumns(s)) dimarraycolumns = if hassamedims(s) map(vec, layers(s)) @@ -107,43 +146,79 @@ function DimTable(s::AbstractDimStack; mergedims=nothing) map(A -> vec(DimExtensionArray(A, dims(s))), layers(s)) end |> collect keys = collect(_colnames(s)) - return DimTable(s, keys, dimcolumns, dimarraycolumns) + return DimTable{Columns}(s, keys, dimcolumns, dimarraycolumns) end -function DimTable(xs::Vararg{AbstractDimArray}; layernames=nothing, mergedims=nothing) +function DimTable(As::AbstractVector{<:AbstractDimArray}; + layernames=nothing, + mergedims=nothing, + preservedims=nothing, +) # Check that dims are compatible - comparedims(xs...) - + comparedims(As) # Construct Layer Names - layernames = isnothing(layernames) ? [Symbol("layer_$i") for i in eachindex(xs)] : layernames - + layernames = isnothing(layernames) ? uniquekeys(As) : layernames # Construct dimension and array columns with DimExtensionArray - xs = isnothing(mergedims) ? xs : map(x -> DimensionalData.mergedims(x, mergedims), xs) - dims_ = dims(first(xs)) + As = isnothing(mergedims) ? As : map(x -> DimensionalData.mergedims(x, mergedims), As) + As = if isnothing(preservedims) + As + else + map(As) do A + _maybe_presevedims(A, preservedims) + end + end + dims_ = dims(first(As)) dimcolumns = collect(_dimcolumns(dims_)) dimnames = collect(map(name, dims_)) - dimarraycolumns = collect(map(vec ∘ parent, xs)) + dimarraycolumns = collect(map(vec ∘ parent, As)) colnames = vcat(dimnames, layernames) # Return DimTable - return DimTable(first(xs), colnames, dimcolumns, dimarraycolumns) + return DimTable{Columns}(first(As), colnames, dimcolumns, dimarraycolumns) end -function DimTable(x::AbstractDimArray; layersfrom=nothing, mergedims=nothing) - if !isnothing(layersfrom) && any(hasdim(x, layersfrom)) - d = dims(x, layersfrom) - nlayers = size(x, d) - layers = [view(x, rebuild(d, i)) for i in 1:nlayers] +function DimTable(A::AbstractDimArray; + layersfrom=nothing, + mergedims=nothing, + preservedims=nothing, +) + if !isnothing(layersfrom) && any(hasdim(A, layersfrom)) + d = dims(A, layersfrom) + nlayers = size(A, d) + layers = [view(A, rebuild(d, i)) for i in 1:nlayers] layernames = if iscategorical(d) Symbol.((name(d),), '_', lookup(d)) else Symbol.(("$(name(d))_$i" for i in 1:nlayers)) end - return DimTable(layers..., layernames=layernames, mergedims=mergedims) + return DimTable(layers; layernames, mergedims, preservedims) else - s = name(x) == NoName() ? DimStack((;value=x)) : DimStack(x) - return DimTable(s, mergedims=mergedims) + A1 = isnothing(mergedims) ? A : DD.mergedims(A, mergedims) + if eltype(A1) <: NamedTuple + if isnothing(preservedims) + dimcolumns = collect(_dimcolumns(A1)) + colnames = collect(_colnames(A1)) + dimarrayrows = vec(parent(A1)) + return DimTable{Rows}(A1, colnames, dimcolumns, dimarrayrows) + else + las = layerarrays(A1) + layernames = collect(keys(las)) + return DimTable(collect(las); layernames, mergedims, preservedims) + end + else + A2 = _maybe_presevedims(A1, preservedims) + dimcolumns = collect(_dimcolumns(A2)) + colnames = collect(_colnames(A2)) + dimarraycolumns = [vec(parent(A2))] + return DimTable{Columns}(A2, colnames, dimcolumns, dimarraycolumns) + end end end +_maybe_presevedims(A, preservedims::Nothing) = A +function _maybe_presevedims(A, preservedims) + S = DimSlices(A; dims=otherdims(A, preservedims)) + rebuild(A; data=OpaqueArray(S), dims=dims(S)) +end + _dimcolumns(x) = map(d -> _dimcolumn(x, d), dims(x)) function _dimcolumn(x, d::Dimension) lookupvals = parent(lookup(d)) @@ -155,8 +230,6 @@ function _dimcolumn(x, d::Dimension) end end - - dimcolumns(t::DimTable) = getfield(t, :dimcolumns) dimarraycolumns(t::DimTable) = getfield(t, :dimarraycolumns) colnames(t::DimTable) = Tuple(getfield(t, :colnames)) @@ -174,12 +247,26 @@ Tables.columnaccess(::Type{<:DimTable}) = true Tables.columns(t::DimTable) = t Tables.columnnames(c::DimTable) = colnames(c) -function Tables.schema(t::DimTable) - types = vcat([map(eltype, dimcolumns(t))...], [map(eltype, dimarraycolumns(t))...]) +function Tables.schema(t::DimTable) + types = vcat([map(eltype, dimcolumns(t))...], _dimarraycolumn_eltypes(t)) Tables.Schema(colnames(t), types) end -@inline function Tables.getcolumn(t::DimTable, i::Int) +_dimarraycolumn_eltypes(t::DimTable{Columns}) = [map(eltype, dimarraycolumns(t))...] +_dimarraycolumn_eltypes(t::DimTable{Rows}) = _eltypes(eltype(dimarraycolumns(t))) +_eltypes(::Type{T}) where T<:NamedTuple = collect(T.types) + +@inline function Tables.getcolumn(t::DimTable{Rows}, i::Int) + nkeys = length(colnames(t)) + if i > length(dims(t)) + map(nt -> nt[i], dimarraycolumns(t)) + elseif i > 0 && i < nkeys + dimcolumns(t)[i] + else + throw(ArgumentError("There is no table column $i")) + end +end +@inline function Tables.getcolumn(t::DimTable{Columns}, i::Int) nkeys = length(colnames(t)) if i > length(dims(t)) dimarraycolumns(t)[i - length(dims(t))] @@ -189,12 +276,19 @@ end throw(ArgumentError("There is no table column $i")) end end - -@inline function Tables.getcolumn(t::DimTable, dim::DimOrDimType) +@inline function Tables.getcolumn(t::DimTable, dim::Union{Dimension,Type{<:Dimension}}) dimcolumns(t)[dimnum(t, dim)] end - -@inline function Tables.getcolumn(t::DimTable, key::Symbol) +@inline function Tables.getcolumn(t::DimTable{Rows}, key::Symbol) + key in colnames(t) || throw(ArgumentError("There is no table column $key")) + if hasdim(parent(t), key) + dimcolumns(t)[dimnum(t, key)] + else + # Function barrier + _col_from_rows(dimarraycolumns(t), key) + end +end +@inline function Tables.getcolumn(t::DimTable{Columns}, key::Symbol) keys = colnames(t) i = findfirst(==(key), keys) if isnothing(i) @@ -203,22 +297,20 @@ end return Tables.getcolumn(t, i) end end - @inline function Tables.getcolumn(t::DimTable, ::Type{T}, i::Int, key::Symbol) where T Tables.getcolumn(t, key) end -# TableTraits.jl interface - +_col_from_rows(rows, key) = map(row -> row[key], rows) -function IteratorInterfaceExtensions.getiterator(x::DimTableSources) - return Tables.datavaluerows(Tables.dictcolumntable(x)) -end -IteratorInterfaceExtensions.isiterable(::DimTableSources) = true +# TableTraits.jl interface TableTraits.isiterabletable(::DimTableSources) = true +TableTraits.isiterabletable(::DimTable) = true -function IteratorInterfaceExtensions.getiterator(t::DimTable) - return Tables.datavaluerows(Tables.dictcolumntable(t)) -end +# IteratorInterfaceExtensions.jl interface +IteratorInterfaceExtensions.getiterator(x::DimTableSources) = + Tables.datavaluerows(Tables.dictcolumntable(x)) +IteratorInterfaceExtensions.getiterator(t::DimTable) = + Tables.datavaluerows(Tables.dictcolumntable(t)) +IteratorInterfaceExtensions.isiterable(::DimTableSources) = true IteratorInterfaceExtensions.isiterable(::DimTable) = true -TableTraits.isiterabletable(::DimTable) = true diff --git a/src/utils.jl b/src/utils.jl index 8c9f5cb43..e038b57bc 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,3 +1,5 @@ +const Reorderable = Union{AbstractBasicDimArray,AbstractDimStack,DimTuple} +const DimensionOrLookup = Union{Dimension,Lookup} """ reorder(A::Union{AbstractDimArray,AbstractDimStack}, order::Pair...) @@ -33,33 +35,76 @@ true """ function reorder end -reorder(x, A::Union{AbstractDimArray,AbstractDimStack,AbstractDimIndices}) = reorder(x, dims(A)) -reorder(x, ::Nothing) = throw(ArgumentError("object has no dimensions")) -reorder(x, p::Pair, ps::Vararg{Pair}) = reorder(x, (p, ps...)) -reorder(x, ps::Tuple{Vararg{Pair}}) = reorder(x, Dimensions.pairs2dims(ps...)) -# Reorder specific dims. -reorder(x, dimwrappers::Tuple) = _reorder(x, dimwrappers) -# Reorder all dims. -reorder(x, ot::Order) = reorder(x, typeof(ot)) -reorder(x, ot::Type{<:Order}) = _reorder(x, map(d -> rebuild(d, ot), dims(x))) -reorder(dim::Dimension, ot::Type{<:Order}) = - ot <: basetypeof(order(dim)) ? dim : reverse(dim) - +reorder(x::Reorderable, A::Union{AbstractDimArray,AbstractDimStack,AbstractDimIndices}) = + reorder(x, dims(A)) +reorder(x::Reorderable, ::Nothing) = throw(ArgumentError("object has no dimensions")) +reorder(x::Reorderable, p::Pair, ps::Pair...) = reorder(x, (p, ps...)) +reorder(x::Reorderable, ps::Tuple{Vararg{Pair}}) = reorder(x, Dimensions.pairs2dims(ps...)) +reorder(x::Reorderable, ::Type{O}) where O<:Order = reorder(x, O()) +function reorder(x::Reorderable, o::Order) + ds = dims(x) + isnothing(ds) && _dimsnotdefinederror() + reorder(x, map(d -> rebuild(d, o), ds)) +end +reorder(x::Reorderable, o::Tuple{Vararg{Order}}) = reorder(x, map(rebuild, dims(x), o)) # Recursive reordering. x may be reversed here -function _reorder(x, orderdims::DimTuple) +function reorder(x::Reorderable, orderdims::DimTuple) ods = commondims(orderdims, dims(x)) - _reorder(reorder(x, ods[1]), tail(ods)) + reorder(reorder(x, ods[1]), tail(ods)) +end +reorder(x::Reorderable, orderdims::Tuple{}) = x +function reorder(ds::DimTuple, orderdims::DimTuple) + ods = commondims(orderdims, ds) + map(ds) do d + hasdim(ods, d) ? reorder(d, val(dims(ods, d))) : d + end +end +reorder(ds::DimTuple, orderdims::Tuple{}) = ds + +reorder(x::Reorderable, orderdim::Dimension{<:Order}) = _reorder(x, dims(x, orderdim), val(orderdim)) +reorder(x::Reorderable, orderdim::Dimension{<:Lookup}) = _reorder(x, dims(x, orderdim), order(orderdim)) + +# Unordered: do nothing, just set the order to Unordered +_reorder(x::Reorderable, dim::Dimension, o::Unordered) = unsafe_set(x, dim => o) +# Ordered: leave, reverse or sort +_reorder(x::Reorderable, dim::Dimension, o::Ordered) = _reorder(x, dim, order(dim), o) +# Order matches, nothing to do +_reorder(x::Reorderable, dim::Dimension, ::O, ::O) where O<:Ordered = x +# dimensional reverse can handle this +_reorder(x::Reorderable, dim::Dimension, ::Ordered, ::Ordered) = + reverse(x; dims=basedims(dim)) +function _reorder(x::AbstractDimIndices, dim::Dimension, o1::Unordered, o2::Ordered) + newdim = _reorder(dims(x, dim), o1, o2) + return unsafe_set(x, newdim) +end +# We need to sort the data along this dimension +function _reorder( + x::Union{AbstractDimArray,AbstractDimStack}, dim::Dimension, ::Unordered, o::Ordered +) + l = lookup(dim) + # Sort forwards or reverse + idxs = o isa ForwardOrdered ? sortperm(l) : sortperm(l; rev=true) + # Reorder the values by indexing into dimension of dim + output = x[rebuild(dim, idxs)] + # Set the order + return unsafe_set(output, dim => o) end -_reorder(x, orderdims::Tuple{}) = x - -reorder(x, orderdim::Dimension) = _reorder(val(orderdim), x, dims(x, orderdim)) -reorder(x, orderdim::Dimension{<:Lookup}) = _reorder(order(orderdim), x, dims(x, orderdim)) -_reorder(neworder::Order, x, dim::Dimension) = _reorder(basetypeof(neworder), x, dim) -# Reverse the dimension index -_reorder(::Type{O}, x, dim::Dimension) where O<:Ordered = - order(dim) isa O ? x : reverse(x; dims=dim) -_reorder(ot::Type{Unordered}, x, dim::Dimension) = x +reorder(x::Dimension, o::Order) = rebuild(x, _reorder(lookup(x), o)) +reorder(x::Dimension, l::Lookup) = _reorder(x, order(l)) +reorder(x::Dimension, d::Dimension) = _reorder(x, order(d)) + +# Unordered: do nothing, just set the order to Unordered +_reorder(x::DimensionOrLookup, o::Unordered) = unsafe_set(x, o) +# Ordered: leave, reverse or sort +_reorder(x::DimensionOrLookup, o::Ordered) = _reorder(x, order(x), o) +# Order matches, nothing to do +_reorder(x::DimensionOrLookup, ::O, ::O) where O<:Ordered = x +# reverse can handle this +_reorder(x::DimensionOrLookup, ::Ordered, ::Ordered) = reverse(x) +# We need to sort +_reorder(dim::Dimension, ::Unordered, o::Ordered) = + rebuild(dim, reorder(lookup(dim), o)) """ modify(f, A::AbstractDimArray) => AbstractDimArray @@ -160,6 +205,11 @@ function broadcast_dims!(f, dest::AbstractDimArray{<:Any,N}, As::AbstractBasicDi od = map(A -> otherdims(dest, dims(A)), As) return _broadcast_dims_inner!(f, dest, As, od) end +function broadcast_dims!(f, dest::AbstractDimStack, stacks::AbstractDimStack...) + maplayers(dest, stacks...) do d, layers... + broadcast_dims!(f, d, layers...) + end +end # Function barrier function _broadcast_dims_inner!(f, dest, As, od) @@ -187,15 +237,18 @@ end function uniquekeys(das::Vector{<:AbstractDimArray}) length(das) == 0 ? Symbol[] : uniquekeys(map(Symbol ∘ name, das)) end -function uniquekeys(keys::Vector{Symbol}) - map(enumerate(keys)) do (id, k) +function uniquekeys(keys::Vector{Symbol}) map(enumerate(keys)) do (id, k) count(k1 -> k == k1, keys) > 1 ? Symbol(:layer, id) : k end end function uniquekeys(keys::Tuple{Symbol,Vararg{Symbol}}) - ids = ntuple(x -> x, length(keys)) + ids = ntuple(identity, length(keys)) map(keys, ids) do k, id - count(k1 -> k == k1, keys) > 1 ? Symbol(:layer, id) : k + if k == Symbol("") + Symbol(:layer, id) + else + count(k1 -> k == k1, keys) > 1 ? Symbol(:layer, id) : k + end end end uniquekeys(t::Tuple) = ntuple(i -> Symbol(:layer, i), length(t)) diff --git a/test/dimindices.jl b/test/dimindices.jl index ed7a59bc3..7a7311feb 100644 --- a/test/dimindices.jl +++ b/test/dimindices.jl @@ -38,7 +38,7 @@ A = zeros(X(4.0:7.0), Y(10.0:12.0)) @test @inferred size(A1[di[2:4, 1:2], Ti=1]) == (3, 2) @test @inferred A1[di] isa DimArray{Float64,3} @test @inferred A1[X=1][di] isa DimArray{Float64,2} - @test @inferred A1[X=1, Y=1][di] isa DimArray{Float64,1} + @test @inferred A1[X=1, Y=1] isa DimArray{Float64,1} # Indexing with no matching dims still returns a DimArray @test @inferred view(A1, X=1, Y=1, Ti=1)[di] == fill(0.0) @@ -207,11 +207,16 @@ end @testset "DimSlices" begin A = DimArray(((1:4) * (1:3)'), (X(4.0:7.0), Y(10.0:12.0)); name=:foo) - axisdims = map(dims(A, (X,))) do d - rebuild(d, axes(lookup(d), 1)) - end - ds = DimensionalData.DimSlices(A; dims=axisdims) + ds = DimensionalData.DimSlices(A; dims=X) + @test ds == ds[X=:] + # Works just like Slices + @test sum(ds) == sum(eachslice(A; dims=X)) @test ds == ds[X=:] + @test ds[X=At(7.0)] == [4, 8, 12] # Works just like Slices @test sum(ds) == sum(eachslice(A; dims=X)) + @test axes(ds) == axes(eachslice(A; dims=X)) + ds0 = DimensionalData.DimSlices(A; dims=()); + @test sum(ds0) == sum(eachslice(parent(A); dims=())) + @test axes(ds0) == axes(eachslice(parent(A); dims=())) end diff --git a/test/groupby.jl b/test/groupby.jl index 3f6e1f7ce..4e6c53f8c 100644 --- a/test/groupby.jl +++ b/test/groupby.jl @@ -8,7 +8,6 @@ days = DateTime(2000):Day(1):DateTime(2000, 12, 31) A = DimArray((1:6) * (1:366)', (X(1:0.2:2), Ti(days))) st = DimStack((a=A, b=A, c=A[X=1])) - @testset "group eltype matches indexed values" begin da = rand(X(1:10), Y(1:10)) grps = groupby(da, X => isodd) @@ -22,10 +21,16 @@ end mean(A[Ti=dayofyear(m):dayofyear(m)+daysinmonth(m)-1]) end @test mean.(groupby(A, Ti=>month)) == manualmeans + combinedmeans = combine(mean, groupby(A, Ti=>month)) + @test combinedmeans isa DimArray + @test combinedmeans == manualmeans manualmeans_st = map(months) do m mean(st[Ti=dayofyear(m):dayofyear(m)+daysinmonth(m)-1]) end @test mean.(groupby(st, Ti=>month)) == manualmeans_st + combinedmeans_st = combine(mean, groupby(st, Ti=>month)) + @test combinedmeans_st isa DimStack{(:a, :b, :c), @NamedTuple{a::Float64, b::Float64, c::Float64}} + @test collect(combinedmeans_st) == manualmeans_st manualsums = mapreduce(hcat, months) do m vcat(sum(A[Ti=dayofyear(m):dayofyear(m)+daysinmonth(m)-1, X=1 .. 1.5]), @@ -34,8 +39,11 @@ end end |> permutedims gb_sum = sum.(groupby(A, Ti=>month, X => >(1.5))) @test dims(gb_sum, Ti) == Ti(Sampled([1:12...], ForwardOrdered(), Irregular((nothing, nothing)), Points(), NoMetadata())) - @test typeof(dims(gb_sum, X)) == typeof(X(Sampled(BitVector([false, true]), ForwardOrdered(), Irregular((nothing, nothing)), Points(), NoMetadata()))) + @test typeof(dims(gb_sum, X)) == + X{Sampled{Bool, DimensionalData.HiddenVector{Bool, BitVector, Vector{Vector{Int64}}}, ForwardOrdered, Irregular{Tuple{Nothing, Nothing}}, Points, NoMetadata}} @test gb_sum == manualsums + combined_sum = combine(sum, groupby(A, Ti=>month, X => >(1.5))) + @test collect(combined_sum) == manualsums manualsums_st = mapreduce(hcat, months) do m vcat(sum(st[Ti=dayofyear(m):dayofyear(m)+daysinmonth(m)-1, X=1 .. 1.5]), @@ -44,12 +52,23 @@ end end |> permutedims gb_sum_st = sum.(groupby(st, Ti=>month, X => >(1.5))) @test dims(gb_sum_st, Ti) == Ti(Sampled([1:12...], ForwardOrdered(), Irregular((nothing, nothing)), Points(), NoMetadata())) - @test typeof(dims(gb_sum_st, X)) == typeof(X(Sampled(BitVector([false, true]), ForwardOrdered(), Irregular((nothing, nothing)), Points(), NoMetadata()))) @test gb_sum_st == manualsums_st + combined_sum_st = combine(sum, groupby(st, Ti=>month, X => >(1.5))) + @test collect(combined_sum_st) == manualsums_st @test_throws ArgumentError groupby(st, Ti=>month, Y=>isodd) end +@testset "partial reductions in combine" begin + months = DateTime(2000):Month(1):DateTime(2000, 12, 31) + using BenchmarkTools + manualmeans = cat(map(months) do m + mean(A[Ti=dayofyear(m):dayofyear(m)+daysinmonth(m)-1]; dims=Ti) + end...; dims=Ti(collect(1:12))) + combinedmeans = combine(mean, groupby(A, Ti()=>month); dims=Ti()) + @test combinedmeans == manualmeans +end + @testset "bins" begin seasons = DateTime(2000):Month(3):DateTime(2000, 12, 31) manualmeans = map(seasons) do s @@ -59,6 +78,7 @@ end @test mean.(groupby(A, Ti=>Bins(month, ranges(1:3:12)))) == manualmeans @test mean.(groupby(A, Ti=>Bins(month, intervals(1:3:12)))) == manualmeans @test mean.(groupby(A, Ti=>Bins(month, 4))) == manualmeans + @test combine(mean, groupby(A, Ti=>Bins(month, ranges(1:3:12)))) == manualmeans end @testset "dimension matching groupby" begin @@ -67,7 +87,7 @@ end B = rand(X(xs; sampling=Intervals(Start())), Ti(dates; sampling=Intervals(Start()))) gb = groupby(A, B) @test size(gb) === size(B) === size(mean.(gb)) - @test dims(gb) === dims(B) === dims(mean.(gb)) + @test parent(lookup(gb, X)) == parent(lookup(B, X)) == parent(lookup(mean.(gb), X)) manualmeans = mapreduce(hcat, intervals(dates)) do d map(intervals(xs)) do x mean(A[X=x, Ti=d]) @@ -75,9 +95,10 @@ end end @test all(collect(mean.(gb)) .=== manualmeans) @test all(mean.(gb) .=== manualmeans) + @test all(combine(mean, gb) .=== manualmeans) end -@testset "broadcastdims runs after groupby" begin +@testset "broadcast_dims runs after groupby" begin dimlist = ( Ti(Date("2021-12-01"):Day(1):Date("2022-12-31")), X(range(1, 10, length=10)), @@ -87,7 +108,7 @@ end data = rand(396, 10, 15, 2) A = DimArray(data, dimlist) month_length = DimArray(daysinmonth, dims(A, Ti)) - g_tempo = DimensionalData.groupby(month_length, Ti=>seasons(; start=December)) + g_tempo = DimensionalData.groupby(month_length, Ti => seasons(; start=December)) sum_days = sum.(g_tempo, dims=Ti) @test sum_days isa DimArray weights = map(./, g_tempo, sum_days) diff --git a/test/indexing.jl b/test/indexing.jl index b2b9d147b..cf18a682a 100644 --- a/test/indexing.jl +++ b/test/indexing.jl @@ -208,6 +208,7 @@ end @test b == da[1:2:end] == da[Begin:2:End] v = @inferred da[1, :] + @test v isa DimVector @test @inferred v[1:2] isa DimArray @test @inferred v[rand(Bool, length(v))] isa DimArray b = v[[!iseven(i) for i in 1:length(v)]] @@ -500,6 +501,7 @@ end @testset "mixed dimensions" begin a = [[1 2 3; 4 5 6];;; [11 12 13; 14 15 16];;;] da = DimArray(a, (X(143.0:2:145.0), Y(-38.0:-36.0), Ti(100:100:200)); name=:test) + da[1, End(), Begin(), 1] da[Ti=1, DimIndices(da[Ti=1])] da[DimIndices(da[Ti=1]), Ti(2)] da[DimIndices(da[Ti=1])[:], Ti(2)] diff --git a/test/runtests.jl b/test/runtests.jl index 25330be32..18962daf4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,7 +19,6 @@ end @time @safetestset "merged" begin include("merged.jl") end @time @safetestset "DimUnitRange" begin include("dimunitrange.jl") end @time @safetestset "format" begin include("format.jl") end - @time @safetestset "array" begin include("array.jl") end @time @safetestset "stack" begin include("stack.jl") end @time @safetestset "indexing" begin include("indexing.jl") end diff --git a/test/set.jl b/test/set.jl index e4ce8e999..f9b18d1a8 100644 --- a/test/set.jl +++ b/test/set.jl @@ -1,14 +1,14 @@ using DimensionalData, Test using DimensionalData.Lookups, DimensionalData.Dimensions -using DimensionalData.Lookups: _set +using DimensionalData: unsafe_set using DimensionalData: layerdims a = [1 2; 3 4] dimz = (X(143.0:2.0:145.0; lookup=Sampled(order=ForwardOrdered()), metadata=Metadata(Dict(:meta => "X"))), Y(-38.0:2.0:-36.0; lookup=Sampled(order=ForwardOrdered()), metadata=Metadata(Dict(:meta => "Y")))) da = DimArray(a, dimz; name=:test) -interval_da = set(da, X=Intervals(), Y=Intervals()) +interval_da = set(da, X => Intervals(), Y => Intervals()) a2 = [1 2 3 4 3 4 5 6 @@ -17,166 +17,310 @@ dimz2 = (Dim{:row}(10.0:10.0:30.0), Dim{:column}(-2:1.0:1.0)) da2 = DimArray(a2, dimz2; name=:test2) s = DimStack(da2, DimArray(2a2, dimz2; name=:test3)) +da2slice = set(da2[:,1], name=:onedim) +sslice = DimStack(da2, DimArray(2a2, dimz2; name=:test3), da2slice) +smix = DimStack(da, da2) +ssame = DimStack(da, set(da, X=>Z)) -@testset "Array fields" begin - @test parent(set(da2, fill(9, 3, 4))) == fill(9, 3, 4) +@testset "set DimArray parent" begin + a = fill(9, 3, 4) + @test parent(set(da2, a)) === a # A differently sized array can't be set - @test_throws ArgumentError parent(set(da2, [9 9; 9 9])) == [9 9; 9 9] + a_bad = [9 9; 9 9] + @test_throws DimensionMismatch parent(set(da2, a_bad)) + @test parent(unsafe_set(da2, a_bad)) === a_bad end -@testset "DimStack fields" begin +@testset "set DimStack parent" begin + s2 = set(s, (test2=zero(a2), test3=3a2)) + s3 = unsafe_set(s, (test2=zero(a2), test3=3a2)) + @test keys(s2) == keys(s3) == (:test2, :test3) + @test parent(s2) == parent(s3) == (test2=zero(a2), test3=3a2) + @test_throws ArgumentError set(s, (x=a2,)) @test_throws ArgumentError set(s, (x=a2, y=3a2)) - @test_throws ArgumentError set(s, (test2=a2, test3=hcat(a2, a2))) - s2 = set(s, (test2=a2, test3=3a2)) - @test keys(s2) == (:test2, :test3) - @test values(s2) == (a2, 3a2) + @test_throws DimensionMismatch set(s, (test2=a2, test3=hcat(a2, a2))) + @testset "set subset of layers" begin + a = ones(size(a2)) + @test parent(set(s, (; test2=a))) === (; test2=a, test3=parent(s).test3) + @test parent(unsafe_set(s, (; test2=a))) === (; test2=a) + end end -@testset "DimStack Dimension" begin - s1 = set(s, row=X, column=Z) +@testset "set DimStack Dimension type" begin + s1 = set(s, :row => X, :column => Z) @test typeof(dims(s1)) <: Tuple{<:X,<:Z} @test layerdims(s1) == (; test2=(X(), Z()), test3=(X(), Z())) - @test typeof(dims(set(s, row=X(), column=Z()))) <: Tuple{<:X,<:Z} - s1 = set(s, row=:row2, column=:column2) + @test typeof(dims(set(s, :row => X(), :column => Z()))) <: Tuple{<:X,<:Z} + # This should throw but doesn't at the moment. + #@test_throws ArgumentError set(smix, :row => X, :column => Z) + s1 = set(s, :row => :row2, :column => :column2) @test typeof(dims(s1)) <: Tuple{<:Dim{:row2},<:Dim{:column2}} @test layerdims(s1) == (; test2=(Dim{:row2}(), Dim{:column2}()), test3=(Dim{:row2}(), Dim{:column2}())) + s1 = set(ssame, :Z=>X) + @test_broken typeof(dims(s1)) <: Tuple{<:X,<:Y} + @test typeof(dims(set(s, :column => Ti(), :row => Z))) <: Tuple{<:Z,<:Ti} @test typeof(dims(set(s, Dim{:row}(Y()), Dim{:column}(X())))) <: Tuple{<:Y,<:X} @test typeof(dims(set(s, (Dim{:row}(Y), Dim{:column}(X))))) <: Tuple{<:Y,<:X} - @test parent(lookup(set(s, Dim{:row}([:x, :y, :z])), :row)) == [:x, :y, :z] + @test lookup(set(s, Dim{:row}([:x, :y, :z])), :row) == [:x, :y, :z] + @test lookup(set(s, Dim{:row}([:x, :y, :z])), :row) isa Sampled end -@testset "DimArray Dimension" begin - @test typeof(dims(set(da, X=:a, Y=:b))) <: Tuple{<:Dim{:a},<:Dim{:b}} +@testset "set DimArray Dimension type" begin + @test typeof(dims(set(da, X => :a, Y => :b))) <: Tuple{<:Dim{:a},<:Dim{:b}} @test typeof(dims(set(da2, Dim{:row}(Y()), Dim{:column}(X())))) <: Tuple{<:Y,<:X} @test typeof(dims(set(da, X => Ti(), Y => Z()))) <: Tuple{<:Ti,<:Z} @test typeof(dims(set(da2, :column => Ti(), :row => Z()))) <: Tuple{<:Z,<:Ti} @test typeof(dims(set(da2, (Dim{:row}(Y), Dim{:column}(X))))) <: Tuple{<:Y,<:X} - @test typeof(dims(set(da2, row=X, column=Z))) <: Tuple{<:X,<:Z} - @test typeof(dims(set(da2, row=X(), column=Z()))) <: Tuple{<:X,<:Z} - @test typeof(dims(set(da2, row=:row2, column=:column2))) <: Tuple{<:Dim{:row2},<:Dim{:column2}} + @test typeof(dims(set(da2, :row=>X, :column=>Z))) <: Tuple{<:X,<:Z} + @test typeof(dims(set(da2, :row=>X(), :column=>Z()))) <: Tuple{<:X,<:Z} + @test typeof(dims(set(da2, :row=>:row2, :column=>:column2))) <: Tuple{<:Dim{:row2},<:Dim{:column2}} @test index(set(da2, Dim{:row}([:x, :y, :z])), :row) == [:x, :y, :z] end -@testset "Dimension index" begin - @test index(set(da2, :column => [:a, :b, :c, :d], :row => 4:6)) == - (4:6, [:a, :b, :c, :d]) - @test index(set(s, :column => 10:5:20, :row => 4:6)) == (4:6, 10:5:20) - @test step.(span(set(da2, :column => 10:5:20, :row => 4:6))) == (1, 5) +@testset "set Lookup values" begin + dimz3 = (Dim{:row}(10.0:10.0:30.0), Dim{:column}(2.0:-1.0:-1.0)) + da3 = DimArray(a2, dimz3; name=:test2) + @testset "Order maintained" begin + c = [8, 4, 2, 1] + r = 4:6 + da_set = set(da3, :column => c, :row => r) + da_uset = unsafe_set(da3, :column => c, :row => r) + @test parent.(lookup(da_uset)) === parent.(lookup(da_set)) === (r, c) + @test order(da_set) == order(da_uset) == (ForwardOrdered(), ReverseOrdered()) + @test sampling(da_set) == sampling(da_uset) == (Points(), Points()) + @test span(da_set) == (Regular(1), Irregular((nothing, nothing))) + @test span(da_uset) === (Regular(10.0), Regular(-1.0)) + end + @testset "Order reversed" begin + c = [1, 2, 4, 8] + r = 6:-1:4 + da_set = set(da3, :column => c, :row => r) + da_uset = unsafe_set(da3, :column => c, :row => r) + @test parent.(lookup(da_uset)) === parent.(lookup(da_set)) === (r, c) + @test sampling(da_set) == sampling(da_uset) == (Points(), Points()) + @test span(da_set) == (Regular(-1), Irregular((nothing, nothing))) + @test span(da_uset) === (Regular(10.0), Regular(-1.0)) + @test order(da_set) == (ReverseOrdered(), ForwardOrdered()) + @test order(da_uset) == (ForwardOrdered(), ReverseOrdered()) + end + @testset "Ordered to Unordered" begin + c = [8, 2, 4, 1] + r = [6, 4, 5] + da_set = set(da3, :column => c, :row => r) + da_uset = unsafe_set(da3, :column => c, :row => r) + @test parent.(lookup(da_uset)) === parent.(lookup(da_set)) === (r, c) + @test sampling(da_set) == sampling(da_uset) == (Points(), Points()) + @test span(da_set) == (Irregular((nothing, nothing)), Irregular((nothing, nothing))) + @test span(da_uset) === (Regular(10.0), Regular(-1.0)) + @test order(da_set) == (Unordered(), Unordered()) + @test order(da_uset) == (ForwardOrdered(), ReverseOrdered()) + end + @testset "Unordered to Ordered" begin + # First define the unordered DimArray + c = [8, 2, 4, 1] + r = [6, 4, 5] + da4 = set(da3, :column => c, :row => r) + c = 8:-2:2 + r = 4:6 + da_set = set(da4, :column => c, :row => r) + da_uset = unsafe_set(da4, :column => c, :row => r) + @test parent.(lookup(da_uset)) === parent.(lookup(da_set)) === (r, c) + @test order(da_set) == (ForwardOrdered(), ReverseOrdered()) + @test order(da_uset) == (Unordered(), Unordered()) + @test sampling(da_set) == sampling(da_uset) == (Points(), Points()) + @test span(da_set) == (Regular(1), Regular(-2)) + @test span(da_uset) == (Irregular((nothing, nothing)), Irregular((nothing, nothing))) + end end -@testset "dim lookup" begin - - @test lookup(set(dims(da2), NoLookup())) == - (NoLookup(Base.OneTo(3)), NoLookup(Base.OneTo(4))) - @test lookup(set(da2, NoLookup())) == - (NoLookup(Base.OneTo(3)), NoLookup(Base.OneTo(4))) +# @testset "set Lookup" begin + @testset "to NoLookup" begin + @test lookup(set(dims(da2), NoLookup())) == + (NoLookup(Base.OneTo(3)), NoLookup(Base.OneTo(4))) + @test lookup(set(da2, NoLookup())) == + (NoLookup(Base.OneTo(3)), NoLookup(Base.OneTo(4))) + end @test lookup(set(da2, Categorical)) == (Categorical(10.0:10.0:30.0, ForwardOrdered(), NoMetadata()), Categorical(-2.0:1.0:1.0, ForwardOrdered(), NoMetadata())) @test lookup(set(da2, :column => NoLookup(), :row => Sampled(sampling=Intervals(Center())))) == (Sampled(10.0:10.0:30.0, ForwardOrdered(), Regular(10.0), Intervals(Center()), NoMetadata()), NoLookup(Base.OneTo(4))) - @test lookup(set(da2, column=NoLookup())) == + @test lookup(set(da2, Dim{:column}(NoLookup()))) == (Sampled(10.0:10.0:30.0, ForwardOrdered(), Regular(10.0), Points(), NoMetadata()), NoLookup(Base.OneTo(4))) + cat_da2 = set(da2, Categorical) + cat_da2_sample = set(da2, Sampled) + @test_broken cat_da2_sample == da2 @test lookup(set(da2, :column => NoLookup(), :row => Sampled())) == (Sampled(10.0:10.0:30.0, ForwardOrdered(), Regular(10.0), Points(), NoMetadata()), NoLookup(Base.OneTo(4))) - cat_da = set(da, X=NoLookup(), Y=Categorical()) + cat_da = set(da, X=>NoLookup(), Y=>Categorical()) @test index(cat_da) == (NoLookup(Base.OneTo(2)), Categorical(-38.0:2.0:-36.0, Unordered(), NoMetadata())) cat_da_m = set(dims(cat_da, Y), X(DimensionalData.AutoValues(); metadata=Dict())) + @test cat_da_m isa X @test metadata(cat_da_m) == Dict() +end - @testset "span" begin - @test span(set(da2, Irregular)) == - (Irregular((10.0, 30.0)), Irregular((-2.0, 1.0))) - @test span(set(da2, Regular)) == (Regular(10.0), Regular(1.0)) - # TODO: should this error? the span step doesn't match the index step - @test span(set(da2, row=Irregular(10, 12), column=Regular(9.9))) == - (Irregular(10, 12), Regular(9.9)) - @test set(Sampled(), AutoSpan()) == Sampled() - @test set(Sampled(), Irregular()) == Sampled(; span=Irregular()) - @test set(Sampled(), Regular()) == Sampled(; span=Regular()) - @test set(Sampled(1:2:10), Regular()) == Sampled(1:2:10; span=Regular(2)) - end +# @testset "set Span" begin + @test set(Regular(), AutoSpan()) == Regular() + @test set(Regular(), Irregular()) == Irregular() + @test set(Irregular(), Regular()) == Regular() - @testset "locus" begin - @test_throws ArgumentError set(da2, (End(), Center())) - @test locus(set(interval_da, X(End()), Y(Center()))) == (End(), Center()) - @test locus(set(interval_da, X=>End(), Y=>Center())) == (End(), Center()) - @test locus(set(interval_da, X=End, Y=Center)) == (End(), Center()) - @test locus(set(da, Y=Center())) == (Center(), Center()) - @test _set(Points(), Intervals()) == Intervals(Center()) - @test _set(Intervals(Center()), Start()) == Intervals(Start()) - @test _set(Intervals(Center()), AutoLocus()) == Intervals(Center()) - @test _set(Points(), Center()) == Points() - @test_throws ArgumentError _set(Points(), Start()) - @test_throws ArgumentError _set(Points(), End()) - end + @test set(Sampled(), Irregular()) == Sampled(; span=Irregular()) + @test set(Sampled(), Regular()) == Sampled(; span=Regular()) + @test set(Sampled(1:2:10), Regular()) == Sampled(1:2:10; span=Regular(2)) + @test set(Sampled([3, 6, 9, 12]), Regular()) == Sampled([3, 6, 9, 12]; span=Regular(3)) - @testset "sampling" begin - @test sampling(interval_da) == (Intervals(Center()), Intervals(Center())) - @test sampling(set(da, (X(Intervals(End())), Y(Intervals(Start()))))) == - (Intervals(End()), Intervals(Start())) - @test set(Sampled(), AutoSampling()) == Sampled() - @test set(Sampled(), Intervals) == Sampled(; sampling=Intervals()) - @test _set(Points(), AutoSampling()) == Points() - @test _set(AutoSampling(), Intervals()) == Intervals() - @test _set(AutoSampling(), AutoSampling()) == AutoSampling() - end + @test span(set(da2, Irregular)) == + (Irregular((10.0, 30.0)), Irregular((-2.0, 1.0))) + @test span(set(da2, Regular)) == (Regular(10.0), Regular(1.0)) + # TODO: should this error? the span step doesn't match the index step + @test span(set(da2, :row=>Irregular(10, 12), :column=>Regular(9.9))) == + (Irregular(10, 12), Regular(9.9)) +end - @testset "order" begin - uda = set(da, Y(Unordered())) - @test order(uda) == (ForwardOrdered(), Unordered()) - @test order(set(uda, X=ReverseOrdered())) == (ReverseOrdered(), Unordered()) - end +@testset "set Locus" begin + @test locus(set(interval_da, X(End()), Y(Center()))) == (End(), Center()) + @test locus(set(interval_da, X => End(), Y => Center())) == (End(), Center()) + @test set(set(interval_da, End), Center) == interval_da + @test set(set(interval_da, Start()), Center) == interval_da + @test locus(set(da, Y=>Center())) == (Center(), Center()) + @test set(Points(), Intervals()) == Intervals(Center()) + @test set(Intervals(Center()), Start()) == Intervals(Start()) + @test set(Intervals(Center()), AutoLocus()) == Intervals(Center()) + @test set(Points(), Center()) == Points() + @test_throws ArgumentError set(Points(), Start()) + @test_throws ArgumentError set(Points(), End()) +end - # issue #478 - @testset "tuple dims and/or Symbol/Dim{Colon}/Colon replacement" begin - @test set(Dim{:foo}(), :bar) === Dim{:bar}() - @test set(Dim{:foo}(2:11), :bar) === Dim{:bar}(2:11) - @test set(Dim{:foo}(), Dim{:bar}()) === Dim{:bar}() - @test set(Dim{:foo}(2:11), Dim{:bar}()) === Dim{:bar}(2:11) - @test set(Dim{:foo}(Lookups.Sampled(2:11)), Dim{:bar}(Lookups.Sampled(0:9))) === - set(set(Dim{:foo}(Lookups.Sampled(2:11)), :bar), Lookups.Sampled(0:9)) - @test set((Dim{:foo}(),), :foo => :bar) === (Dim{:bar}(),) - @test set((Dim{:foo}(2:11),), :foo => :bar) === (Dim{:bar}(2:11),) - @test set(dimz, :X => :foo, :Y => :bar) === - (set(dims(dimz, :X), :foo), set(dims(dimz, :Y), :bar)) +@testset "set Sampling" begin + @test sampling(interval_da) == (Intervals(Center()), Intervals(Center())) + @test sampling(set(da, (X(Intervals(End())), Y(Intervals(Start()))))) == + (Intervals(End()), Intervals(Start())) + @test set(Sampled(), AutoSampling()) == Sampled() + @test set(Sampled(), Intervals) == Sampled(; sampling=Intervals()) + @test set(Points(), AutoSampling()) == Points() + @test set(AutoSampling(), Intervals()) == Intervals() + @test set(AutoSampling(), AutoSampling()) == AutoSampling() +end + +@testset "set Order" begin + a = [1 2 3; 4 5 6; 7 8 9] + dimz = (X(Sampled(100.0:10.0:120.0; metadata=Metadata(Dict(:meta => "X")))), + Y(Categorical([:a, :b, :c]; metadata=Metadata(Dict(:meta => "Y"))))) + da_o = DimArray(a, dimz; name=:test) + @testset "new Order is taken" begin + @test set(ForwardOrdered(), Unordered()) == Unordered() + end + @testset "old Order is kep for AutoOrder" begin + @test set(ReverseOrdered(), AutoOrder()) == ReverseOrdered() + end + @testset "Unordered does not affect parent" begin + uda = set(da_o, Unordered()) + uuda = unsafe_set(da_o, Unordered()) + @test_broken span(uda) == (Regular(2.0), Irregular((nothing, nothing))) + @test span(uuda) == (Regular(10.0), NoSpan()) + @test order(uda) == order(uuda) == (Unordered(), Unordered()) + @test parent(uda) == parent(uuda) == parent(da) + @test lookup(uda) == (100.0:10.0:120.0, [:a, :b, :c]) + @test lookup(uuda) == (100.0:10.0:120.0, [:a, :b, :c]) + end + @testset "Ordered reverses ordered parent" begin + rda = set(da_o, ReverseOrdered()) + urda = unsafe_set(uda, ReverseOrdered()) + @test order(rda) == (ReverseOrdered(), ReverseOrdered()) + @test lookup(rda) == (120.0:-10.0:100.0, [:c, :b, :a]) + @test lookup(uroda) == ([110.0, 120.0, 100.0], [:c, :a, :b]) + @test parent(rda) == parent(reverse(da; dims=(X, Y))) + @test parent(urda) == parent(da) + end + @testset "Ordered sorts unordered parent" begin + da_u = set(da_o[X=[2, 3, 1], Y=[3, 1, 2]], Unordered()) + fda = set(da_u, ForwardOrdered()) + ufda = unsafe_set(da_u, ForwardOrdered()) + rda = set(dau, ReverseOrdered()) + urda = unsafe_set(da_u, ReverseOrdered()) + @test order(fda) == order(ufda) == (ForwardOrdered(), ForwardOrdered()) + @test order(rda) == order(urda) == (ReverseOrdered(), ReverseOrdered()) + @test lookup(fda) == (100.0:10.0:120.0, [:a, :b, :c]) + @test lookup(rda) == (120.0:-10.0:100.0, [:c, :b, :a]) + @test map(parent, lookup(ufda)) == map(parent, lookup(urda)) == ([110.0, 120.0, 100.0], [:c, :a, :b]) + @test parent(fda) == parent(da_o) + @test parent(rda) == parent(reverse(da_o; dims=(X, Y))) + @test parent(ufda) == parent(urda) == parent(da_u) end end -@testset "metadata" begin +# # issue #478 +# @testset "tuple dims and/or Symbol/Dim{Colon}/Colon replacement" begin + @test set(Dim{:foo}(), :bar) === Dim{:bar}() + @test set(Dim{:foo}(2:11), :bar) === Dim{:bar}(2:11) + @test set(Dim{:foo}(), Dim{:bar}()) === Dim{:bar}() + @test set(Dim{:foo}(2:11), Dim{:bar}()) === Dim{:bar}(2:11) + @test set(Dim{:foo}(Lookups.Sampled(2:11)), Dim{:bar}(Lookups.Sampled(0:9))) === + set(set(Dim{:foo}(Lookups.Sampled(2:11)), :bar), Lookups.Sampled(0:9)) + @test set((Dim{:foo}(),), :foo => :bar) === (Dim{:bar}(),) + @test set((Dim{:foo}(2:11),), :foo => :bar) === (Dim{:bar}(2:11),) + @test set(dimz, :X => :foo, :Y => :bar) === + (set(dims(dimz, :X), :foo), set(dims(dimz, :Y), :bar)) +end + +@testset "set metadata" begin @test metadata(set(Sampled(), Metadata(Dict(:a=>1, :b=>2)))).val == Dict(:a=>1, :b=>2) dax = set(da, X => Metadata(Dict(:a=>1, :b=>2))) @test metadata(dims(dax), X).val == Dict(:a=>1, :b=>2) @test metadata(dims(dax), Y).val == Dict(:meta => "Y") dax = set(da, X => Metadata(Dict(:a=>1, :b=>2))) @test metadata(dims(dax, X)).val == Dict(:a=>1, :b=>2) - dax = set(da2, row=Metadata(Dict(:a=>1, :b=>2))) + dax = set(da2, :row=>Metadata(Dict(:a=>1, :b=>2))) @test metadata(dims(dax, :row)).val == Dict(:a=>1, :b=>2) - dax = set(da2, column=Metadata(Dict(:a=>1, :b=>2))) - @test metadata(dims(dax, :column)).val == Dict(:a=>1, :b=>2) + dax = set(da2, :column=>Dict(:a=>1, :b=>2)) + @test metadata(dims(dax, :column)) == Dict(:a=>1, :b=>2) end -@testset "all dim fields" begin +@testset "all lookup fields updated" begin md = Metadata(Dict(:a=>1, :b=>2)) - dax = set(da, X(20:-10:10; metadata=md)) - x = dims(dax, X) - @test index(x) === 20:-10:10 + da_set = set(da, X(20:-10:10; metadata=md)); + x = dims(da_set, X) + @test parent(lookup(x)) === 20:-10:10 @test order(x) === ReverseOrdered() @test span(x) === Regular(-10) + @test metadata(x).val == Dict(:a=>1, :b=>2) @test lookup(x) == Sampled(20:-10:10, ReverseOrdered(), Regular(-10), Points(), md) + + # Or not with `unsafe_set`... + da_uset = DimensionalData.unsafe_set(da, X(20:-10:10; metadata=md)); + x = dims(da_uset, X) + @test order(x) === ForwardOrdered() + @test span(x) === Regular(2.0) @test metadata(x).val == Dict(:a=>1, :b=>2) + @test lookup(x) == Sampled(20:-10:10, ForwardOrdered(), Regular(2.0), Points(), md) +end + +@testset "reordering with set" begin + # set changes the data, order and span + @test parent(set(da, ForwardOrdered)) === parent(da) + @test parent(set(da, ReverseOrdered)) == parent(reverse(da; dims=(X, Y))) + @test span(set(da, ReverseOrdered)) === map(reverse, span(da)) + # unsafe_set does not change the data or span + @test parent(unsafe_set(da, ForwardOrdered)) === parent(unsafe_set(da, ReverseOrdered)) === parent(da) + @test span(unsafe_set(da, ForwardOrdered)) === span(unsafe_set(da, ReverseOrdered)) === span(da) + # But it changes the order + @test order(unsafe_set(da, ForwardOrdered)) === (ForwardOrdered(), ForwardOrdered()) + @test order(unsafe_set(da, ReverseOrdered)) === (ReverseOrdered(), ReverseOrdered()) + # Setting Unordered is the same for set and unsafe_set + @test set(da, Unordered) === unsafe_set(da, Unordered) + @test parent(set(da, Unordered)) == parent(unsafe_set(da, Unordered)) == da + @test order(set(da, Unordered)) == order(unsafe_set(da, Unordered)) == (Unordered(), Unordered()) end @testset "errors with set" begin - @test_throws ArgumentError set(da, X=7) @test_throws ArgumentError set(dims(da, X), X(7)) - @test_throws ArgumentError set(da, notadimname=Sampled()) + @test_throws ArgumentError set(da, notafield=Sampled()) end -@testset "_set nothing" begin - @test _set(nothing, nothing) == nothing - @test _set(1, nothing) == 1 - @test _set(nothing, 2) == 2 -end +# @testset "_set nothing" begin +# @test _set(nothing, nothing) == nothing +# @test _set(1, nothing) == 1 +# @test _set(nothing, 2) == 2 +# end diff --git a/test/stack.jl b/test/stack.jl index cfde24449..1d7d36242 100644 --- a/test/stack.jl +++ b/test/stack.jl @@ -3,7 +3,7 @@ using DimensionalData, Test, LinearAlgebra, Statistics, ConstructionBase, Random using DimensionalData: data using DimensionalData: Sampled, Categorical, AutoLookup, NoLookup, Transformed, Regular, Irregular, Points, Intervals, Start, Center, End, - Metadata, NoMetadata, ForwardOrdered, ReverseOrdered, Unordered, layers, basedims + Metadata, NoMetadata, ForwardOrdered, ReverseOrdered, Unordered, basedims, layerdims, layers, metadata A = [1.0 2.0 3.0; 4.0 5.0 6.0] @@ -94,11 +94,23 @@ end @test all(maplayers(similar(mixed), mixed) do s, m dims(s) == dims(m) && dims(s) === dims(m) && eltype(s) === eltype(m) end) - @test eltype(similar(s, Int)) === @NamedTuple{one::Int, two::Int, three::Int} + @test eltype(similar(s, Int)) === + @NamedTuple{one::Int, two::Int, three::Int} + @test eltype(similar(s, @NamedTuple{one::Int, two::Float32, three::Bool})) === + @NamedTuple{one::Int, two::Float32, three::Bool} st2 = similar(mixed, Bool, x, y) @test dims(st2) === (x, y) @test dims(st2[:one]) === (x, y) @test eltype(st2) === @NamedTuple{one::Bool, two::Bool, extradim::Bool} + @test eltype(similar(mixed)) == eltype(mixed) + @test size(similar(mixed)) == size(mixed) + @test keys(similar(mixed)) == keys(mixed) + @test layerdims(similar(mixed)) == layerdims(mixed) + xy = (X(), Y()) + @test layerdims(similar(mixed, dims(mixed, (X, Y)))) == (one=xy, two=xy, extradim=xy) + st3 = similar(mixed, @NamedTuple{one::Int, two::Float32, extradim::Bool}, (Z([:a, :b, :c]), Ti(1:12), X(1:3))) + @test layerdims(st3) == (one=(Ti(), X()), two=(Ti(), X()), extradim=(Z(), Ti(), X())) + @test eltype(st3) == @NamedTuple{one::Int, two::Float32, extradim::Bool} end @testset "merge" begin diff --git a/test/tables.jl b/test/tables.jl index b5bd416ea..26466fb4f 100644 --- a/test/tables.jl +++ b/test/tables.jl @@ -1,4 +1,10 @@ -using DimensionalData, IteratorInterfaceExtensions, TableTraits, Tables, Test, DataFrames +using DataFrames +using Dates +using DimensionalData +using IteratorInterfaceExtensions +using TableTraits +using Tables +using Test using DimensionalData.Lookups, DimensionalData.Dimensions using DimensionalData: DimTable, DimExtensionArray @@ -144,13 +150,67 @@ end @testset "DimTable mergelayers" begin a = DimStack([DimArray(rand(32, 32, 3), (X,Y,Ti)) for _ in 1:3]) - b = DimArray(rand(32, 32, 3), (X,Y,Dim{:band})) - t1 = DimTable(a, mergedims=(:X,:Y)=>:geometry) - t2 = DimTable(a, mergedims=(:X,:Y,:Z)=>:geometry) # Merge missing dimension - t3 = DimTable(a, mergedims=(X,:Y,Ti)=>:dimensions) # Mix symbols and dimensions - t4 = DimTable(b, mergedims=(:X,:Y)=>:geometry) # Test DimArray + b = DimArray(rand(32, 32, 3), (X, Y, Dim{:band})) + t1 = DimTable(a, mergedims=(:X, :Y) => :geometry) + t2 = DimTable(a, mergedims=(:X, :Y, :Z) => :geometry) # Merge missing dimension + t3 = DimTable(a, mergedims=(X, :Y, Ti) => :dimensions) # Mix symbols and dimensions + t4 = DimTable(b, mergedims=(:X, :Y) => :geometry) # Test DimArray @test Tables.columnnames(t1) == (:Ti, :geometry, :layer1, :layer2, :layer3) @test Tables.columnnames(t2) == (:Ti, :geometry, :layer1, :layer2, :layer3) @test Tables.columnnames(t3) == (:dimensions, :layer1, :layer2, :layer3) @test Tables.columnnames(t4) == (:band, :geometry, :value) end + +@testset "DimTable preservedims" begin + x, y, t = X(1.0:32.0), Y(1.0:10.0), Ti(DateTime.([2001, 2002, 2003])) + st = DimStack([rand(x, y, t; name) for name in [:a, :b, :c]]) + A = rand(x, y, Dim{:band}(1:3); name=:vals) + t1 = DimTable(st, preservedims=(X, Y)) + a3 = Tables.getcolumn(t1, :a)[3] + @test Tables.columnnames(t1) == propertynames(t1) == (:Ti, :a, :b, :c) + @test a3 == st.a[Ti=3] + @test dims(a3) == dims(st, (X, Y)) + t2 = DimTable(A; preservedims=:band) + val10 = Tables.getcolumn(t2, :vals)[10] + @test Tables.columnnames(t2) == propertynames(t2) == (:X, :Y, :vals) + @test val10 == A[X(10), Y(1)] + @test dims(val10) == dims(A, (:band,)) + @testset "preservedims with mergedims" begin + t3 = DimTable(A; mergedims=(X, Y) => :geometry, preservedims=:band) + @test only(dims(t3)) isa Dim{:geometry} + @test Tables.getcolumn(t2, :vals)[1] isa DimArray + end +end + +@testset "DimTable NamedTuple" begin + @testset "Vector of NamedTuple" begin + da = DimArray([(; a=1.0f0i, b=2.0i) for i in 1:10], X) + t = DimTable(da) + s = Tables.schema(t) + @test s.names == (:X, :a, :b) + @test s.types == (Int, Float32, Float64) + @test all(t.a .=== 1.0f0:10.0f0) + @test all(t.b .=== 2.0:2.0:20.0) + end + + @testset "Matrix of NamedTuple" begin + da = [(; a=1.0f0x*y, b=2.0x*y) for x in X(1:10), y in Y(1:5)] + t = DimTable(da); + s = Tables.schema(t) + @test s.names == (:X, :Y, :a, :b) + @test s.types == (Int, Int, Float32, Float64) + @test all(t.a .=== reduce(vcat, [1.0f0y:y:10.0f0y for y in 1:5])) + @test all(t.b .=== reduce(vcat, [2.0y:2.0y:20.0y for y in 1:5])) + end + @testset "Matrix of NamedTuple with preservedims" begin + da = [(; a=1.0f0x*y, b=2.0x*y) for x in X(1:10), y in Y(1:5)] + t = DimTable(da; preservedims=X); + s = Tables.schema(t) + @test s.names == (:Y, :a, :b) + @test s.types[1] <: Int + @test s.types[2] <: DimVector + @test s.types[2] <: DimVector + @test all(t.a .== [[1.0f0x*y for x in X(1:10)] for y in Y(1:5)]) + @test all(t.b .== [[2.0x*y for x in X(1:10)] for y in Y(1:5)]) + end +end diff --git a/test/utils.jl b/test/utils.jl index 91dc30a6d..1fbb2b1cc 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -97,11 +97,20 @@ end @test reo_s == s @test dims(reo_s) == dims(s) - @testset "reorder handles extra dimensions" begin @test reorder(da[X=1], X=>ReverseOrdered(), Y=>ForwardOrdered()) == rev[X=1] @test reorder(rev_s[X=1], da) == s[X=1] end + + @testset "unordered are sorted" begin + x, y = X([:a, :c, :b]), Y([1, 4, 2, 3]) + A = rand(x, y) + Af = reorder(A, ForwardOrdered()) + @test parent(Af) == A[X=[1, 3, 2], Y=[1, 3, 4, 2]] + @test lookup(Af) == lookup(reorder(dims(A), ForwardOrdered())) == + (Categorical([:a, :b, :c], ForwardOrdered(), NoMetadata()), + Sampled([1, 2, 3, 4], ForwardOrdered(), Irregular(nothing,nothing), Points(), NoMetadata())) + end end @testset "modify" begin