Skip to content

[clickhouse] Add handling for complex attributes to ClickHouse storage #7627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Nov 2, 2025
Merged

[clickhouse] Add handling for complex attributes to ClickHouse storage #7627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
dc9e94c
Update Documentation
mahadzaryab1 Oct 28, 2025
e5c5c49
Update Translation Layer
mahadzaryab1 Oct 28, 2025
8f3431c
Remove Anonymous Struct
mahadzaryab1 Oct 28, 2025
b3da693
Fix
mahadzaryab1 Oct 28, 2025
133b21e
Update Test For Warnings
mahadzaryab1 Oct 28, 2025
7b1fd2a
Update Tests With Test Data
mahadzaryab1 Oct 30, 2025
cb617ad
Simplify Map To Avoid Flakiness
mahadzaryab1 Oct 31, 2025
46715c7
Add Default
mahadzaryab1 Nov 1, 2025
b34584e
Improve Test Coverage
mahadzaryab1 Nov 1, 2025
6a1b5b5
Fix
mahadzaryab1 Nov 1, 2025
1848a8e
Refactor To Store as JSON String
mahadzaryab1 Nov 1, 2025
948e95e
Update Documentation
mahadzaryab1 Nov 1, 2025
b33fcb3
Add Warnings
mahadzaryab1 Nov 1, 2025
da1ad98
Run Formatter
mahadzaryab1 Nov 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions internal/storage/v2/clickhouse/tracestore/assert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/ptrace"
"go.opentelemetry.io/collector/pdata/xpdata"

"github.com/jaegertracing/jaeger/internal/jptrace"
"github.com/jaegertracing/jaeger/internal/storage/v2/clickhouse/tracestore/dbmodel"
Expand Down Expand Up @@ -126,6 +127,24 @@ func requireComplexAttrs(t *testing.T, expectedKeys []string, expectedVals []str
decoded, err := base64.StdEncoding.DecodeString(expectedVals[i])
require.NoError(t, err)
require.Equal(t, decoded, val.Bytes().AsRaw())
case strings.HasPrefix(k, "@map@"):
key := strings.TrimPrefix(expectedKeys[i], "@map@")
val, ok := attrs.Get(key)
require.True(t, ok)

m := &xpdata.JSONUnmarshaler{}
expectedVal, err := m.UnmarshalValue([]byte(expectedVals[i]))
require.NoError(t, err)
require.True(t, expectedVal.Map().Equal(val.Map()))
case strings.HasPrefix(k, "@slice@"):
key := strings.TrimPrefix(expectedKeys[i], "@slice@")
val, ok := attrs.Get(key)
require.True(t, ok)

m := &xpdata.JSONUnmarshaler{}
expectedVal, err := m.UnmarshalValue([]byte(expectedVals[i]))
require.NoError(t, err)
require.True(t, expectedVal.Slice().Equal(val.Slice()))
default:
t.Fatalf("unsupported complex attribute key: %s", k)
}
Expand Down
42 changes: 30 additions & 12 deletions internal/storage/v2/clickhouse/tracestore/dbmodel/dbmodel_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/ptrace"
"go.opentelemetry.io/collector/pdata/xpdata"

"github.com/jaegertracing/jaeger/internal/telemetry/otelsemconv"
)
Expand All @@ -32,7 +33,7 @@ func TestRoundTrip(t *testing.T) {
})

t.Run("FromRow->ToRow", func(t *testing.T) {
spanRow := createTestSpanRow(now, duration)
spanRow := createTestSpanRow(t, now, duration)

trace := FromRow(spanRow)
rs := trace.ResourceSpans().At(0).Resource()
Expand Down Expand Up @@ -114,10 +115,27 @@ func addTestAttributes(attrs pcommon.Map) {
attrs.PutInt("int_attr", 42)
attrs.PutStr("string_attr", "string_value")
attrs.PutEmptyBytes("bytes_attr").FromRaw([]byte("bytes_value"))
attrs.PutEmptyMap("map_attr").FromRaw(map[string]any{"key": "value"})
attrs.PutEmptySlice("slice_attr").FromRaw([]any{1, 2, 3})
}

func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
func createTestSpanRow(t *testing.T, now time.Time, duration time.Duration) *SpanRow {
t.Helper()
encodedBytes := base64.StdEncoding.EncodeToString([]byte("bytes_value"))

vm := pcommon.NewValueMap()
vm.Map().PutStr("key", "value")
m := &xpdata.JSONMarshaler{}
vmJSON, err := m.MarshalValue(vm)
require.NoError(t, err)

vs := pcommon.NewValueSlice()
vs.Slice().AppendEmpty().SetInt(1)
vs.Slice().AppendEmpty().SetInt(2)
vs.Slice().AppendEmpty().SetInt(3)
vsJSON, err := m.MarshalValue(vs)
require.NoError(t, err)

return &SpanRow{
ID: "0000000000000001",
TraceID: "00000000000000000000000000000001",
Expand All @@ -138,8 +156,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
IntValues: []int64{42},
StrKeys: []string{"string_attr"},
StrValues: []string{"string_value"},
ComplexKeys: []string{"@bytes@bytes_attr"},
ComplexValues: []string{encodedBytes},
ComplexKeys: []string{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"},
ComplexValues: []string{encodedBytes, string(vmJSON), string(vsJSON)},
},
EventNames: []string{"test-event"},
EventTimestamps: []time.Time{now},
Expand All @@ -152,8 +170,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
IntValues: [][]int64{{42}},
StrKeys: [][]string{{"string_attr"}},
StrValues: [][]string{{"string_value"}},
ComplexKeys: [][]string{{"@bytes@bytes_attr"}},
ComplexValues: [][]string{{encodedBytes}},
ComplexKeys: [][]string{{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"}},
ComplexValues: [][]string{{encodedBytes, string(vmJSON), string(vsJSON)}},
},
LinkTraceIDs: []string{"00000000000000000000000000000003"},
LinkSpanIDs: []string{"0000000000000004"},
Expand All @@ -167,8 +185,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
IntValues: [][]int64{{42}},
StrKeys: [][]string{{"string_attr"}},
StrValues: [][]string{{"string_value"}},
ComplexKeys: [][]string{{"@bytes@bytes_attr"}},
ComplexValues: [][]string{{encodedBytes}},
ComplexKeys: [][]string{{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"}},
ComplexValues: [][]string{{encodedBytes, string(vmJSON), string(vsJSON)}},
},
ServiceName: "test-service",
ResourceAttributes: Attributes{
Expand All @@ -180,8 +198,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
IntValues: []int64{42},
StrKeys: []string{"service.name", "string_attr"},
StrValues: []string{"test-service", "string_value"},
ComplexKeys: []string{"@bytes@bytes_attr"},
ComplexValues: []string{encodedBytes},
ComplexKeys: []string{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"},
ComplexValues: []string{encodedBytes, string(vmJSON), string(vsJSON)},
},
ScopeName: "test-scope",
ScopeVersion: "v1.0.0",
Expand All @@ -194,8 +212,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
IntValues: []int64{42},
StrKeys: []string{"string_attr"},
StrValues: []string{"string_value"},
ComplexKeys: []string{"@bytes@bytes_attr"},
ComplexValues: []string{encodedBytes},
ComplexKeys: []string{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"},
ComplexValues: []string{encodedBytes, string(vmJSON), string(vsJSON)},
},
}
}
40 changes: 39 additions & 1 deletion internal/storage/v2/clickhouse/tracestore/dbmodel/from.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/ptrace"
"go.opentelemetry.io/collector/pdata/xpdata"

"github.com/jaegertracing/jaeger/internal/jptrace"
"github.com/jaegertracing/jaeger/internal/telemetry/otelsemconv"
Expand Down Expand Up @@ -169,14 +170,51 @@ func putAttributes(
attrs.PutStr(storedAttrs.StrKeys[i], storedAttrs.StrValues[i])
}
for i := 0; i < len(storedAttrs.ComplexKeys); i++ {
if strings.HasPrefix(storedAttrs.ComplexKeys[i], "@bytes@") {
switch {
case strings.HasPrefix(storedAttrs.ComplexKeys[i], "@bytes@"):
decoded, err := base64.StdEncoding.DecodeString(storedAttrs.ComplexValues[i])
if err != nil {
jptrace.AddWarnings(spanForWarnings, fmt.Sprintf("failed to decode bytes attribute %q: %s", storedAttrs.ComplexKeys[i], err.Error()))
continue
}
k := strings.TrimPrefix(storedAttrs.ComplexKeys[i], "@bytes@")
attrs.PutEmptyBytes(k).FromRaw(decoded)
case strings.HasPrefix(storedAttrs.ComplexKeys[i], "@slice@"):
Copy link
Member

@yurishkuro yurishkuro Nov 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aside from prefix the code is identical to next case:, can move to shared helper

Copy link
Collaborator Author

@mahadzaryab1 mahadzaryab1 Nov 1, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yurishkuro The slice and map are added differently

attrs.PutEmptySlice(k).FromRaw(val.Slice().AsRaw())

vs.

attrs.PutEmptyMap(k).FromRaw(val.Map().AsRaw())

The warning messages are slightly different as well.

k := strings.TrimPrefix(storedAttrs.ComplexKeys[i], "@slice@")
m := &xpdata.JSONUnmarshaler{}
val, err := m.UnmarshalValue([]byte(storedAttrs.ComplexValues[i]))
if err != nil {
jptrace.AddWarnings(
spanForWarnings,
fmt.Sprintf(
"failed to unmarshal slice attribute %q: %s",
storedAttrs.ComplexKeys[i],
err.Error(),
),
)
continue
}
attrs.PutEmptySlice(k).FromRaw(val.Slice().AsRaw())
case strings.HasPrefix(storedAttrs.ComplexKeys[i], "@map@"):
k := strings.TrimPrefix(storedAttrs.ComplexKeys[i], "@map@")
m := &xpdata.JSONUnmarshaler{}
val, err := m.UnmarshalValue([]byte(storedAttrs.ComplexValues[i]))
if err != nil {
jptrace.AddWarnings(
spanForWarnings,
fmt.Sprintf("failed to unmarshal map attribute %q: %s",
storedAttrs.ComplexKeys[i],
err.Error(),
),
)
continue
}
attrs.PutEmptyMap(k).FromRaw(val.Map().AsRaw())
default:
jptrace.AddWarnings(
spanForWarnings,
fmt.Sprintf("unsupported complex attribute key: %q", storedAttrs.ComplexKeys[i]),
)
}
}
}
68 changes: 50 additions & 18 deletions internal/storage/v2/clickhouse/tracestore/dbmodel/from_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ func TestFromRow(t *testing.T) {
now := time.Now().UTC()
duration := 2 * time.Second

spanRow := createTestSpanRow(now, duration)
spanRow := createTestSpanRow(t, now, duration)

expected := createTestTrace(now, duration)

Expand Down Expand Up @@ -89,23 +89,55 @@ func TestFromRow_DecodeID(t *testing.T) {
}

func TestPutAttributes_Warnings(t *testing.T) {
t.Run("bytes attribute with invalid base64", func(t *testing.T) {
span := ptrace.NewSpan()
attributes := pcommon.NewMap()
tests := []struct {
name string
complexKeys []string
complexValues []string
expectedWarnContains string
}{
{
name: "bytes attribute with invalid base64",
complexKeys: []string{"@bytes@bytes-key"},
complexValues: []string{"invalid-base64"},
expectedWarnContains: "failed to decode bytes attribute \"@bytes@bytes-key\"",
},
{
name: "failed to unmarshal slice attribute",
complexKeys: []string{"@slice@slice-key"},
complexValues: []string{"notjson"},
expectedWarnContains: "failed to unmarshal slice attribute \"@slice@slice-key\"",
},
{
name: "failed to unmarshal map attribute",
complexKeys: []string{"@map@map-key"},
complexValues: []string{"notjson"},
expectedWarnContains: "failed to unmarshal map attribute \"@map@map-key\"",
},
{
name: "unsupported complex attribute key",
complexKeys: []string{"unsupported"},
complexValues: []string{"{\"kvlistValue\":{\"values\":[{\"key\":\"key\",\"value\":{\"stringValue\":\"value\"}}]}}"},
expectedWarnContains: "unsupported complex attribute key: \"unsupported\"",
},
}

putAttributes(
attributes,
&Attributes{
ComplexKeys: []string{"@bytes@bytes-key"},
ComplexValues: []string{"invalid-base64"},
},
span,
)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
span := ptrace.NewSpan()
attributes := pcommon.NewMap()

putAttributes(
attributes,
&Attributes{
ComplexKeys: tt.complexKeys,
ComplexValues: tt.complexValues,
},
span,
)

_, ok := attributes.Get("bytes-key")
require.False(t, ok)
warnings := jptrace.GetWarnings(span)
require.Len(t, warnings, 1)
require.Contains(t, warnings[0], "failed to decode bytes attribute \"@bytes@bytes-key\"")
})
warnings := jptrace.GetWarnings(span)
require.Len(t, warnings, 1)
require.Contains(t, warnings[0], tt.expectedWarnContains)
})
}
}
24 changes: 15 additions & 9 deletions internal/storage/v2/clickhouse/tracestore/dbmodel/spanrow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,22 @@ import (
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)

// SpanRow represents a single row in the ClickHouse `spans` table.
// SpanRow represents a single record in the ClickHouse `spans` table.
//
// Complex attributes are attributes that are not of a primitive type and hence need special handling.
// The following OTLP types are stored in the complex attributes fields:
// - AnyValue_BytesValue: This OTLP type is stored as a base64-encoded string. The key
// for this type will begin with `@bytes@`.
// - AnyValue_ArrayValue: This OTLP type is stored as a JSON-encoded string.
// The key for this type will begin with `@array@`.
// - AnyValue_KVListValue: This OTLP type is stored as a JSON-encoded string.
// The key for this type will begin with `@kvlist@`.
// Complex attributes are non-primitive OTLP types that require special serialization
// before being stored. These types are encoded as follows:
//
// - pcommon.ValueTypeBytes:
// Represents raw byte data. The value is Base64-encoded and stored as a string.
// Keys for this type are prefixed with `@bytes@`.
//
// - pcommon.ValueTypeSlice:
// Represents an OTLP slice (array). The value is serialized to JSON and stored
// as a string. Keys for this type are prefixed with `@slice@`.
//
// - pcommon.ValueTypeMap:
// Represents an OTLP map. The value is serialized to JSON and stored
// as a string. Keys for this type are prefixed with `@map@`.
type SpanRow struct {
// --- Span ---
ID string
Expand Down
Loading
Loading