[clickhouse] Add handling for complex attributes to ClickHouse storage (#7627)

## Which problem is this PR solving?
- Towards #7134 and #7135 

## Description of the changes
- This PR adds support for map and slice attributes in ClickHouse
storage. It does so by leveraging the new API added in
open-telemetry/opentelemetry-collector#13945 to
Marshal the attributes to JSON and then storing them as Base64 encoded
strings

## How was this change tested?
- CI / unit tests 

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [x] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `npm run lint` and `npm run test`

---------

Signed-off-by: Mahad Zaryab <[email protected]>

Author: mahadzaryab1

internal/storage/v2/clickhouse/tracestore/assert_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.opentelemetry.io/collector/pdata/xpdata"
 
 	"github.com/jaegertracing/jaeger/internal/jptrace"
 	"github.com/jaegertracing/jaeger/internal/storage/v2/clickhouse/tracestore/dbmodel"
@@ -126,6 +127,24 @@ func requireComplexAttrs(t *testing.T, expectedKeys []string, expectedVals []str
 			decoded, err := base64.StdEncoding.DecodeString(expectedVals[i])
 			require.NoError(t, err)
 			require.Equal(t, decoded, val.Bytes().AsRaw())
+		case strings.HasPrefix(k, "@map@"):
+			key := strings.TrimPrefix(expectedKeys[i], "@map@")
+			val, ok := attrs.Get(key)
+			require.True(t, ok)
+
+			m := &xpdata.JSONUnmarshaler{}
+			expectedVal, err := m.UnmarshalValue([]byte(expectedVals[i]))
+			require.NoError(t, err)
+			require.True(t, expectedVal.Map().Equal(val.Map()))
+		case strings.HasPrefix(k, "@slice@"):
+			key := strings.TrimPrefix(expectedKeys[i], "@slice@")
+			val, ok := attrs.Get(key)
+			require.True(t, ok)
+
+			m := &xpdata.JSONUnmarshaler{}
+			expectedVal, err := m.UnmarshalValue([]byte(expectedVals[i]))
+			require.NoError(t, err)
+			require.True(t, expectedVal.Slice().Equal(val.Slice()))
 		default:
 			t.Fatalf("unsupported complex attribute key: %s", k)
 		}

internal/storage/v2/clickhouse/tracestore/dbmodel/dbmodel_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.opentelemetry.io/collector/pdata/xpdata"
 
 	"github.com/jaegertracing/jaeger/internal/telemetry/otelsemconv"
 )
@@ -32,7 +33,7 @@ func TestRoundTrip(t *testing.T) {
 	})
 
 	t.Run("FromRow->ToRow", func(t *testing.T) {
-		spanRow := createTestSpanRow(now, duration)
+		spanRow := createTestSpanRow(t, now, duration)
 
 		trace := FromRow(spanRow)
 		rs := trace.ResourceSpans().At(0).Resource()
@@ -114,10 +115,27 @@ func addTestAttributes(attrs pcommon.Map) {
 	attrs.PutInt("int_attr", 42)
 	attrs.PutStr("string_attr", "string_value")
 	attrs.PutEmptyBytes("bytes_attr").FromRaw([]byte("bytes_value"))
+	attrs.PutEmptyMap("map_attr").FromRaw(map[string]any{"key": "value"})
+	attrs.PutEmptySlice("slice_attr").FromRaw([]any{1, 2, 3})
 }
 
-func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
+func createTestSpanRow(t *testing.T, now time.Time, duration time.Duration) *SpanRow {
+	t.Helper()
 	encodedBytes := base64.StdEncoding.EncodeToString([]byte("bytes_value"))
+
+	vm := pcommon.NewValueMap()
+	vm.Map().PutStr("key", "value")
+	m := &xpdata.JSONMarshaler{}
+	vmJSON, err := m.MarshalValue(vm)
+	require.NoError(t, err)
+
+	vs := pcommon.NewValueSlice()
+	vs.Slice().AppendEmpty().SetInt(1)
+	vs.Slice().AppendEmpty().SetInt(2)
+	vs.Slice().AppendEmpty().SetInt(3)
+	vsJSON, err := m.MarshalValue(vs)
+	require.NoError(t, err)
+
 	return &SpanRow{
 		ID:            "0000000000000001",
 		TraceID:       "00000000000000000000000000000001",
@@ -138,8 +156,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
 			IntValues:     []int64{42},
 			StrKeys:       []string{"string_attr"},
 			StrValues:     []string{"string_value"},
-			ComplexKeys:   []string{"@bytes@bytes_attr"},
-			ComplexValues: []string{encodedBytes},
+			ComplexKeys:   []string{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"},
+			ComplexValues: []string{encodedBytes, string(vmJSON), string(vsJSON)},
 		},
 		EventNames:      []string{"test-event"},
 		EventTimestamps: []time.Time{now},
@@ -152,8 +170,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
 			IntValues:     [][]int64{{42}},
 			StrKeys:       [][]string{{"string_attr"}},
 			StrValues:     [][]string{{"string_value"}},
-			ComplexKeys:   [][]string{{"@bytes@bytes_attr"}},
-			ComplexValues: [][]string{{encodedBytes}},
+			ComplexKeys:   [][]string{{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"}},
+			ComplexValues: [][]string{{encodedBytes, string(vmJSON), string(vsJSON)}},
 		},
 		LinkTraceIDs:    []string{"00000000000000000000000000000003"},
 		LinkSpanIDs:     []string{"0000000000000004"},
@@ -167,8 +185,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
 			IntValues:     [][]int64{{42}},
 			StrKeys:       [][]string{{"string_attr"}},
 			StrValues:     [][]string{{"string_value"}},
-			ComplexKeys:   [][]string{{"@bytes@bytes_attr"}},
-			ComplexValues: [][]string{{encodedBytes}},
+			ComplexKeys:   [][]string{{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"}},
+			ComplexValues: [][]string{{encodedBytes, string(vmJSON), string(vsJSON)}},
 		},
 		ServiceName: "test-service",
 		ResourceAttributes: Attributes{
@@ -180,8 +198,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
 			IntValues:     []int64{42},
 			StrKeys:       []string{"service.name", "string_attr"},
 			StrValues:     []string{"test-service", "string_value"},
-			ComplexKeys:   []string{"@bytes@bytes_attr"},
-			ComplexValues: []string{encodedBytes},
+			ComplexKeys:   []string{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"},
+			ComplexValues: []string{encodedBytes, string(vmJSON), string(vsJSON)},
 		},
 		ScopeName:    "test-scope",
 		ScopeVersion: "v1.0.0",
@@ -194,8 +212,8 @@ func createTestSpanRow(now time.Time, duration time.Duration) *SpanRow {
 			IntValues:     []int64{42},
 			StrKeys:       []string{"string_attr"},
 			StrValues:     []string{"string_value"},
-			ComplexKeys:   []string{"@bytes@bytes_attr"},
-			ComplexValues: []string{encodedBytes},
+			ComplexKeys:   []string{"@bytes@bytes_attr", "@map@map_attr", "@slice@slice_attr"},
+			ComplexValues: []string{encodedBytes, string(vmJSON), string(vsJSON)},
 		},
 	}
 }

internal/storage/v2/clickhouse/tracestore/dbmodel/from.go

@@ -12,6 +12,7 @@ import (
 
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/ptrace"
+	"go.opentelemetry.io/collector/pdata/xpdata"
 
 	"github.com/jaegertracing/jaeger/internal/jptrace"
 	"github.com/jaegertracing/jaeger/internal/telemetry/otelsemconv"
@@ -169,14 +170,51 @@ func putAttributes(
 		attrs.PutStr(storedAttrs.StrKeys[i], storedAttrs.StrValues[i])
 	}
 	for i := 0; i < len(storedAttrs.ComplexKeys); i++ {
-		if strings.HasPrefix(storedAttrs.ComplexKeys[i], "@bytes@") {
+		switch {
+		case strings.HasPrefix(storedAttrs.ComplexKeys[i], "@bytes@"):
 			decoded, err := base64.StdEncoding.DecodeString(storedAttrs.ComplexValues[i])
 			if err != nil {
 				jptrace.AddWarnings(spanForWarnings, fmt.Sprintf("failed to decode bytes attribute %q: %s", storedAttrs.ComplexKeys[i], err.Error()))
 				continue
 			}
 			k := strings.TrimPrefix(storedAttrs.ComplexKeys[i], "@bytes@")
 			attrs.PutEmptyBytes(k).FromRaw(decoded)
+		case strings.HasPrefix(storedAttrs.ComplexKeys[i], "@slice@"):
+			k := strings.TrimPrefix(storedAttrs.ComplexKeys[i], "@slice@")
+			m := &xpdata.JSONUnmarshaler{}
+			val, err := m.UnmarshalValue([]byte(storedAttrs.ComplexValues[i]))
+			if err != nil {
+				jptrace.AddWarnings(
+					spanForWarnings,
+					fmt.Sprintf(
+						"failed to unmarshal slice attribute %q: %s",
+						storedAttrs.ComplexKeys[i],
+						err.Error(),
+					),
+				)
+				continue
+			}
+			attrs.PutEmptySlice(k).FromRaw(val.Slice().AsRaw())
+		case strings.HasPrefix(storedAttrs.ComplexKeys[i], "@map@"):
+			k := strings.TrimPrefix(storedAttrs.ComplexKeys[i], "@map@")
+			m := &xpdata.JSONUnmarshaler{}
+			val, err := m.UnmarshalValue([]byte(storedAttrs.ComplexValues[i]))
+			if err != nil {
+				jptrace.AddWarnings(
+					spanForWarnings,
+					fmt.Sprintf("failed to unmarshal map attribute %q: %s",
+						storedAttrs.ComplexKeys[i],
+						err.Error(),
+					),
+				)
+				continue
+			}
+			attrs.PutEmptyMap(k).FromRaw(val.Map().AsRaw())
+		default:
+			jptrace.AddWarnings(
+				spanForWarnings,
+				fmt.Sprintf("unsupported complex attribute key: %q", storedAttrs.ComplexKeys[i]),
+			)
 		}
 	}
 }

internal/storage/v2/clickhouse/tracestore/dbmodel/from_test.go

@@ -18,7 +18,7 @@ func TestFromRow(t *testing.T) {
 	now := time.Now().UTC()
 	duration := 2 * time.Second
 
-	spanRow := createTestSpanRow(now, duration)
+	spanRow := createTestSpanRow(t, now, duration)
 
 	expected := createTestTrace(now, duration)
 
@@ -89,23 +89,55 @@ func TestFromRow_DecodeID(t *testing.T) {
 }
 
 func TestPutAttributes_Warnings(t *testing.T) {
-	t.Run("bytes attribute with invalid base64", func(t *testing.T) {
-		span := ptrace.NewSpan()
-		attributes := pcommon.NewMap()
+	tests := []struct {
+		name                 string
+		complexKeys          []string
+		complexValues        []string
+		expectedWarnContains string
+	}{
+		{
+			name:                 "bytes attribute with invalid base64",
+			complexKeys:          []string{"@bytes@bytes-key"},
+			complexValues:        []string{"invalid-base64"},
+			expectedWarnContains: "failed to decode bytes attribute \"@bytes@bytes-key\"",
+		},
+		{
+			name:                 "failed to unmarshal slice attribute",
+			complexKeys:          []string{"@slice@slice-key"},
+			complexValues:        []string{"notjson"},
+			expectedWarnContains: "failed to unmarshal slice attribute \"@slice@slice-key\"",
+		},
+		{
+			name:                 "failed to unmarshal map attribute",
+			complexKeys:          []string{"@map@map-key"},
+			complexValues:        []string{"notjson"},
+			expectedWarnContains: "failed to unmarshal map attribute \"@map@map-key\"",
+		},
+		{
+			name:                 "unsupported complex attribute key",
+			complexKeys:          []string{"unsupported"},
+			complexValues:        []string{"{\"kvlistValue\":{\"values\":[{\"key\":\"key\",\"value\":{\"stringValue\":\"value\"}}]}}"},
+			expectedWarnContains: "unsupported complex attribute key: \"unsupported\"",
+		},
+	}
 
-		putAttributes(
-			attributes,
-			&Attributes{
-				ComplexKeys:   []string{"@bytes@bytes-key"},
-				ComplexValues: []string{"invalid-base64"},
-			},
-			span,
-		)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			span := ptrace.NewSpan()
+			attributes := pcommon.NewMap()
+
+			putAttributes(
+				attributes,
+				&Attributes{
+					ComplexKeys:   tt.complexKeys,
+					ComplexValues: tt.complexValues,
+				},
+				span,
+			)
 
-		_, ok := attributes.Get("bytes-key")
-		require.False(t, ok)
-		warnings := jptrace.GetWarnings(span)
-		require.Len(t, warnings, 1)
-		require.Contains(t, warnings[0], "failed to decode bytes attribute \"@bytes@bytes-key\"")
-	})
+			warnings := jptrace.GetWarnings(span)
+			require.Len(t, warnings, 1)
+			require.Contains(t, warnings[0], tt.expectedWarnContains)
+		})
+	}
 }

internal/storage/v2/clickhouse/tracestore/dbmodel/spanrow.go

@@ -9,16 +9,22 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 )
 
-// SpanRow represents a single row in the ClickHouse `spans` table.
+// SpanRow represents a single record in the ClickHouse `spans` table.
 //
-// Complex attributes are attributes that are not of a primitive type and hence need special handling.
-// The following OTLP types are stored in the complex attributes fields:
-//   - AnyValue_BytesValue: This OTLP type is stored as a base64-encoded string. The key
-//     for this type will begin with `@bytes@`.
-//   - AnyValue_ArrayValue: This OTLP type is stored as a JSON-encoded string.
-//     The key for this type will begin with `@array@`.
-//   - AnyValue_KVListValue: This OTLP type is stored as a JSON-encoded string.
-//     The key for this type will begin with `@kvlist@`.
+// Complex attributes are non-primitive OTLP types that require special serialization
+// before being stored. These types are encoded as follows:
+//
+//   - pcommon.ValueTypeBytes:
+//     Represents raw byte data. The value is Base64-encoded and stored as a string.
+//     Keys for this type are prefixed with `@bytes@`.
+//
+//   - pcommon.ValueTypeSlice:
+//     Represents an OTLP slice (array). The value is serialized to JSON and stored
+//     as a string. Keys for this type are prefixed with `@slice@`.
+//
+//   - pcommon.ValueTypeMap:
+//     Represents an OTLP map. The value is serialized to JSON and stored
+//     as a string. Keys for this type are prefixed with `@map@`.
 type SpanRow struct {
 	// --- Span ---
 	ID              string