diff --git a/internal/storage/v2/clickhouse/sql/create_spans_table.sql b/internal/storage/v2/clickhouse/sql/create_spans_table.sql index c4f16c9be67..7aba29e1c2d 100644 --- a/internal/storage/v2/clickhouse/sql/create_spans_table.sql +++ b/internal/storage/v2/clickhouse/sql/create_spans_table.sql @@ -41,5 +41,10 @@ CREATE TABLE resource_str_attributes Nested (key String, value String), resource_complex_attributes Nested (key String, value String), scope_name String, - scope_version String + scope_version String, + scope_bool_attributes Nested (key String, value Bool), + scope_double_attributes Nested (key String, value Float64), + scope_int_attributes Nested (key String, value Int64), + scope_str_attributes Nested (key String, value String), + scope_complex_attributes Nested (key String, value String), ) ENGINE = MergeTree PRIMARY KEY (trace_id) \ No newline at end of file diff --git a/internal/storage/v2/clickhouse/sql/queries.go b/internal/storage/v2/clickhouse/sql/queries.go index 74092498d86..2dc05de21b6 100644 --- a/internal/storage/v2/clickhouse/sql/queries.go +++ b/internal/storage/v2/clickhouse/sql/queries.go @@ -56,6 +56,16 @@ INSERT INTO resource_complex_attributes.value, scope_name, scope_version, + scope_bool_attributes.key, + scope_bool_attributes.value, + scope_double_attributes.key, + scope_double_attributes.value, + scope_int_attributes.key, + scope_int_attributes.value, + scope_str_attributes.key, + scope_str_attributes.value, + scope_complex_attributes.key, + scope_complex_attributes.value ) VALUES ( @@ -106,6 +116,16 @@ VALUES ?, ?, ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, ? ) ` @@ -169,7 +189,17 @@ SELECT resource_complex_attributes.key, resource_complex_attributes.value, scope_name, - scope_version + scope_version, + scope_bool_attributes.key, + scope_bool_attributes.value, + scope_double_attributes.key, + scope_double_attributes.value, + scope_int_attributes.key, + scope_int_attributes.value, + scope_str_attributes.key, + scope_str_attributes.value, + scope_complex_attributes.key, + scope_complex_attributes.value, FROM spans WHERE diff --git a/internal/storage/v2/clickhouse/tracestore/dbmodel/from.go b/internal/storage/v2/clickhouse/tracestore/dbmodel/from.go index ffaeb02a009..1758261f172 100644 --- a/internal/storage/v2/clickhouse/tracestore/dbmodel/from.go +++ b/internal/storage/v2/clickhouse/tracestore/dbmodel/from.go @@ -35,7 +35,7 @@ func FromRow(storedSpan *SpanRow) ptrace.Traces { rs.CopyTo(resource) scope := scopeSpans.Scope() - sc := convertScope(storedSpan) + sc := convertScope(storedSpan, span) sc.CopyTo(scope) return trace @@ -46,21 +46,21 @@ func convertResource(sr *SpanRow, spanForWarnings ptrace.Span) pcommon.Resource resource.Attributes().PutStr(otelsemconv.ServiceNameKey, sr.ServiceName) putAttributes( resource.Attributes(), + &sr.ResourceAttributes, spanForWarnings, - sr.ResourceAttributes.BoolKeys, sr.ResourceAttributes.BoolValues, - sr.ResourceAttributes.DoubleKeys, sr.ResourceAttributes.DoubleValues, - sr.ResourceAttributes.IntKeys, sr.ResourceAttributes.IntValues, - sr.ResourceAttributes.StrKeys, sr.ResourceAttributes.StrValues, - sr.ResourceAttributes.ComplexKeys, sr.ResourceAttributes.ComplexValues, ) return resource } -func convertScope(s *SpanRow) pcommon.InstrumentationScope { +func convertScope(sr *SpanRow, spanForWarnings ptrace.Span) pcommon.InstrumentationScope { scope := ptrace.NewScopeSpans().Scope() - scope.SetName(s.ScopeName) - scope.SetVersion(s.ScopeVersion) - // TODO: populate attributes + scope.SetName(sr.ScopeName) + scope.SetVersion(sr.ScopeVersion) + putAttributes( + scope.Attributes(), + &sr.ScopeAttributes, + spanForWarnings, + ) return scope } @@ -94,27 +94,15 @@ func convertSpan(sr *SpanRow) (ptrace.Span, error) { putAttributes( span.Attributes(), + &sr.Attributes, span, - sr.Attributes.BoolKeys, sr.Attributes.BoolValues, - sr.Attributes.DoubleKeys, sr.Attributes.DoubleValues, - sr.Attributes.IntKeys, sr.Attributes.IntValues, - sr.Attributes.StrKeys, sr.Attributes.StrValues, - sr.Attributes.ComplexKeys, sr.Attributes.ComplexValues, ) for i, e := range sr.EventNames { event := span.Events().AppendEmpty() event.SetName(e) event.SetTimestamp(pcommon.NewTimestampFromTime(sr.EventTimestamps[i])) - putAttributes( - event.Attributes(), - span, - sr.EventAttributes.BoolKeys[i], sr.EventAttributes.BoolValues[i], - sr.EventAttributes.DoubleKeys[i], sr.EventAttributes.DoubleValues[i], - sr.EventAttributes.IntKeys[i], sr.EventAttributes.IntValues[i], - sr.EventAttributes.StrKeys[i], sr.EventAttributes.StrValues[i], - sr.EventAttributes.ComplexKeys[i], sr.EventAttributes.ComplexValues[i], - ) + putAttributes2D(event.Attributes(), &sr.EventAttributes, i, span) } for i, l := range sr.LinkTraceIDs { @@ -133,49 +121,61 @@ func convertSpan(sr *SpanRow) (ptrace.Span, error) { link.SetSpanID(pcommon.SpanID(spanID)) link.TraceState().FromRaw(sr.LinkTraceStates[i]) - putAttributes( - link.Attributes(), - span, - sr.LinkAttributes.BoolKeys[i], sr.LinkAttributes.BoolValues[i], - sr.LinkAttributes.DoubleKeys[i], sr.LinkAttributes.DoubleValues[i], - sr.LinkAttributes.IntKeys[i], sr.LinkAttributes.IntValues[i], - sr.LinkAttributes.StrKeys[i], sr.LinkAttributes.StrValues[i], - sr.LinkAttributes.ComplexKeys[i], sr.LinkAttributes.ComplexValues[i], - ) + putAttributes2D(link.Attributes(), &sr.LinkAttributes, i, span) } return span, nil } +func putAttributes2D( + attrs pcommon.Map, + storedAttrs *Attributes2D, + idx int, + spanForWarnings ptrace.Span, +) { + putAttributes( + attrs, + &Attributes{ + BoolKeys: storedAttrs.BoolKeys[idx], + BoolValues: storedAttrs.BoolValues[idx], + DoubleKeys: storedAttrs.DoubleKeys[idx], + DoubleValues: storedAttrs.DoubleValues[idx], + IntKeys: storedAttrs.IntKeys[idx], + IntValues: storedAttrs.IntValues[idx], + StrKeys: storedAttrs.StrKeys[idx], + StrValues: storedAttrs.StrValues[idx], + ComplexKeys: storedAttrs.ComplexKeys[idx], + ComplexValues: storedAttrs.ComplexValues[idx], + }, + spanForWarnings, + ) +} + func putAttributes( attrs pcommon.Map, + storedAttrs *Attributes, spanForWarnings ptrace.Span, - boolKeys []string, boolValues []bool, - doubleKeys []string, doubleValues []float64, - intKeys []string, intValues []int64, - strKeys []string, strValues []string, - complexKeys []string, complexValues []string, ) { - for i := 0; i < len(boolKeys); i++ { - attrs.PutBool(boolKeys[i], boolValues[i]) + for i := 0; i < len(storedAttrs.BoolKeys); i++ { + attrs.PutBool(storedAttrs.BoolKeys[i], storedAttrs.BoolValues[i]) } - for i := 0; i < len(doubleKeys); i++ { - attrs.PutDouble(doubleKeys[i], doubleValues[i]) + for i := 0; i < len(storedAttrs.DoubleKeys); i++ { + attrs.PutDouble(storedAttrs.DoubleKeys[i], storedAttrs.DoubleValues[i]) } - for i := 0; i < len(intKeys); i++ { - attrs.PutInt(intKeys[i], intValues[i]) + for i := 0; i < len(storedAttrs.IntKeys); i++ { + attrs.PutInt(storedAttrs.IntKeys[i], storedAttrs.IntValues[i]) } - for i := 0; i < len(strKeys); i++ { - attrs.PutStr(strKeys[i], strValues[i]) + for i := 0; i < len(storedAttrs.StrKeys); i++ { + attrs.PutStr(storedAttrs.StrKeys[i], storedAttrs.StrValues[i]) } - for i := 0; i < len(complexKeys); i++ { - if strings.HasPrefix(complexKeys[i], "@bytes@") { - decoded, err := base64.StdEncoding.DecodeString(complexValues[i]) + for i := 0; i < len(storedAttrs.ComplexKeys); i++ { + if strings.HasPrefix(storedAttrs.ComplexKeys[i], "@bytes@") { + decoded, err := base64.StdEncoding.DecodeString(storedAttrs.ComplexValues[i]) if err != nil { - jptrace.AddWarnings(spanForWarnings, fmt.Sprintf("failed to decode bytes attribute %q: %s", complexKeys[i], err.Error())) + jptrace.AddWarnings(spanForWarnings, fmt.Sprintf("failed to decode bytes attribute %q: %s", storedAttrs.ComplexKeys[i], err.Error())) continue } - k := strings.TrimPrefix(complexKeys[i], "@bytes@") + k := strings.TrimPrefix(storedAttrs.ComplexKeys[i], "@bytes@") attrs.PutEmptyBytes(k).FromRaw(decoded) } } diff --git a/internal/storage/v2/clickhouse/tracestore/dbmodel/from_test.go b/internal/storage/v2/clickhouse/tracestore/dbmodel/from_test.go index d38a33f3e42..716d3801fb5 100644 --- a/internal/storage/v2/clickhouse/tracestore/dbmodel/from_test.go +++ b/internal/storage/v2/clickhouse/tracestore/dbmodel/from_test.go @@ -171,12 +171,11 @@ func TestPutAttributes_Warnings(t *testing.T) { putAttributes( attributes, + &Attributes{ + ComplexKeys: []string{"@bytes@bytes-key"}, + ComplexValues: []string{"invalid-base64"}, + }, span, - nil, nil, - nil, nil, - nil, nil, - nil, nil, - []string{"@bytes@bytes-key"}, []string{"invalid-base64"}, ) _, ok := attributes.Get("bytes-key") diff --git a/internal/storage/v2/clickhouse/tracestore/dbmodel/spanrow.go b/internal/storage/v2/clickhouse/tracestore/dbmodel/spanrow.go index 49864ec7678..134b66802c2 100644 --- a/internal/storage/v2/clickhouse/tracestore/dbmodel/spanrow.go +++ b/internal/storage/v2/clickhouse/tracestore/dbmodel/spanrow.go @@ -45,8 +45,9 @@ type SpanRow struct { ResourceAttributes Attributes // --- Scope --- - ScopeName string - ScopeVersion string + ScopeName string + ScopeVersion string + ScopeAttributes Attributes } type Attributes struct { @@ -136,6 +137,16 @@ func ScanRow(rows driver.Rows) (*SpanRow, error) { &sr.ResourceAttributes.ComplexValues, &sr.ScopeName, &sr.ScopeVersion, + &sr.ScopeAttributes.BoolKeys, + &sr.ScopeAttributes.BoolValues, + &sr.ScopeAttributes.DoubleKeys, + &sr.ScopeAttributes.DoubleValues, + &sr.ScopeAttributes.IntKeys, + &sr.ScopeAttributes.IntValues, + &sr.ScopeAttributes.StrKeys, + &sr.ScopeAttributes.StrValues, + &sr.ScopeAttributes.ComplexKeys, + &sr.ScopeAttributes.ComplexValues, ) if err != nil { return nil, err diff --git a/internal/storage/v2/clickhouse/tracestore/dbmodel/testdata/dbmodel.json b/internal/storage/v2/clickhouse/tracestore/dbmodel/testdata/dbmodel.json index 6bdf8da1d29..0857b66c017 100644 --- a/internal/storage/v2/clickhouse/tracestore/dbmodel/testdata/dbmodel.json +++ b/internal/storage/v2/clickhouse/tracestore/dbmodel/testdata/dbmodel.json @@ -51,6 +51,30 @@ "ComplexValues": [["eyJkYl9jb250ZXh0IjoidXNlcmRiIn0="]] }, "ServiceName": "db-service", + "ResourceAttributes": { + "BoolKeys": ["resource.persistent", "resource.pooled"], + "BoolValues": [true, true], + "DoubleKeys": ["resource.cpu_limit", "resource.memory_limit"], + "DoubleValues": [1.5, 512.0], + "IntKeys": ["resource.instance_id", "resource.max_connections"], + "IntValues": [67890, 100], + "StrKeys": ["service.name", "resource.host", "resource.database_type"], + "StrValues": ["db-service", "db-host-1", "postgresql"], + "ComplexKeys": ["@bytes@resource.config"], + "ComplexValues": ["eyJkYl90eXBlIjoicG9zdGdyZXNxbCJ9"] + }, "ScopeName": "db-scope", - "ScopeVersion": "v1.0.0" + "ScopeVersion": "v1.0.0", + "ScopeAttributes": { + "BoolKeys": ["scope.enabled", "scope.persistent"], + "BoolValues": [true, false], + "DoubleKeys": ["scope.version_number", "scope.priority"], + "DoubleValues": [1.0, 0.8], + "IntKeys": ["scope.instance_count", "scope.max_spans"], + "IntValues": [5, 1000], + "StrKeys": ["scope.environment", "scope.component"], + "StrValues": ["production", "database"], + "ComplexKeys": ["@bytes@scope.metadata"], + "ComplexValues": ["eyJzY29wZV90eXBlIjoiZGF0YWJhc2UifQ=="] + } } diff --git a/internal/storage/v2/clickhouse/tracestore/dbmodel/to.go b/internal/storage/v2/clickhouse/tracestore/dbmodel/to.go index eb84a52bc93..ae2dbc1e45c 100644 --- a/internal/storage/v2/clickhouse/tracestore/dbmodel/to.go +++ b/internal/storage/v2/clickhouse/tracestore/dbmodel/to.go @@ -46,6 +46,7 @@ func ToRow( sr.appendLink(link) } appendAttributes(&sr.ResourceAttributes, resource.Attributes()) + appendAttributes(&sr.ScopeAttributes, scope.Attributes()) return sr } diff --git a/internal/storage/v2/clickhouse/tracestore/dbmodel/to_test.go b/internal/storage/v2/clickhouse/tracestore/dbmodel/to_test.go index 69529a567cf..438823f0d6b 100644 --- a/internal/storage/v2/clickhouse/tracestore/dbmodel/to_test.go +++ b/internal/storage/v2/clickhouse/tracestore/dbmodel/to_test.go @@ -40,6 +40,7 @@ func createTestScope() pcommon.InstrumentationScope { sc := pcommon.NewInstrumentationScope() sc.SetName("test-scope") sc.SetVersion("v1.0.0") + addTestAttributes(sc.Attributes()) return sc } @@ -155,5 +156,17 @@ func createExpectedSpanRow(now time.Time, duration time.Duration) *SpanRow { }, ScopeName: "test-scope", ScopeVersion: "v1.0.0", + ScopeAttributes: Attributes{ + BoolKeys: []string{"bool_attr"}, + BoolValues: []bool{true}, + DoubleKeys: []string{"double_attr"}, + DoubleValues: []float64{3.14}, + IntKeys: []string{"int_attr"}, + IntValues: []int64{42}, + StrKeys: []string{"string_attr"}, + StrValues: []string{"string_value"}, + ComplexKeys: []string{"@bytes@bytes_attr"}, + ComplexValues: []string{encodedBytes}, + }, } } diff --git a/internal/storage/v2/clickhouse/tracestore/reader_test.go b/internal/storage/v2/clickhouse/tracestore/reader_test.go index 3a2f0b35802..012e2ac466b 100644 --- a/internal/storage/v2/clickhouse/tracestore/reader_test.go +++ b/internal/storage/v2/clickhouse/tracestore/reader_test.go @@ -26,8 +26,8 @@ func scanSpanRowFn() func(dest any, src *dbmodel.SpanRow) error { if !ok { return fmt.Errorf("expected []any for dest, got %T", dest) } - if len(ptrs) != 58 { - return fmt.Errorf("expected 58 destination arguments, got %d", len(ptrs)) + if len(ptrs) != 68 { + return fmt.Errorf("expected 68 destination arguments, got %d", len(ptrs)) } values := []any{ @@ -89,6 +89,16 @@ func scanSpanRowFn() func(dest any, src *dbmodel.SpanRow) error { &src.ResourceAttributes.ComplexValues, &src.ScopeName, &src.ScopeVersion, + &src.ScopeAttributes.BoolKeys, + &src.ScopeAttributes.BoolValues, + &src.ScopeAttributes.DoubleKeys, + &src.ScopeAttributes.DoubleValues, + &src.ScopeAttributes.IntKeys, + &src.ScopeAttributes.IntValues, + &src.ScopeAttributes.StrKeys, + &src.ScopeAttributes.StrValues, + &src.ScopeAttributes.ComplexKeys, + &src.ScopeAttributes.ComplexValues, } for i := range ptrs { diff --git a/internal/storage/v2/clickhouse/tracestore/spans_test.go b/internal/storage/v2/clickhouse/tracestore/spans_test.go index ed9f30bda43..aed00bb8a89 100644 --- a/internal/storage/v2/clickhouse/tracestore/spans_test.go +++ b/internal/storage/v2/clickhouse/tracestore/spans_test.go @@ -82,6 +82,18 @@ var singleSpan = []*dbmodel.SpanRow{ }, ScopeName: "auth-scope", ScopeVersion: "v1.0.0", + ScopeAttributes: dbmodel.Attributes{ + BoolKeys: []string{"scope.enabled", "scope.persistent"}, + BoolValues: []bool{true, false}, + DoubleKeys: []string{"scope.version_number", "scope.priority"}, + DoubleValues: []float64{1.0, 0.8}, + IntKeys: []string{"scope.instance_count", "scope.max_spans"}, + IntValues: []int64{5, 1000}, + StrKeys: []string{"scope.environment", "scope.component"}, + StrValues: []string{"production", "auth"}, + ComplexKeys: []string{"@bytes@scope.metadata"}, + ComplexValues: []string{"eyJzY29wZV90eXBlIjoiYXV0aGVudGljYXRpb24ifQ=="}, + }, }, } @@ -152,6 +164,18 @@ var multipleSpans = []*dbmodel.SpanRow{ }, ScopeName: "auth-scope", ScopeVersion: "v1.0.0", + ScopeAttributes: dbmodel.Attributes{ + BoolKeys: []string{"scope.enabled", "scope.persistent"}, + BoolValues: []bool{true, false}, + DoubleKeys: []string{"scope.version_number", "scope.priority"}, + DoubleValues: []float64{1.0, 0.8}, + IntKeys: []string{"scope.instance_count", "scope.max_spans"}, + IntValues: []int64{5, 1000}, + StrKeys: []string{"scope.environment", "scope.component"}, + StrValues: []string{"production", "auth"}, + ComplexKeys: []string{"@bytes@scope.metadata"}, + ComplexValues: []string{"eyJzY29wZV90eXBlIjoiYXV0aGVudGljYXRpb24ifQ=="}, + }, }, { ID: "0000000000000003", @@ -220,5 +244,17 @@ var multipleSpans = []*dbmodel.SpanRow{ }, ScopeName: "db-scope", ScopeVersion: "v1.0.0", + ScopeAttributes: dbmodel.Attributes{ + BoolKeys: []string{"scope.enabled", "scope.persistent"}, + BoolValues: []bool{true, false}, + DoubleKeys: []string{"scope.version_number", "scope.priority"}, + DoubleValues: []float64{1.0, 0.8}, + IntKeys: []string{"scope.instance_count", "scope.max_spans"}, + IntValues: []int64{5, 1000}, + StrKeys: []string{"scope.environment", "scope.component"}, + StrValues: []string{"production", "database"}, + ComplexKeys: []string{"@bytes@scope.metadata"}, + ComplexValues: []string{"eyJzY29wZV90eXBlIjoiZGF0YWJhc2UifQ=="}, + }, }, } diff --git a/internal/storage/v2/clickhouse/tracestore/writer.go b/internal/storage/v2/clickhouse/tracestore/writer.go index 4403a24fc86..aac96ffd493 100644 --- a/internal/storage/v2/clickhouse/tracestore/writer.go +++ b/internal/storage/v2/clickhouse/tracestore/writer.go @@ -86,6 +86,16 @@ func (w *Writer) WriteTraces(ctx context.Context, td ptrace.Traces) error { sr.ResourceAttributes.ComplexValues, sr.ScopeName, sr.ScopeVersion, + sr.ScopeAttributes.BoolKeys, + sr.ScopeAttributes.BoolValues, + sr.ScopeAttributes.DoubleKeys, + sr.ScopeAttributes.DoubleValues, + sr.ScopeAttributes.IntKeys, + sr.ScopeAttributes.IntValues, + sr.ScopeAttributes.StrKeys, + sr.ScopeAttributes.StrValues, + sr.ScopeAttributes.ComplexKeys, + sr.ScopeAttributes.ComplexValues, ) if err != nil { return fmt.Errorf("failed to append span to batch: %w", err) diff --git a/internal/storage/v2/clickhouse/tracestore/writer_test.go b/internal/storage/v2/clickhouse/tracestore/writer_test.go index 7831a8398b8..b088647cbd4 100644 --- a/internal/storage/v2/clickhouse/tracestore/writer_test.go +++ b/internal/storage/v2/clickhouse/tracestore/writer_test.go @@ -126,6 +126,16 @@ func TestWriter_Success(t *testing.T) { require.Equal(t, expected.ResourceAttributes.ComplexValues, row[45]) // Resource complex attribute values require.Equal(t, expected.ScopeName, row[46]) // Scope name require.Equal(t, expected.ScopeVersion, row[47]) // Scope version + require.Equal(t, expected.ScopeAttributes.BoolKeys, row[48]) // Scope bool attribute keys + require.Equal(t, expected.ScopeAttributes.BoolValues, row[49]) // Scope bool attribute values + require.Equal(t, expected.ScopeAttributes.DoubleKeys, row[50]) // Scope double attribute keys + require.Equal(t, expected.ScopeAttributes.DoubleValues, row[51]) // Scope double attribute values + require.Equal(t, expected.ScopeAttributes.IntKeys, row[52]) // Scope int attribute keys + require.Equal(t, expected.ScopeAttributes.IntValues, row[53]) // Scope int attribute values + require.Equal(t, expected.ScopeAttributes.StrKeys, row[54]) // Scope str attribute keys + require.Equal(t, expected.ScopeAttributes.StrValues, row[55]) // Scope str attribute values + require.Equal(t, expected.ScopeAttributes.ComplexKeys, row[56]) // Scope complex attribute keys + require.Equal(t, expected.ScopeAttributes.ComplexValues, row[57]) // Scope complex attribute values } }