Skip to content

Commit 02f7291

Browse files
shengxilshengxil
authored
AWS X-Ray exporter: Support and default to convert span attributes to X-Ray metadata (#808)
* AWS X-Ray exporter: indexed attributes settings / default to metadata * Fix a spacing issue
1 parent 618cb4e commit 02f7291

File tree

7 files changed

+126
-38
lines changed

7 files changed

+126
-38
lines changed

‎exporter/awsxrayexporter/awsxray.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ func NewTraceExporter(config configmodels.Exporter, logger *zap.Logger, cn connA
6969
continue
7070
}
7171

72-
document, localErr := translator.MakeSegmentDocumentString(span, resource)
72+
document, localErr := translator.MakeSegmentDocumentString(span, resource,
73+
config.(*Config).IndexedAttributes, config.(*Config).IndexAllAttributes)
7374
if localErr != nil {
7475
totalDroppedSpans++
7576
continue

‎exporter/awsxrayexporter/config.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,11 @@ type Config struct {
3939
ResourceARN string `mapstructure:"resource_arn"`
4040
// IAM role to upload segments to a different account.
4141
RoleARN string `mapstructure:"role_arn"`
42+
// By default, OpenTelemetry attributes are converted to X-Ray metadata, which are not indexed.
43+
// Specify a list of attribute names to be converted to X-Ray annotations instead, which will be indexed.
44+
// See annotation vs. metadata: https://docs.aws.amazon.com/xray/latest/devguide/xray-concepts.html#xray-concepts-annotations
45+
IndexedAttributes []string `mapstructure:"indexed_attributes"`
46+
// Set to true to convert all OpenTelemetry attributes to X-Ray annotation (indexed) ignoring the IndexedAttributes option.
47+
// Default value: false
48+
IndexAllAttributes bool `mapstructure:"index_all_attributes"`
4249
}

‎exporter/awsxrayexporter/config_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,5 +57,7 @@ func TestLoadConfig(t *testing.T) {
5757
LocalMode: false,
5858
ResourceARN: "arn:aws:ec2:us-east1:123456789:instance/i-293hiuhe0u",
5959
RoleARN: "arn:aws:iam::123456789:role/monitoring-EKS-NodeInstanceRole",
60+
IndexedAttributes: []string{"indexed_attr_0", "indexed_attr_1"},
61+
IndexAllAttributes: false,
6062
})
6163
}

‎exporter/awsxrayexporter/testdata/config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ exporters:
1010
region: eu-west-1
1111
resource_arn: "arn:aws:ec2:us-east1:123456789:instance/i-293hiuhe0u"
1212
role_arn: "arn:aws:iam::123456789:role/monitoring-EKS-NodeInstanceRole"
13+
indexed_attributes: ["indexed_attr_0", "indexed_attr_1"]
1314

1415
service:
1516
pipelines:

‎exporter/awsxrayexporter/translator/segment.go‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,9 @@ var (
6565
writers = newWriterPool(2048)
6666
)
6767

68-
// MakeSegmentDocumentString converts an OpenCensus Span to an X-Ray Segment and then serialzies to JSON
69-
func MakeSegmentDocumentString(span pdata.Span, resource pdata.Resource) (string, error) {
70-
segment := MakeSegment(span, resource)
68+
// MakeSegmentDocumentString converts an OpenTelemetry Span to an X-Ray Segment and then serialzies to JSON
69+
func MakeSegmentDocumentString(span pdata.Span, resource pdata.Resource, indexedAttrs []string, indexAllAttrs bool) (string, error) {
70+
segment := MakeSegment(span, resource, indexedAttrs, indexAllAttrs)
7171
w := writers.borrow()
7272
if err := w.Encode(segment); err != nil {
7373
return "", err
@@ -77,8 +77,8 @@ func MakeSegmentDocumentString(span pdata.Span, resource pdata.Resource) (string
7777
return jsonStr, nil
7878
}
7979

80-
// MakeSegment converts an OpenCensus Span to an X-Ray Segment
81-
func MakeSegment(span pdata.Span, resource pdata.Resource) awsxray.Segment {
80+
// MakeSegment converts an OpenTelemetry Span to an X-Ray Segment
81+
func MakeSegment(span pdata.Span, resource pdata.Resource, indexedAttrs []string, indexAllAttrs bool) awsxray.Segment {
8282
var (
8383
traceID = convertToAmazonTraceID(span.TraceID())
8484
startTime = timestampToFloatSeconds(span.StartTime())
@@ -90,7 +90,7 @@ func MakeSegment(span pdata.Span, resource pdata.Resource) awsxray.Segment {
9090
awsfiltered, aws = makeAws(causefiltered, resource)
9191
service = makeService(resource)
9292
sqlfiltered, sql = makeSQL(awsfiltered)
93-
user, annotations = makeAnnotations(sqlfiltered)
93+
user, annotations, metadata = makeXRayAttributes(sqlfiltered, indexedAttrs, indexAllAttrs)
9494
name string
9595
namespace string
9696
segmentType string
@@ -185,7 +185,7 @@ func MakeSegment(span pdata.Span, resource pdata.Resource) awsxray.Segment {
185185
Service: service,
186186
SQL: sql,
187187
Annotations: annotations,
188-
Metadata: nil,
188+
Metadata: metadata,
189189
Type: awsP.String(segmentType),
190190
}
191191
}
@@ -290,30 +290,50 @@ func timestampToFloatSeconds(ts pdata.TimestampUnixNano) float64 {
290290
return float64(ts) / float64(time.Second)
291291
}
292292

293-
func sanitizeAndTransferAnnotations(dest map[string]interface{}, src map[string]string) {
294-
for key, value := range src {
295-
key = fixAnnotationKey(key)
296-
dest[key] = value
297-
}
298-
}
299-
300-
func makeAnnotations(attributes map[string]string) (string, map[string]interface{}) {
293+
func makeXRayAttributes(attributes map[string]string, indexedAttrs []string, indexAllAttrs bool) (
294+
string, map[string]interface{}, map[string]map[string]interface{}) {
301295
var (
302-
result = map[string]interface{}{}
303-
user string
296+
annotations = map[string]interface{}{}
297+
metadata = map[string]map[string]interface{}{}
298+
user string
304299
)
305300
delete(attributes, semconventions.AttributeComponent)
306301
userid, ok := attributes[semconventions.AttributeEnduserID]
307302
if ok {
308303
user = userid
309304
delete(attributes, semconventions.AttributeEnduserID)
310305
}
311-
sanitizeAndTransferAnnotations(result, attributes)
312306

313-
if len(result) == 0 {
314-
return user, nil
307+
if len(attributes) == 0 {
308+
return user, nil, nil
309+
}
310+
311+
if indexAllAttrs {
312+
for key, value := range attributes {
313+
key = fixAnnotationKey(key)
314+
annotations[key] = value
315+
}
316+
} else {
317+
defaultMetadata := map[string]interface{}{}
318+
indexedKeys := map[string]interface{}{}
319+
for _, name := range indexedAttrs {
320+
indexedKeys[name] = true
321+
}
322+
323+
for key, value := range attributes {
324+
if _, ok := indexedKeys[key]; ok {
325+
key = fixAnnotationKey(key)
326+
annotations[key] = value
327+
} else {
328+
defaultMetadata[key] = value
329+
}
330+
}
331+
if len(defaultMetadata) > 0 {
332+
metadata["default"] = defaultMetadata
333+
}
315334
}
316-
return user, result
335+
336+
return user, annotations, metadata
317337
}
318338

319339
// fixSegmentName removes any invalid characters from the span name. AWS X-Ray defines

‎exporter/awsxrayexporter/translator/segment_test.go‎

Lines changed: 70 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,12 @@ func TestClientSpanWithAwsSdkClient(t *testing.T) {
5151
resource := constructDefaultResource()
5252
span := constructClientSpan(parentSpanID, spanName, 0, "OK", attributes)
5353

54-
segment := MakeSegment(span, resource)
54+
segment := MakeSegment(span, resource, nil, false)
5555
assert.Equal(t, "DynamoDB", *segment.Name)
5656
assert.Equal(t, "aws", *segment.Namespace)
5757
assert.Equal(t, "subsegment", *segment.Type)
5858

59-
jsonStr, err := MakeSegmentDocumentString(span, resource)
59+
jsonStr, err := MakeSegmentDocumentString(span, resource, nil, false)
6060

6161
assert.NotNil(t, jsonStr)
6262
assert.Nil(t, err)
@@ -82,7 +82,7 @@ func TestClientSpanWithPeerService(t *testing.T) {
8282
resource := constructDefaultResource()
8383
span := constructClientSpan(parentSpanID, spanName, 0, "OK", attributes)
8484

85-
segment := MakeSegment(span, resource)
85+
segment := MakeSegment(span, resource, nil, false)
8686
assert.Equal(t, "cats-table", *segment.Name)
8787
}
8888

@@ -106,7 +106,7 @@ func TestServerSpanWithInternalServerError(t *testing.T) {
106106
timeEvents := constructTimedEventsWithSentMessageEvent(span.StartTime())
107107
timeEvents.CopyTo(span.Events())
108108

109-
segment := MakeSegment(span, resource)
109+
segment := MakeSegment(span, resource, nil, false)
110110

111111
assert.NotNil(t, segment)
112112
assert.NotNil(t, segment.Cause)
@@ -130,7 +130,7 @@ func TestServerSpanNoParentId(t *testing.T) {
130130
resource := constructDefaultResource()
131131
span := constructServerSpan(parentSpanID, spanName, 0, "OK", nil)
132132

133-
segment := MakeSegment(span, resource)
133+
segment := MakeSegment(span, resource, nil, false)
134134

135135
assert.Empty(t, segment.ParentID)
136136
}
@@ -145,7 +145,7 @@ func TestSpanWithNoStatus(t *testing.T) {
145145
span.SetStartTime(pdata.TimestampUnixNano(time.Now().UnixNano()))
146146
span.SetEndTime(pdata.TimestampUnixNano(time.Now().Add(10).UnixNano()))
147147

148-
segment := MakeSegment(span, pdata.NewResource())
148+
segment := MakeSegment(span, pdata.NewResource(), nil, false)
149149
assert.NotNil(t, segment)
150150
}
151151

@@ -165,13 +165,15 @@ func TestClientSpanWithDbComponent(t *testing.T) {
165165
resource := constructDefaultResource()
166166
span := constructClientSpan(parentSpanID, spanName, 0, "OK", attributes)
167167

168-
segment := MakeSegment(span, resource)
168+
segment := MakeSegment(span, resource, nil, false)
169169

170170
assert.NotNil(t, segment)
171171
assert.NotNil(t, segment.SQL)
172172
assert.NotNil(t, segment.Service)
173173
assert.NotNil(t, segment.AWS)
174-
assert.NotNil(t, segment.Annotations)
174+
assert.NotNil(t, segment.Metadata)
175+
assert.Equal(t, 0, len(segment.Annotations))
176+
assert.Equal(t, enterpriseAppID, segment.Metadata["default"]["enterprise.app.id"])
175177
assert.Nil(t, segment.Cause)
176178
assert.Nil(t, segment.HTTP)
177179
assert.Equal(t, "[email protected]", *segment.Name)
@@ -185,6 +187,7 @@ func TestClientSpanWithDbComponent(t *testing.T) {
185187
}
186188
jsonStr := w.String()
187189
testWriters.release(w)
190+
fmt.Println(jsonStr)
188191
assert.True(t, strings.Contains(jsonStr, spanName))
189192
assert.True(t, strings.Contains(jsonStr, enterpriseAppID))
190193
}
@@ -203,7 +206,7 @@ func TestClientSpanWithHttpHost(t *testing.T) {
203206
resource := constructDefaultResource()
204207
span := constructClientSpan(parentSpanID, spanName, 0, "OK", attributes)
205208

206-
segment := MakeSegment(span, resource)
209+
segment := MakeSegment(span, resource, nil, false)
207210

208211
assert.NotNil(t, segment)
209212
assert.Equal(t, "foo.com", *segment.Name)
@@ -222,7 +225,7 @@ func TestClientSpanWithoutHttpHost(t *testing.T) {
222225
resource := constructDefaultResource()
223226
span := constructClientSpan(parentSpanID, spanName, 0, "OK", attributes)
224227

225-
segment := MakeSegment(span, resource)
228+
segment := MakeSegment(span, resource, nil, false)
226229

227230
assert.NotNil(t, segment)
228231
assert.Equal(t, "bar.com", *segment.Name)
@@ -242,7 +245,7 @@ func TestClientSpanWithRpcHost(t *testing.T) {
242245
resource := constructDefaultResource()
243246
span := constructClientSpan(parentSpanID, spanName, 0, "OK", attributes)
244247

245-
segment := MakeSegment(span, resource)
248+
segment := MakeSegment(span, resource, nil, false)
246249

247250
assert.NotNil(t, segment)
248251
assert.Equal(t, "com.foo.AnimalService", *segment.Name)
@@ -265,7 +268,7 @@ func TestSpanWithInvalidTraceId(t *testing.T) {
265268
traceID[0] = 0x11
266269
span.SetTraceID(traceID)
267270

268-
jsonStr, err := MakeSegmentDocumentString(span, resource)
271+
jsonStr, err := MakeSegmentDocumentString(span, resource, nil, false)
269272

270273
assert.NotNil(t, jsonStr)
271274
assert.Nil(t, err)
@@ -324,14 +327,68 @@ func TestServerSpanWithNilAttributes(t *testing.T) {
324327
timeEvents.CopyTo(span.Events())
325328
pdata.NewAttributeMap().CopyTo(span.Attributes())
326329

327-
segment := MakeSegment(span, resource)
330+
segment := MakeSegment(span, resource, nil, false)
328331

329332
assert.NotNil(t, segment)
330333
assert.NotNil(t, segment.Cause)
331334
assert.Equal(t, "signup_aggregator", *segment.Name)
332335
assert.True(t, *segment.Fault)
333336
}
334337

338+
func TestSpanWithAttributesDefaultNotIndexed(t *testing.T) {
339+
spanName := "/api/locations"
340+
parentSpanID := newSegmentID()
341+
attributes := make(map[string]interface{})
342+
attributes["attr1@1"] = "val1"
343+
attributes["attr2@2"] = "val2"
344+
resource := constructDefaultResource()
345+
span := constructServerSpan(parentSpanID, spanName, tracetranslator.OCInternal, "OK", attributes)
346+
347+
segment := MakeSegment(span, resource, nil, false)
348+
349+
assert.NotNil(t, segment)
350+
assert.Equal(t, 0, len(segment.Annotations))
351+
assert.Equal(t, 2, len(segment.Metadata["default"]))
352+
assert.Equal(t, "val1", segment.Metadata["default"]["attr1@1"])
353+
assert.Equal(t, "val2", segment.Metadata["default"]["attr2@2"])
354+
}
355+
356+
func TestSpanWithAttributesPartlyIndexed(t *testing.T) {
357+
spanName := "/api/locations"
358+
parentSpanID := newSegmentID()
359+
attributes := make(map[string]interface{})
360+
attributes["attr1@1"] = "val1"
361+
attributes["attr2@2"] = "val2"
362+
resource := constructDefaultResource()
363+
span := constructServerSpan(parentSpanID, spanName, tracetranslator.OCInternal, "OK", attributes)
364+
365+
segment := MakeSegment(span, resource, []string{"attr1@1", "not_exist"}, false)
366+
367+
assert.NotNil(t, segment)
368+
assert.Equal(t, 1, len(segment.Annotations))
369+
assert.Equal(t, "val1", segment.Annotations["attr1_1"])
370+
assert.Equal(t, 1, len(segment.Metadata["default"]))
371+
assert.Equal(t, "val2", segment.Metadata["default"]["attr2@2"])
372+
}
373+
374+
func TestSpanWithAttributesAllIndexed(t *testing.T) {
375+
spanName := "/api/locations"
376+
parentSpanID := newSegmentID()
377+
attributes := make(map[string]interface{})
378+
attributes["attr1@1"] = "val1"
379+
attributes["attr2@2"] = "val2"
380+
resource := constructDefaultResource()
381+
span := constructServerSpan(parentSpanID, spanName, tracetranslator.OCInternal, "OK", attributes)
382+
383+
segment := MakeSegment(span, resource, []string{"attr1@1", "not_exist"}, true)
384+
385+
assert.NotNil(t, segment)
386+
assert.Equal(t, 2, len(segment.Annotations))
387+
assert.Equal(t, "val1", segment.Annotations["attr1_1"])
388+
assert.Equal(t, "val2", segment.Annotations["attr2_2"])
389+
assert.Equal(t, 0, len(segment.Metadata["default"]))
390+
}
391+
335392
func constructClientSpan(parentSpanID []byte, name string, code int32, message string, attributes map[string]interface{}) pdata.Span {
336393
var (
337394
traceID = newTraceID()

‎exporter/awsxrayexporter/translator/writer_pool_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func TestWriterPoolBasic(t *testing.T) {
3535
assert.NotNil(t, w.encoder)
3636
assert.Equal(t, size, w.buffer.Cap())
3737
assert.Equal(t, 0, w.buffer.Len())
38-
if err := w.Encode(MakeSegment(span, pdata.NewResource())); err != nil {
38+
if err := w.Encode(MakeSegment(span, pdata.NewResource(), nil, false)); err != nil {
3939
assert.Fail(t, "invalid json")
4040
}
4141
jsonStr := w.String()
@@ -51,7 +51,7 @@ func BenchmarkWithoutPool(b *testing.B) {
5151
b.StartTimer()
5252
buffer := bytes.NewBuffer(make([]byte, 0, 2048))
5353
encoder := json.NewEncoder(buffer)
54-
encoder.Encode(MakeSegment(span, pdata.NewResource()))
54+
encoder.Encode(MakeSegment(span, pdata.NewResource(), nil, false))
5555
logger.Info(buffer.String())
5656
}
5757
}
@@ -64,7 +64,7 @@ func BenchmarkWithPool(b *testing.B) {
6464
span := constructWriterPoolSpan()
6565
b.StartTimer()
6666
w := wp.borrow()
67-
w.Encode(MakeSegment(span, pdata.NewResource()))
67+
w.Encode(MakeSegment(span, pdata.NewResource(), nil, false))
6868
logger.Info(w.String())
6969
}
7070
}

0 commit comments

Comments
 (0)