Speedscope profiles dropped by compaction #4584
Description
Describe the bug
Uploading a Speedscope profile via the /ingest endpoint will not fully persist. The initial upload is correct, but series (observed by seeing missing symbols in the UI) will slowly drop over time.
To Reproduce
- Start Pyroscope on localhost:4040
$ curl -X POST "localhost:4040/ingest?name=MyServiceName&from=$(gdate -d '6 minutes ago' +%s)&until=$(gdate -d '15 minutes ago' +%s)&format=speedscope" \
> -H "Content-Type: application/json" \
> -d @speedscope.json
- Observe the
Thread (*)symbols atlocalhost:4040being dropped over time as compaction runs
[1] speedscope.json
Additional Context
Originally reported: #3989 (comment)
This diff applied to the speedscope-ingestion-fix branch is a hack that resolves the issue, showing the root cause.
diff --git a/pkg/og/convert/speedscope/parser.go b/pkg/og/convert/speedscope/parser.go
index 5e0214912..a183c858d 100644
--- a/pkg/og/convert/speedscope/parser.go
+++ b/pkg/og/convert/speedscope/parser.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "time"
"github.com/grafana/pyroscope/pkg/og/ingestion"
"github.com/grafana/pyroscope/pkg/og/storage"
@@ -52,8 +53,8 @@ func parseAll(rawData []byte, md ingestion.Metadata) ([]*storage.PutInput, error
LabelSet: md.LabelSet,
}
- for _, prof := range file.Profiles {
- putInput, err := parseOne(&prof, input, file.Shared.Frames, len(file.Profiles) > 1)
+ for i, prof := range file.Profiles {
+ putInput, err := parseOne(i, &prof, input, file.Shared.Frames, len(file.Profiles) > 1)
if err != nil {
return nil, err
}
@@ -62,7 +63,7 @@ func parseAll(rawData []byte, md ingestion.Metadata) ([]*storage.PutInput, error
return results, nil
}
-func parseOne(prof *profile, putInput storage.PutInput, frames []frame, multi bool) (*storage.PutInput, error) {
+func parseOne(i int, prof *profile, putInput storage.PutInput, frames []frame, multi bool) (*storage.PutInput, error) {
// Fixup some metadata
putInput.Units = prof.Unit.chooseMetadataUnit()
putInput.AggregationType = metadata.SumAggregationType
@@ -76,6 +77,8 @@ func parseOne(prof *profile, putInput storage.PutInput, frames []frame, multi bo
putInput.SampleRate = uint32(prof.Unit.defaultSampleRate())
}
+ putInput.StartTime = putInput.StartTime.Add(time.Second * time.Duration(i))
+
var err error
tr := tree.New()
switch prof.Type {