Add parca_target_samples_total metric to track total scraped samples (#5988)

* Add `parca_target_samples_total` metric to track total scraped samples

* Suppress staticcheck warning for Prometheus `model.LabelName` usage

Author: metalmatze

pkg/normalizer/normalizer.go

@@ -512,6 +512,7 @@ func NormalizeWriteRawRequest(ctx context.Context, req *profilestorepb.WriteRawR
 				continue
 			}
 
+			//nolint:staticcheck // SA1019: Update when we actually use the latest Prometheus
 			if valid := model.LabelName(l.Name).IsValid(); !valid {
 				return NormalizedWriteRawRequest{}, status.Errorf(codes.InvalidArgument, "invalid label name: %v", l.Name)
 			}

pkg/scrape/manager.go

@@ -94,6 +94,11 @@ func NewManager(
 				Name: "parca_target_scrapes_sample_out_of_bounds_total",
 				Help: "Total number of samples rejected due to timestamp falling outside of the time bounds",
 			}),
+		sampleCount: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Name: "parca_target_samples_total",
+				Help: "Total number of samples that were scraped",
+			}),
 	}
 
 	reg.MustRegister(
@@ -105,6 +110,7 @@ func NewManager(
 		m.targetScrapeSampleDuplicate,
 		m.targetScrapeSampleOutOfOrder,
 		m.targetScrapeSampleOutOfBounds,
+		m.sampleCount,
 	)
 
 	c := make(map[string]*config.ScrapeConfig)
@@ -142,6 +148,7 @@ type Manager struct {
 	targetScrapeSampleDuplicate   prometheus.Counter
 	targetScrapeSampleOutOfOrder  prometheus.Counter
 	targetScrapeSampleOutOfBounds prometheus.Counter
+	sampleCount                   prometheus.Counter
 }
 
 // Run starts the manager with a set of scrape configs.
@@ -196,16 +203,23 @@ func (m *Manager) reload() {
 				level.Error(m.logger).Log("msg", "error reloading target set", "err", "invalid config id:"+setName)
 				return
 			}
-			sp = newScrapePool(scrapeConfig, m.store, log.With(m.logger, "scrape_pool", setName), m.externalLabels, &scrapePoolMetrics{
-				targetIntervalLength:          m.targetIntervalLength,
-				targetReloadIntervalLength:    m.targetReloadIntervalLength,
-				targetSyncIntervalLength:      m.targetSyncIntervalLength,
-				targetScrapePoolSyncsCounter:  m.targetScrapePoolSyncsCounter,
-				targetScrapeSampleLimit:       m.targetScrapeSampleLimit,
-				targetScrapeSampleDuplicate:   m.targetScrapeSampleDuplicate,
-				targetScrapeSampleOutOfOrder:  m.targetScrapeSampleOutOfOrder,
-				targetScrapeSampleOutOfBounds: m.targetScrapeSampleOutOfBounds,
-			})
+			sp = newScrapePool(
+				scrapeConfig,
+				m.store,
+				log.With(m.logger, "scrape_pool", setName),
+				m.externalLabels,
+				m.sampleCount,
+				&scrapePoolMetrics{
+					targetIntervalLength:          m.targetIntervalLength,
+					targetReloadIntervalLength:    m.targetReloadIntervalLength,
+					targetSyncIntervalLength:      m.targetSyncIntervalLength,
+					targetScrapePoolSyncsCounter:  m.targetScrapePoolSyncsCounter,
+					targetScrapeSampleLimit:       m.targetScrapeSampleLimit,
+					targetScrapeSampleDuplicate:   m.targetScrapeSampleDuplicate,
+					targetScrapeSampleOutOfOrder:  m.targetScrapeSampleOutOfOrder,
+					targetScrapeSampleOutOfBounds: m.targetScrapeSampleOutOfBounds,
+				},
+			)
 			m.scrapePools[setName] = sp
 		} else {
 			sp = existing

pkg/scrape/scrape.go

@@ -76,6 +76,7 @@ func newScrapePool(
 	store profilepb.ProfileStoreServiceServer,
 	logger log.Logger,
 	externalLabels labels.Labels,
+	sampleCount prometheus.Counter,
 	metrics *scrapePoolMetrics,
 ) *scrapePool {
 	if logger == nil {
@@ -109,6 +110,7 @@ func newScrapePool(
 			log.With(logger, "target", t),
 			externalLabels,
 			sp.metrics.targetIntervalLength,
+			sampleCount,
 			buffers,
 			store,
 			cfg.NormalizedAddresses,
@@ -370,6 +372,7 @@ type scrapeLoop struct {
 	scraper        scraper
 	l              log.Logger
 	intervalLength *prometheus.SummaryVec
+	sampleCount    prometheus.Counter
 	lastScrapeSize int
 	externalLabels labels.Labels
 
@@ -390,6 +393,7 @@ func newScrapeLoop(ctx context.Context,
 	l log.Logger,
 	externalLabels labels.Labels,
 	targetIntervalLength *prometheus.SummaryVec,
+	sampleCount prometheus.Counter,
 	buffers *pool.Pool,
 	store profilepb.ProfileStoreServiceServer,
 	normalizedAddresses bool,
@@ -409,6 +413,7 @@ func newScrapeLoop(ctx context.Context,
 		l:                   l,
 		externalLabels:      externalLabels,
 		intervalLength:      targetIntervalLength,
+		sampleCount:         sampleCount,
 		ctx:                 ctx,
 		normalizedAddresses: normalizedAddresses,
 	}
@@ -533,6 +538,8 @@ func processScrapeResp(buf *bytes.Buffer, sl *scrapeLoop, profileType string) er
 		return err
 	}
 
+	sl.sampleCount.Add(float64(len(p.Sample)))
+
 	var executableInfo []*profilepb.ExecutableInfo
 	for _, comment := range p.Comments {
 		if strings.HasPrefix(comment, "executableInfo=") {