renames only (#2051)

Author: hrissan

cmd/statshouse/statshouse-tools.go

@@ -718,10 +718,10 @@ func massUpdateMetadata() int {
 		//if !strings.HasPrefix(meta.Name, "gbuteyko") {
 		//	continue
 		//}
-		if meta.ShardStrategy == format.ShardByTagsHash || meta.ShardStrategy == format.ShardFixed || meta.ShardStrategy == format.ShardBuiltin {
+		if meta.ShardStrategy == format.ShardByTagsHash || meta.ShardStrategy == format.ShardFixed || meta.ShardStrategy == format.ShardBuiltinDist {
 			continue
 		}
-		if meta.ShardStrategy != format.ShardByMetric && meta.ShardStrategy != "" {
+		if meta.ShardStrategy != format.ShardByMetricID && meta.ShardStrategy != "" {
 			_, _ = fmt.Fprintf(os.Stderr, "Metric %d (%q) unexpected strategy %s shard %d\n", meta.MetricID, meta.Name, meta.ShardStrategy, meta.ShardNum)
 			continue
 		}

internal/agent/agent.go

@@ -683,9 +683,9 @@ func (s *Agent) addBuiltInsHeartbeatsLocked(nowUnix uint32, count float64) {
 	}
 	vKey.STags[9] = owner
 
-	vShardId := s.shard(&vKey, &s.builtinMetricMetaHeartbeatVersion, nil)
+	vShardNum := s.shard(&vKey, &s.builtinMetricMetaHeartbeatVersion, nil)
 	// resolutionHash will be 0 for built-in metrics, we are OK with this
-	vShard := s.Shards[vShardId]
+	vShard := s.Shards[vShardNum]
 	vShard.AddValueCounterStringHost(&vKey, 0, data_model.TagUnion{S: build.Commit()}, uptimeSec, count, data_model.TagUnionBytes{}, &s.builtinMetricMetaHeartbeatVersion)
 
 	aKey := data_model.Key{
@@ -697,9 +697,9 @@ func (s *Agent) addBuiltInsHeartbeatsLocked(nowUnix uint32, count float64) {
 	aKey.Tags[3] = s.argsHash
 	aKey.Tags[9] = s.argsLen
 
-	aShardId := s.shard(&aKey, &s.builtinMetricMetaHeartbeatArgs, nil)
+	aShardNum := s.shard(&aKey, &s.builtinMetricMetaHeartbeatArgs, nil)
 	// resolutionHash will be 0 for built-in metrics, we are OK with this
-	aShard := s.Shards[aShardId]
+	aShard := s.Shards[aShardNum]
 	aShard.AddValueCounterStringHost(&aKey, 0, data_model.TagUnion{S: s.args}, uptimeSec, count, data_model.TagUnionBytes{}, &s.builtinMetricMetaHeartbeatArgs)
 }
 
@@ -760,21 +760,20 @@ func (s *Agent) ApplyMetric(m tlstatshouse.MetricBytes, h data_model.MappedMetri
 	// Simply writing everything we know about metric ingestion errors would easily double how much metrics data we write
 	// So below is basically a compromise. All info is stored in MappingMetricHeader, if needed we can easily write more
 	// by changing code below
-	var shardId uint32
+	shard := s.Shards[0]
 	if h.MetricMeta != nil {
 		// ingestion statuses for unknown metric (metric not found) go to the first shard.
 		// for known metric with fixed shard, go to shard together with metric
 		// for known metric with hash_by_tags strategy, go to random shard together with metric
-		shardId = s.shard(&h.Key, h.MetricMeta, scratch)
-	}
-	if shardId >= uint32(len(s.Shards)) {
-		shard := s.Shards[0]
-		shard.AddCounterHostSrcIngestionStatus(0, format.BuiltinMetricMetaIngestionStatus,
-			[]int32{h.Key.Tags[0], h.Key.Metric, format.TagValueIDSrcIngestionStatusErrShardingFailed, 0},
-			1)
-		return
+		shardNum := s.shard(&h.Key, h.MetricMeta, scratch)
+		if shardNum >= uint32(len(s.Shards)) {
+			shard.AddCounterHostSrcIngestionStatus(0, format.BuiltinMetricMetaIngestionStatus,
+				[]int32{h.Key.Tags[0], h.Key.Metric, format.TagValueIDSrcIngestionStatusErrShardingFailed, 0},
+				1)
+			return
+		}
+		shard = s.Shards[shardNum]
 	}
-	shard := s.Shards[shardId]
 	if h.IngestionStatus != 0 {
 		// h.InvalidString was validated before mapping attempt.
 		// In case of utf decoding error, it contains hex representation of original string
@@ -892,9 +891,9 @@ func (s *Agent) AddCounterHostAERA(t uint32, metricInfo *format.MetricMetaValue,
 		key.Tags[format.RouteTag] = aera.Route
 		key.Tags[format.BuildArchTag] = aera.BuildArch
 	}
-	shardId := s.shard(&key, metricInfo, nil)
+	shardNum := s.shard(&key, metricInfo, nil)
 	// resolutionHash will be 0 for built-in metrics, we are OK with this
-	shard := s.Shards[shardId]
+	shard := s.Shards[shardNum]
 	shard.AddCounterHost(&key, 0, count, hostTag, metricInfo)
 }
 
@@ -920,9 +919,9 @@ func (s *Agent) AddCounterHostStringBytesAERA(t uint32, metricInfo *format.Metri
 		key.Tags[format.RouteTag] = aera.Route
 		key.Tags[format.BuildArchTag] = aera.BuildArch
 	}
-	shardId := s.shard(&key, metricInfo, nil)
+	shardNum := s.shard(&key, metricInfo, nil)
 	// resolutionHash will be 0 for built-in metrics, we are OK with this
-	shard := s.Shards[shardId]
+	shard := s.Shards[shardNum]
 	shard.AddCounterHostStringBytes(&key, 0, data_model.TagUnionBytes{S: str, I: 0}, count, hostTag, metricInfo)
 }
 
@@ -951,9 +950,9 @@ func (s *Agent) AddValueCounterHostAERA(t uint32, metricInfo *format.MetricMetaV
 		key.Tags[format.RouteTag] = aera.Route
 		key.Tags[format.BuildArchTag] = aera.BuildArch
 	}
-	shardId := s.shard(&key, metricInfo, nil)
+	shardNum := s.shard(&key, metricInfo, nil)
 	// resolutionHash will be 0 for built-in metrics, we are OK with this
-	shard := s.Shards[shardId]
+	shard := s.Shards[shardNum]
 	shard.AddValueCounterHost(&key, 0, value, counter, hostTag, metricInfo)
 }
 
@@ -978,9 +977,9 @@ func (s *Agent) AddValueCounterStringHostAERA(t uint32, metricInfo *format.Metri
 		key.Tags[format.RouteTag] = aera.Route
 		key.Tags[format.BuildArchTag] = aera.BuildArch
 	}
-	shardId := s.shard(&key, metricInfo, nil)
+	shardNum := s.shard(&key, metricInfo, nil)
 	// resolutionHash will be 0 for built-in metrics, we are OK with this
-	shard := s.Shards[shardId]
+	shard := s.Shards[shardNum]
 	shard.AddValueCounterStringHost(&key, 0, topValue, value, counter, hostTag, metricInfo)
 }
 
@@ -995,9 +994,9 @@ func (s *Agent) MergeItemValue(t uint32, metricInfo *format.MetricMetaValue, tag
 		key.Tags[format.AggShardTag] = s.AggregatorShardKey
 		key.Tags[format.AggReplicaTag] = s.AggregatorReplicaKey
 	}
-	shardId := s.shard(&key, metricInfo, nil)
+	shardNum := s.shard(&key, metricInfo, nil)
 	// resolutionHash will be 0 for built-in metrics, we are OK with this
-	shard := s.Shards[shardId]
+	shard := s.Shards[shardNum]
 	shard.MergeItemValue(&key, 0, item, metricInfo)
 }
 

internal/aggregator/migration.go

@@ -38,7 +38,7 @@ func getBuiltinShardingMetrics() map[int32]int {
 	shardingMetrics := make(map[int32]int)
 
 	for metricID, metric := range format.BuiltinMetrics {
-		if metric.ShardStrategy == format.ShardBuiltin {
+		if metric.ShardStrategy == format.ShardBuiltinDist {
 			// For metrics with MetricTagIndex > 0, use the specified tag
 			// For metrics with MetricTagIndex = 0 (like contributors_log), they appear in all shards
 			if metric.MetricTagIndex > 0 {
@@ -53,7 +53,7 @@ func getBuiltinShardingMetrics() map[int32]int {
 }
 
 // getConditionForSelect returns the complete condition for SELECT queries
-// This handles both regular metrics and ShardBuiltin metrics in a single query
+// This handles both regular metrics and ShardBuiltinDist metrics in a single query
 func getConditionForSelect(totalShards int, shardKey int32, tagPrefix string) string {
 	// Get all builtin metrics with special sharding
 	shardingMetrics := getBuiltinShardingMetrics()
@@ -84,13 +84,13 @@ func getConditionForSelect(totalShards int, shardKey int32, tagPrefix string) st
 }
 
 // getConditionForSelectV2 returns the complete condition for SELECT queries on V2 tables
-// This handles both regular metrics and ShardBuiltin metrics in a single query
+// This handles both regular metrics and ShardBuiltinDist metrics in a single query
 func getConditionForSelectV2(totalShards int, shardKey int32) string {
 	return getConditionForSelect(totalShards, shardKey, "key")
 }
 
 // getConditionForSelectV3 returns the complete condition for SELECT queries on V3 tables
-// This handles both regular metrics and ShardBuiltin metrics in a single query
+// This handles both regular metrics and ShardBuiltinDist metrics in a single query
 func getConditionForSelectV3(totalShards int, shardKey int32) string {
 	return getConditionForSelect(totalShards, shardKey, "tag")
 }
@@ -232,7 +232,7 @@ func migrateSingleStep(httpClient *http.Client, khAddr, khUser, khPassword strin
 
 	// Step 1: Select data from V2 table for the given timestamp and shard
 	// Include all aggregate fields - our parsers handle ClickHouse internal format correctly
-	// For ShardBuiltin metrics, we need special sharding conditions
+	// For ShardBuiltinDist metrics, we need special sharding conditions
 	shardingCondition := getConditionForSelectV2(config.TotalShards, shardKey)
 	selectQuery := fmt.Sprintf(`
 		SELECT metric, time, key0, key1, key2, key3, key4, key5, key6, key7, key8, key9, key10, key11, key12, key13, key14, key15, skey,

internal/format/builtin_metrics.go

@@ -31,7 +31,7 @@ Set only if greater than 1.`,
 		RawKind:     "int",
 	}},
 	PreKeyTagID:    "1",
-	ShardStrategy:  ShardBuiltin, // sharded the same way as metric it describes
+	ShardStrategy:  ShardBuiltinDist, // sharded the same way as metric it describes
 	MetricTagIndex: 1,
 }
 
@@ -154,7 +154,7 @@ So avg() of this metric shows estimated full cardinality with or without groupin
 		BuiltinKind: BuiltinKindMetric,
 	}},
 	PreKeyTagID:    "4",
-	ShardStrategy:  ShardBuiltin, // sharded the same way as metric it describes
+	ShardStrategy:  ShardBuiltinDist, // sharded the same way as metric it describes
 	MetricTagIndex: 4,
 }
 
@@ -180,7 +180,7 @@ Set only if greater than 1.`,
 		}),
 	}},
 	PreKeyTagID:    "4",
-	ShardStrategy:  ShardBuiltin, // sharded the same way as metrtic it describes
+	ShardStrategy:  ShardBuiltinDist, // sharded the same way as metrtic it describes
 	MetricTagIndex: 4,
 }
 
@@ -243,7 +243,7 @@ This metric uses sampling budgets of metric it refers to, so flooding by errors
 		Description: "tag_id",
 	}},
 	PreKeyTagID:    "1",
-	ShardStrategy:  ShardBuiltin, // sharded the same way as metrtic it describes
+	ShardStrategy:  ShardBuiltinDist, // sharded the same way as metrtic it describes
 	MetricTagIndex: 1,
 }
 
@@ -702,7 +702,7 @@ var BuiltinMetricMetaBadges = &MetricMetaValue{
 		BuiltinKind: BuiltinKindMetric,
 	}},
 	PreKeyTagID:    "2",
-	ShardStrategy:  ShardBuiltin, // sharded the same way as metrtic it describes
+	ShardStrategy:  ShardBuiltinDist, // sharded the same way as metrtic it describes
 	MetricTagIndex: 2,
 }
 
@@ -1062,7 +1062,7 @@ To see which seconds change when, use __contributors_log_rev`,
 		Description: "timestamp",
 		RawKind:     "timestamp",
 	}},
-	ShardStrategy: ShardBuiltin, // marshalled by aggregator but does not have metric tag
+	ShardStrategy: ShardBuiltinDist, // marshalled by aggregator but does not have metric tag
 }
 
 const BuiltinMetricIDContributorsLogRev = -62
@@ -1081,7 +1081,7 @@ Value is delta between second value and time it was inserted.`,
 		Description: "insert_timestamp",
 		RawKind:     "timestamp",
 	}},
-	ShardStrategy: ShardBuiltin, // marshalled by aggregator but does not have metric tag
+	ShardStrategy: ShardBuiltinDist, // marshalled by aggregator but does not have metric tag
 }
 
 var BuiltinMetricMetaGroupSizeBeforeSampling = &MetricMetaValue{

internal/format/format.go

@@ -230,10 +230,10 @@ type MetricsGroup struct {
 
 // possible sharding strategies
 const (
-	ShardByTagsHash = "tags_hash"
-	ShardFixed      = "fixed_shard"
-	ShardByMetric   = "metric_id" // shard = metric_id % num_shards
-	ShardBuiltin    = "builtin"   // for special builtin metrics that are written in all shards (only dist)
+	ShardByTagsHash  = "tags_hash"   // TODO: remove after nginx metrics are moved out to its own shard
+	ShardFixed       = "fixed_shard" // should be the only strategy for all user metrics
+	ShardByMetricID  = "metric_id"   // shard = metric_id % shardByMetricCount, for most built-in and hardware metrics.
+	ShardBuiltinDist = "builtin"     // for several builtin metrics which are written by aggregator into the same shard with the metrics they describe
 )
 
 // This struct is immutable, it is accessed by mapping code without any locking
@@ -648,7 +648,7 @@ func (metric *MetricMetaValue) NewSharding(timestamp, newShardingStart int64) bo
 		return false
 	}
 	switch metric.ShardStrategy {
-	case ShardFixed, ShardByMetric, "":
+	case ShardFixed, ShardByMetricID, "":
 		return newShardingStart != 0 && timestamp >= newShardingStart
 	default:
 		return false
@@ -662,7 +662,7 @@ func (m *MetricMetaValue) Shard(numShards int) int {
 	switch m.ShardStrategy {
 	case ShardFixed:
 		return int(m.ShardNum)
-	case ShardByMetric, "":
+	case ShardByMetricID, "":
 		return int(uint32(m.MetricID) % uint32(numShards))
 	default:
 		return -1

internal/sharding/sharding.go

@@ -10,10 +10,10 @@ func Shard(key *data_model.Key, meta *format.MetricMetaValue, shardByMetricCount
 	switch meta.ShardStrategy {
 	case format.ShardFixed:
 		return meta.ShardNum
-	case "", format.ShardByMetric:
+	case "", format.ShardByMetricID:
 		shard := uint32(key.Metric) % shardByMetricCount
 		return shard
-	case format.ShardBuiltin:
+	case format.ShardBuiltinDist:
 		tagIndex := meta.MetricTagIndex
 		// for builtin metrics we always use row values
 		metric := key.Tags[tagIndex]

internal/sharding/sharding_test.go

@@ -52,7 +52,7 @@ func TestShard(t *testing.T) {
 
 		{"ok-by-metric-id", args{
 			key:       metric1,
-			meta:      &format.MetricMetaValue{Name: "a", ShardStrategy: format.ShardByMetric},
+			meta:      &format.MetricMetaValue{Name: "a", ShardStrategy: format.ShardByMetricID},
 			numShards: 16,
 		}, 1},