[Feat] Metrics Rollup Rules: Support all aggregation types for MEASUREMENT (#221)

* go mod tidy

* Update provider.go

* Add `metricsRollupRules` support

* docs

* Add tests

* testdata

* Add validation

* uppercase

* Update resource_metrics_rollup_rules_test.go

* Add drop metric examples

* Add retries resource user

* Add unique id kibana object

* revert

* update `logzio_terraform_client` 1.27.0 -> 1.27.1

* update rollup rules according to latest api spec

* tests

* examples

* readme

* Update resource_user.go

* Update group_2.txt

* updated docs

* validate aggregation by metric type + remove name length validation

* tests

* docs + examples

* validate sum agg for `COUNTER`, `DELTA_COUNTER`, and `CUMULATIVE_COUNTER` + tests

* docs + examples

* logzio_terraform_client -> 1.27.2

* Update group_2.txt

* fix test regex errors

* pr comments fixes

* Update group_2.txt

* Update CHANGELOG.md

* Update resource_metrics_rollup_rules_test.go

* Update resource_metrics_rollup_rules_test.go

* Update resource_metrics_rollup_rules_test.go

* Update resource_metrics_rollup_rules_test.go

* Update resource_metrics_rollup_rules_test.go

* fix tests and adding helpers

* remove unused helpers and improve test checks

* add random id to kibana tests

* make tests fail faster

* `MEASUREMENT` support all aggegations

* tests

* docs

* Update group_2.txt

* logzio_terraform_client: `v1.27.2` -> `v1.27.3`

* Update CHANGELOG.md

* Update metrics_rollup_rules.md

---------

Co-authored-by: 8naama <[email protected]>

Author: yotamloe

.github/test-groups/group_2.txt

@@ -20,7 +20,7 @@ TestAccLogzioGrafanaAlertRule_CreateUpdateAlertRule
 TestAccLogzioGrafanaNotificationPolicy_InvalidMatchType
 TestAccLogzioGrafanaNotificationPolicy_ManageGrafanaNotificationPolicy
 TestAccLogzioMetricsRollupRules_CreateMeasurement
-TestAccLogzioMetricsRollupRules_CreateMeasurementInvalid
+TestAccLogzioMetricsRollupRules_CreateMeasurementWithP99
 TestAccLogzioMetricsRollupRules_CreateWithFilter
 TestAccLogzioMetricsRollupRules_CreateSimple
 TestAccLogzioMetricsRollupRules_CreateComplex

CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changes by Version
 
 <!-- next version -->
+## v1.21.1
+- Upgrade `logzio_client_terraform` to `1.27.3`.
+- Metrics Rollup Rules: Support all aggregation types for `MEASUREMENT`
+
 ## v1.21.0
 - Upgrade `logzio_client_terraform` to `1.27.2`.
 -  Add support for [Metrics Rollup Rules](./docs/resources/metrics_rollup_rules.md)

docs/data-sources/metrics_rollup_rules.md

@@ -76,7 +76,7 @@ The following attributes are exported:
 * `name` - The human-readable name of the rollup rule.
 * `metric_name` - The name of the metric (if rule is metric name-based).
 * `metric_type` - The type of the metric (GAUGE, COUNTER, DELTA_COUNTER, CUMULATIVE_COUNTER, or MEASUREMENT).
-* `rollup_function` - The aggregation function used for rolling up the metric. Always "SUM" for COUNTER, DELTA_COUNTER, and CUMULATIVE_COUNTER types.
+* `rollup_function` - The aggregation function used for rolling up the metric. Always "SUM" for COUNTER, DELTA_COUNTER, and CUMULATIVE_COUNTER types. Can be any valid aggregation function for GAUGE and MEASUREMENT types.
 * `labels_elimination_method` - The method for eliminating labels (EXCLUDE_BY or GROUP_BY).
 * `labels` - The list of label names being eliminated from the metric. 
 * `new_metric_name_template` - The template for generating new metric names.

docs/resources/metrics_rollup_rules.md

@@ -27,7 +27,7 @@ resource "logzio_metrics_rollup_rules" "frontend_metrics_rollup" {
   rollup_function           = "SUM"
   labels_elimination_method = "GROUP_BY"
   labels                    = ["service", "region"]
-
+  
   filter {
     expression {
       comparison = "EQ"
@@ -40,7 +40,7 @@ resource "logzio_metrics_rollup_rules" "frontend_metrics_rollup" {
       value      = "us-.*"
     }
   }
-
+  
   new_metric_name_template = "rollup.frontend.{{metricName}}"
   drop_original_metric     = true
 }
@@ -53,7 +53,7 @@ resource "logzio_metrics_rollup_rules" "response_time_rollup" {
   account_id                = 123456
   metric_name               = "http_response_time"
   metric_type               = "MEASUREMENT"
-  rollup_function           = "SUM"
+  rollup_function           = "P95"
   labels_elimination_method = "EXCLUDE_BY"
   labels                    = ["endpoint", "method"]
 }
@@ -66,7 +66,7 @@ The following arguments are supported:
 * `account_id` - (Required) The metrics account ID for the metrics rollup rule.
 * `metric_name` - (Optional) The name of the metric for which to create the rollup rule. Either `metric_name` or `filter` must be specified, but not both.
 * `metric_type` - (Required) The type of the metric. Valid values are `GAUGE`, `COUNTER`, `DELTA_COUNTER`, `CUMULATIVE_COUNTER`, and `MEASUREMENT`.
-* `rollup_function` - The aggregation function to use for rolling up the metric. Required for all metric types. For `COUNTER`, `DELTA_COUNTER`, and `CUMULATIVE_COUNTER` types, must be `SUM`. For `MEASUREMENT` metric type, only `SUM`, `MIN`, `MAX`, `COUNT`, `SUMSQ`, `MEAN`, and `LAST` are allowed. For `GAUGE` type, any valid aggregation function is allowed. Valid values include `SUM`, `MIN`, `MAX`, `COUNT`, `LAST`, `MEAN`, `MEDIAN`, `STDEV`, `SUMSQ`, and percentiles (`P10`, `P20`, `P25`, `P30`, `P40`, `P50`, `P60`, `P70`, `P75`, `P80`, `P90`, `P95`, `P99`, `P999`, `P9999`).
+* `rollup_function` - The aggregation function to use for rolling up the metric. Required for all metric types. For `COUNTER`, `DELTA_COUNTER`, and `CUMULATIVE_COUNTER` types, must be `SUM`. For `MEASUREMENT` and `GAUGE` metric types, any valid aggregation function is allowed. Valid values include `SUM`, `MIN`, `MAX`, `COUNT`, `LAST`, `MEAN`, `MEDIAN`, `STDEV`, `SUMSQ`, and percentiles (`P10`, `P20`, `P25`, `P30`, `P40`, `P50`, `P60`, `P70`, `P75`, `P80`, `P90`, `P95`, `P99`, `P999`, `P9999`).
 * `labels_elimination_method` - (Required) The method for eliminating labels. Valid values are `EXCLUDE_BY` and `GROUP_BY`.
 * `labels` - (Required) A list of label names to be eliminated from the metric.
 * `name` - (Optional) A human-readable name for the rollup rule.

examples/metrics_rollup_rules/main.tf

@@ -71,6 +71,7 @@ resource "logzio_metrics_rollup_rules" "frontend_service_rollup" {
     }
   }
 
+
   new_metric_name_template = "rollup.frontend.{{metricName}}"
   drop_original_metric     = true
 }

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/hashicorp/go-cty v1.5.0
 	github.com/hashicorp/terraform-plugin-log v0.9.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.37.0
-	github.com/logzio/logzio_terraform_client v1.27.2
+	github.com/logzio/logzio_terraform_client v1.27.3
 	github.com/stoewer/go-strcase v1.3.1
 	github.com/stretchr/testify v1.10.0
 )

go.sum

@@ -124,8 +124,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/logzio/logzio_terraform_client v1.27.2 h1:zfZ09gtbxVAM6g1FKC5inpK8zeAiN2F+pPhrHs2/Hj0=
-github.com/logzio/logzio_terraform_client v1.27.2/go.mod h1:pzXrr7gGIi6tU1mk2vTtg8dr6ttKMlt03kz6asC53o4=
+github.com/logzio/logzio_terraform_client v1.27.3 h1:Sf58iy88N4H4JBMiyl90joWpHUrDcoUCdB/WEnGnZbU=
+github.com/logzio/logzio_terraform_client v1.27.3/go.mod h1:pzXrr7gGIi6tU1mk2vTtg8dr6ttKMlt03kz6asC53o4=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=

logzio/resource_metrics_rollup_rules.go

@@ -197,19 +197,6 @@ func validateRollup(ctx context.Context, d *schema.ResourceDiff, m interface{})
 		if rollupFunction == "" {
 			return fmt.Errorf("rollup_function must be set for MEASUREMENT metrics")
 		}
-		// Validate MEASUREMENT allows only specific aggregation functions
-		allowedForMeasurement := map[string]bool{
-			string(metrics_rollup_rules.AggSum):   true,
-			string(metrics_rollup_rules.AggMin):   true,
-			string(metrics_rollup_rules.AggMax):   true,
-			string(metrics_rollup_rules.AggCount): true,
-			string(metrics_rollup_rules.AggSumSq): true,
-			string(metrics_rollup_rules.AggMean):  true,
-			string(metrics_rollup_rules.AggLast):  true,
-		}
-		if !allowedForMeasurement[rollupFunction] {
-			return fmt.Errorf("invalid aggregation function %q for MEASUREMENT metric type. Allowed functions: SUM, MIN, MAX, COUNT, SUMSQ, MEAN, LAST", rollupFunction)
-		}
 	case string(metrics_rollup_rules.MetricTypeCounter),
 		string(metrics_rollup_rules.MetricTypeDeltaCounter),
 		string(metrics_rollup_rules.MetricTypeCumulativeCounter):

logzio/resource_metrics_rollup_rules_test.go

@@ -22,7 +22,7 @@ const (
 	metricsRollupRulesResourceUpdate                             = "update_metrics_rollup_rules"
 	metricsRollupRulesResourceCreateWithFilter                   = "create_metrics_rollup_rules_with_filter"
 	metricsRollupRulesResourceCreateMeasurement                  = "create_metrics_rollup_rules_measurement"
-	metricsRollupRulesResourceCreateMeasurementInvalid           = "create_metrics_rollup_rules_measurement_invalid"
+	metricsRollupRulesResourceCreateMeasurementP99               = "create_metrics_rollup_rules_measurement_p99"
 	metricsRollupRulesResourceCreateCounterWithRollupFunction    = "create_metrics_rollup_rules_counter_with_rollup_function"
 	metricsRollupRulesResourceCreateCounterMissingRollupFunction = "create_metrics_rollup_rules_counter_missing_rollup_function"
 )
@@ -309,8 +309,8 @@ func TestAccLogzioMetricsRollupRules_CreateMeasurement(t *testing.T) {
 	})
 }
 
-func TestAccLogzioMetricsRollupRules_CreateMeasurementInvalid(t *testing.T) {
-	resourceName := "test_create_metrics_rollup_rules_measurement_invalid"
+func TestAccLogzioMetricsRollupRules_CreateCounterWithRollupFunction(t *testing.T) {
+	resourceName := "test_create_metrics_rollup_rules_counter_with_rollup_function"
 	accountId := os.Getenv(envLogzioMetricsAccountId)
 
 	defer utils.SleepAfterTest()
@@ -323,15 +323,15 @@ func TestAccLogzioMetricsRollupRules_CreateMeasurementInvalid(t *testing.T) {
 		ProviderFactories: testAccProviderFactories,
 		Steps: []resource.TestStep{
 			{
-				Config:      resourceTestMetricsRollupRules(resourceName, metricsRollupRulesResourceCreateMeasurementInvalid, accountId),
-				ExpectError: regexp.MustCompile("invalid aggregation function \"P99\" for MEASUREMENT metric type"),
+				Config:      resourceTestMetricsRollupRules(resourceName, metricsRollupRulesResourceCreateCounterWithRollupFunction, accountId),
+				ExpectError: regexp.MustCompile("for COUNTER metrics, rollup_function must be SUM"),
 			},
 		},
 	})
 }
 
-func TestAccLogzioMetricsRollupRules_CreateCounterWithRollupFunction(t *testing.T) {
-	resourceName := "test_create_metrics_rollup_rules_counter_with_rollup_function"
+func TestAccLogzioMetricsRollupRules_CreateCounterMissingRollupFunction(t *testing.T) {
+	resourceName := "test_create_metrics_rollup_rules_counter_missing_rollup_function"
 	accountId := os.Getenv(envLogzioMetricsAccountId)
 
 	defer utils.SleepAfterTest()
@@ -344,15 +344,15 @@ func TestAccLogzioMetricsRollupRules_CreateCounterWithRollupFunction(t *testing.
 		ProviderFactories: testAccProviderFactories,
 		Steps: []resource.TestStep{
 			{
-				Config:      resourceTestMetricsRollupRules(resourceName, metricsRollupRulesResourceCreateCounterWithRollupFunction, accountId),
-				ExpectError: regexp.MustCompile("for COUNTER metrics, rollup_function must be SUM"),
+				Config:      resourceTestMetricsRollupRules(resourceName, metricsRollupRulesResourceCreateCounterMissingRollupFunction, accountId),
+				ExpectError: regexp.MustCompile("rollup_function must be set for COUNTER metrics"),
 			},
 		},
 	})
 }
 
-func TestAccLogzioMetricsRollupRules_CreateCounterMissingRollupFunction(t *testing.T) {
-	resourceName := "test_create_metrics_rollup_rules_counter_missing_rollup_function"
+func TestAccLogzioMetricsRollupRules_CreateMeasurementWithP99(t *testing.T) {
+	resourceName := "test_create_metrics_rollup_rules_measurement_p99"
 	accountId := os.Getenv(envLogzioMetricsAccountId)
 
 	defer utils.SleepAfterTest()
@@ -365,8 +365,12 @@ func TestAccLogzioMetricsRollupRules_CreateCounterMissingRollupFunction(t *testi
 		ProviderFactories: testAccProviderFactories,
 		Steps: []resource.TestStep{
 			{
-				Config:      resourceTestMetricsRollupRules(resourceName, metricsRollupRulesResourceCreateCounterMissingRollupFunction, accountId),
-				ExpectError: regexp.MustCompile("rollup_function must be set for COUNTER metrics"),
+				Config: resourceTestMetricsRollupRules(resourceName, metricsRollupRulesResourceCreateMeasurementP99, accountId),
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr("logzio_metrics_rollup_rules."+resourceName, metricsRollupRulesMetricType, "MEASUREMENT"),
+					resource.TestCheckResourceAttr("logzio_metrics_rollup_rules."+resourceName, metricsRollupRulesRollupFunction, "P99"),
+					resource.TestCheckResourceAttr("logzio_metrics_rollup_rules."+resourceName, metricsRollupRulesMetricName, "response_time"),
+				),
 			},
 		},
 	})

logzio/testdata/fixtures/create_metrics_rollup_rules_measurement_p99.tf

@@ -0,0 +1,8 @@
+resource "logzio_metrics_rollup_rules" "%s" {
+  account_id = %s
+  metric_name = "response_time"
+  metric_type = "MEASUREMENT"
+  rollup_function = "P99"
+  labels_elimination_method = "EXCLUDE_BY"
+  labels = ["instance_id"]
+}