[FEATURE] Distributor: Make add-metric-suffixes configurable

CHANGELOG.md

@@ -9,6 +9,7 @@
   * Metrics: Renamed `cortex_parquet_queryable_cache_*` to `cortex_parquet_cache_*`.
   * Flags: Renamed `-querier.parquet-queryable-shard-cache-size` to `-querier.parquet-shard-cache-size` and `-querier.parquet-queryable-shard-cache-ttl` to `-querier.parquet-shard-cache-ttl`.
   * Config: Renamed `parquet_queryable_shard_cache_size` to `parquet_shard_cache_size` and `parquet_queryable_shard_cache_ttl` to `parquet_shard_cache_ttl`.
+* [FEATURE] Distrubutor: Add `-distributor.otlp.add-metric-suffixes` flag. If true, suffixes will be added to the metrics for name normalization. #7286
 * [FEATURE] StoreGateway: Introduces a new parquet mode. #7046
 * [FEATURE] StoreGateway: Add a parquet shard cache to parquet mode. #7166
 * [FEATURE] Distributor: Add a per-tenant flag `-distributor.enable-type-and-unit-labels` that enables adding `__unit__` and `__type__` labels for remote write v2 and OTLP requests. This is a breaking change; the `-distributor.otlp.enable-type-and-unit-labels` flag is now deprecated, operates as a no-op, and has been consolidated into this new flag. #7077

docs/configuration/config-file-reference.md

@@ -3337,6 +3337,10 @@ otlp:
   # Deprecated: Use `-distributor.enable-type-and-unit-labels` flag instead.
   # CLI flag: -distributor.otlp.enable-type-and-unit-labels
   [enable_type_and_unit_labels: <boolean> | default = false]
+
+  # If true, suffixes will be added to the metrics for name normalization.
+  # CLI flag: -distributor.otlp.add-metric-suffixes
+  [add_metric_suffixes: <boolean> | default = true]
 ```
 
 ### `etcd_config`

docs/guides/open-telemetry-collector.md

@@ -77,6 +77,7 @@ distributor:
     disable_target_info: <boolean>
     allow_delta_temporality: <boolean>
     enable_type_and_unit_labels: <boolean>
+    add_metric_suffixes: <boolean>
 ```
 
 ### Ingest `target_info` metric
@@ -156,3 +157,8 @@ overrides:
     promote_resource_attributes: ["attr1", "attr2"]
 `
 ```
+
+### Configure adding suffixes to metrics
+
+The flag `add_metric_suffixes` allows control to add suffixes to the metrics for name normalization.
+This flag is enabled by default.

pkg/distributor/distributor.go

@@ -202,6 +202,7 @@ type OTLPConfig struct {
 	DisableTargetInfo       bool `yaml:"disable_target_info"`
 	AllowDeltaTemporality   bool `yaml:"allow_delta_temporality"`
 	EnableTypeAndUnitLabels bool `yaml:"enable_type_and_unit_labels"`
+	AddMetricSuffixes       bool `yaml:"add_metric_suffixes"`
 }
 
 // RegisterFlags adds the flags required to config this to the given FlagSet
@@ -231,6 +232,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.OTLPConfig.DisableTargetInfo, "distributor.otlp.disable-target-info", false, "If true, a target_info metric is not ingested. (refer to: https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems)")
 	f.BoolVar(&cfg.OTLPConfig.AllowDeltaTemporality, "distributor.otlp.allow-delta-temporality", false, "EXPERIMENTAL: If true, delta temporality otlp metrics to be ingested.")
 	f.BoolVar(&cfg.OTLPConfig.EnableTypeAndUnitLabels, "distributor.otlp.enable-type-and-unit-labels", false, "Deprecated: Use `-distributor.enable-type-and-unit-labels` flag instead.")
+	f.BoolVar(&cfg.OTLPConfig.AddMetricSuffixes, "distributor.otlp.add-metric-suffixes", true, "If true, suffixes will be added to the metrics for name normalization.")
 }
 
 // Validate config and returns error on failure

pkg/util/push/otlp.go

@@ -184,7 +184,7 @@ func convertToPromTS(ctx context.Context, pmetrics pmetric.Metrics, cfg distribu
 	collector := newCollectingAppender()
 	promConverter := prometheusremotewrite.NewPrometheusConverter(collector)
 	settings := prometheusremotewrite.Settings{
-		AddMetricSuffixes:       true,
+		AddMetricSuffixes:       cfg.AddMetricSuffixes,
 		DisableTargetInfo:       cfg.DisableTargetInfo,
 		AllowDeltaTemporality:   cfg.AllowDeltaTemporality,
 		EnableTypeAndUnitLabels: overrides.EnableTypeAndUnitLabels(userID),

pkg/util/push/otlp_test.go

@@ -39,13 +39,15 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 		description             string
 		enableTypeAndUnitLabels bool
 		allowDeltaTemporality   bool
+		addMetricSuffixes       bool
 		otlpSeries              pmetric.Metric
 		expectedLabels          labels.Labels
 		expectedMetadata        prompb.MetricMetadata
 	}{
 		{
 			description:             "[enableTypeAndUnitLabels: true], the '__type__' label should be attached when the type is the gauge",
 			enableTypeAndUnitLabels: true,
+			addMetricSuffixes:       true,
 			otlpSeries:              createOtelSum("test", "seconds", pmetric.AggregationTemporalityCumulative, ts),
 			expectedLabels: labels.FromMap(map[string]string{
 				"__name__":   "test_seconds",
@@ -59,6 +61,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 			description:             "[enableTypeAndUnitLabels: true], the '__type__' label should not be attached when the type is unknown",
 			enableTypeAndUnitLabels: true,
 			allowDeltaTemporality:   true,
+			addMetricSuffixes:       true,
 			otlpSeries:              createOtelSum("test", "seconds", pmetric.AggregationTemporalityDelta, ts),
 			expectedLabels: labels.FromMap(map[string]string{
 				"__name__":   "test_seconds",
@@ -70,6 +73,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 		{
 			description:             "[enableTypeAndUnitLabels: false]",
 			enableTypeAndUnitLabels: false,
+			addMetricSuffixes:       true,
 			otlpSeries:              createOtelSum("test", "seconds", pmetric.AggregationTemporalityCumulative, ts),
 			expectedLabels: labels.FromMap(map[string]string{
 				"__name__":   "test_seconds",
@@ -83,6 +87,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 		t.Run(test.description, func(t *testing.T) {
 			cfg := distributor.OTLPConfig{
 				AllowDeltaTemporality: test.allowDeltaTemporality,
+				AddMetricSuffixes:     test.addMetricSuffixes,
 			}
 			metrics := pmetric.NewMetrics()
 			rm := metrics.ResourceMetrics().AppendEmpty()
@@ -388,6 +393,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: false,
 				DisableTargetInfo:    false,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -410,6 +416,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: false,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -432,6 +439,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: false,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -450,6 +458,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: true,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -484,6 +493,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: true,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -563,6 +573,223 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 	}
 }
 
+func TestOTLPConvertToPromTS_WithoutAddMetricSuffixes(t *testing.T) {
+	logger := log.NewNopLogger()
+	ctx := context.Background()
+	d := pmetric.NewMetrics()
+	resourceMetric := d.ResourceMetrics().AppendEmpty()
+	resourceMetric.Resource().Attributes().PutStr("service.name", "test-service") // converted to job, service_name
+	resourceMetric.Resource().Attributes().PutStr("attr1", "value")
+	resourceMetric.Resource().Attributes().PutStr("attr2", "value")
+	resourceMetric.Resource().Attributes().PutStr("attr3", "value")
+
+	scopeMetric := resourceMetric.ScopeMetrics().AppendEmpty()
+
+	//Generate One Counter
+	timestamp := time.Now()
+	counterMetric := scopeMetric.Metrics().AppendEmpty()
+	counterMetric.SetName("test-counter")
+	counterMetric.SetDescription("test-counter-description")
+	counterMetric.SetEmptySum()
+	counterMetric.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
+	counterMetric.Sum().SetIsMonotonic(true)
+
+	counterDataPoint := counterMetric.Sum().DataPoints().AppendEmpty()
+	counterDataPoint.SetTimestamp(pcommon.NewTimestampFromTime(timestamp))
+	counterDataPoint.SetDoubleValue(10.0)
+
+	tests := []struct {
+		description               string
+		PromoteResourceAttributes []string
+		cfg                       distributor.OTLPConfig
+		expectedLabels            []prompb.Label
+	}{
+		{
+			description:               "target_info should be generated and an attribute that exist in promote resource attributes should be converted",
+			PromoteResourceAttributes: []string{"attr1"},
+			cfg: distributor.OTLPConfig{
+				ConvertAllAttributes: false,
+				DisableTargetInfo:    false,
+				AddMetricSuffixes:    false,
+			},
+			expectedLabels: []prompb.Label{
+				{
+					Name:  "__name__",
+					Value: "test_counter",
+				},
+				{
+					Name:  "attr1",
+					Value: "value",
+				},
+				{
+					Name:  "job",
+					Value: "test-service",
+				},
+			},
+		},
+		{
+			description:               "an attributes that exist in promote resource attributes should be converted",
+			PromoteResourceAttributes: []string{"attr1"},
+			cfg: distributor.OTLPConfig{
+				ConvertAllAttributes: false,
+				DisableTargetInfo:    true,
+				AddMetricSuffixes:    false,
+			},
+			expectedLabels: []prompb.Label{
+				{
+					Name:  "__name__",
+					Value: "test_counter",
+				},
+				{
+					Name:  "attr1",
+					Value: "value",
+				},
+				{
+					Name:  "job",
+					Value: "test-service",
+				},
+			},
+		},
+		{
+			description:               "not exist attribute is ignored",
+			PromoteResourceAttributes: []string{"dummy"},
+			cfg: distributor.OTLPConfig{
+				ConvertAllAttributes: false,
+				DisableTargetInfo:    true,
+				AddMetricSuffixes:    false,
+			},
+			expectedLabels: []prompb.Label{
+				{
+					Name:  "__name__",
+					Value: "test_counter",
+				},
+				{
+					Name:  "job",
+					Value: "test-service",
+				},
+			},
+		},
+		{
+			description:               "should convert all attribute",
+			PromoteResourceAttributes: nil,
+			cfg: distributor.OTLPConfig{
+				ConvertAllAttributes: true,
+				DisableTargetInfo:    true,
+				AddMetricSuffixes:    false,
+			},
+			expectedLabels: []prompb.Label{
+				{
+					Name:  "__name__",
+					Value: "test_counter",
+				},
+				{
+					Name:  "attr1",
+					Value: "value",
+				},
+				{
+					Name:  "attr2",
+					Value: "value",
+				},
+				{
+					Name:  "attr3",
+					Value: "value",
+				},
+				{
+					Name:  "job",
+					Value: "test-service",
+				},
+				{
+					Name:  "service_name",
+					Value: "test-service",
+				},
+			},
+		},
+		{
+			description:               "should convert all attribute regardless of promote resource attributes",
+			PromoteResourceAttributes: []string{"attr1", "attr2"},
+			cfg: distributor.OTLPConfig{
+				ConvertAllAttributes: true,
+				DisableTargetInfo:    true,
+				AddMetricSuffixes:    false,
+			},
+			expectedLabels: []prompb.Label{
+				{
+					Name:  "__name__",
+					Value: "test_counter",
+				},
+				{
+					Name:  "attr1",
+					Value: "value",
+				},
+				{
+					Name:  "attr2",
+					Value: "value",
+				},
+				{
+					Name:  "attr3",
+					Value: "value",
+				},
+				{
+					Name:  "job",
+					Value: "test-service",
+				},
+				{
+					Name:  "service_name",
+					Value: "test-service",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			limits := validation.Limits{
+				PromoteResourceAttributes: test.PromoteResourceAttributes,
+			}
+			overrides := validation.NewOverrides(limits, nil)
+			tsList, metadata, err := convertToPromTS(ctx, d, test.cfg, overrides, "user-1", logger)
+			require.NoError(t, err)
+
+			// test metadata conversion (counter + optionally target_info)
+			expectedMetadataLen := 1
+			if !test.cfg.DisableTargetInfo {
+				expectedMetadataLen = 2 // test_counter + target_info
+			}
+			require.Equal(t, expectedMetadataLen, len(metadata))
+			var counterMetadata *prompb.MetricMetadata
+			for i := range metadata {
+				if metadata[i].MetricFamilyName == "test_counter" {
+					counterMetadata = &metadata[i]
+					break
+				}
+			}
+			require.NotNil(t, counterMetadata)
+			require.Equal(t, prompb.MetricMetadata_MetricType(1), counterMetadata.Type)
+			require.Equal(t, "test_counter", counterMetadata.MetricFamilyName)
+			require.Equal(t, "test-counter-description", counterMetadata.Help)
+
+			if test.cfg.DisableTargetInfo {
+				require.Equal(t, 1, len(tsList)) // test_counter
+			} else {
+				// target_info should exist
+				require.Equal(t, 2, len(tsList)) // test_counter + target_info
+			}
+
+			var counterTs prompb.TimeSeries
+			for _, ts := range tsList {
+				for _, label := range ts.Labels {
+					if label.Name == "__name__" && label.Value == "test_counter" {
+						// get counter ts
+						counterTs = ts
+					}
+				}
+			}
+
+			require.ElementsMatch(t, test.expectedLabels, counterTs.Labels)
+		})
+	}
+}
+
 // for testing
 type resetReader struct {
 	*bytes.Reader

schemas/cortex-config-schema.json

@@ -3883,6 +3883,12 @@
         },
         "otlp": {
           "properties": {
+            "add_metric_suffixes": {
+              "default": true,
+              "description": "If true, suffixes will be added to the metrics for name normalization.",
+              "type": "boolean",
+              "x-cli-flag": "distributor.otlp.add-metric-suffixes"
+            },
             "allow_delta_temporality": {
               "default": false,
               "description": "EXPERIMENTAL: If true, delta temporality otlp metrics to be ingested.",