[processor/transform] introduce aggregate_on_attributes function for …

…metrics (#33334)

**Link to tracking Issue:** #16224 

**Changes:**

- implemented `aggregate_on_attributes` function (supporting
sum/min/max/mean/median) for Sum, Gauge, Histogram, ExponentialHistogram
- tests
- documentation

**Depends on**
#33669

---------

Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com>
Co-authored-by: Evan Bradley <11745660+evan-bradley@users.noreply.github.com>

.chloggen/add_aggregate_on_attributes.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: transformprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Support aggregating metrics based on their attributes."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [16224]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

internal/coreinternal/aggregateutil/type.go

@@ -5,6 +5,7 @@ package aggregateutil // import "github.com/open-telemetry/opentelemetry-collect
 
 import (
 	"fmt"
+	"strings"
 
 	"go.opentelemetry.io/collector/pdata/pmetric"
 )
@@ -32,26 +33,34 @@ const (
 	Count AggregationType = "count"
 )
 
-var AggregationTypes = []AggregationType{Sum, Mean, Min, Max, Count}
+var AggregationTypes = []AggregationType{Sum, Mean, Min, Max, Median, Count}
 
 func (at AggregationType) IsValid() bool {
-	for _, AggregationType := range AggregationTypes {
-		if at == AggregationType {
+	for _, aggregationType := range AggregationTypes {
+		if at == aggregationType {
 			return true
 		}
 	}
 
 	return false
 }
 
+func GetSupportedAggregationFunctionsList() string {
+	slice := make([]string, 0, len(AggregationTypes))
+	for _, a := range AggregationTypes {
+		slice = append(slice, string(a))
+	}
+	return strings.Join(slice, ", ")
+}
+
 type AggGroups struct {
 	gauge        map[string]pmetric.NumberDataPointSlice
 	sum          map[string]pmetric.NumberDataPointSlice
 	histogram    map[string]pmetric.HistogramDataPointSlice
 	expHistogram map[string]pmetric.ExponentialHistogramDataPointSlice
 }
 
-func ConvertToAggregationType(str string) (AggregationType, error) {
+func ConvertToAggregationFunction(str string) (AggregationType, error) {
 	a := AggregationType(str)
 	if a.IsValid() {
 		return a, nil

internal/coreinternal/aggregateutil/type_test.go

@@ -58,9 +58,13 @@ func Test_AggregationType_Convert(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := ConvertToAggregationType(tt.in)
+			got, err := ConvertToAggregationFunction(tt.in)
 			require.Equal(t, tt.want, got)
 			require.Equal(t, tt.wantErr, err)
 		})
 	}
 }
+
+func Test_GetSupportedAggregationFunctionsList(t *testing.T) {
+	require.Equal(t, "sum, mean, min, max, median, count", GetSupportedAggregationFunctionsList())
+}

processor/transformprocessor/README.md

@@ -219,6 +219,7 @@ In addition to OTTL functions, the processor defines its own functions to help w
 - [convert_summary_sum_val_to_sum](#convert_summary_sum_val_to_sum)
 - [copy_metric](#copy_metric)
 - [scale_metric](#scale_metric)
+- [aggregate_on_attributes](#aggregate_on_attributes)
 
 ### convert_sum_to_gauge
 
@@ -369,6 +370,51 @@ Examples:
 - `scale_metric(0.1)`: Scale the metric by a factor of `0.1`. The unit of the metric will not be modified.
 - `scale_metric(10.0, "kWh")`: Scale the metric by a factor of `10.0` and sets the unit to `kWh`.
 
+### aggregate_on_attributes
+
+`aggregate_on_attributes(function, Optional[attributes])`
+
+The `aggregate_on_attributes` function aggregates all datapoints in the metric based on the supplied attributes. `function` is a case-sensitive string that represents the aggregation function and `attributes` is an optional list of attribute keys to aggregate upon.
+
+`aggregate_on_attributes` function removes all attributes that are present in datapoints except the ones that are specified in the `attributes` parameter. If `attributes` parameter is not set, all attributes are removed from datapoints. Afterwards all datapoints are aggregated depending on the attributes left (none or the ones present in the list).
+
+The following metric types can be aggregated:
+
+- sum
+- gauge
+- histogram
+- exponential histogram
+
+Supported aggregation functions are:
+
+- sum
+- max
+- min
+- mean
+- median
+- count
+
+**NOTE:** Only the `sum` aggregation function is supported for histogram and exponential histogram datatypes.
+
+Examples:
+
+- `aggregate_on_attributes("sum", ["attr1", "attr2"]) where name == "system.memory.usage"`
+- `aggregate_on_attributes("max") where name == "system.memory.usage"`
+
+The `aggregate_on_attributes` function can also be used in conjunction with 
+[keep_matching_keys](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs#keep_matching_keys) or
+[delete_matching_keys](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs#delete_matching_keys).
+
+For example, to remove attribute keys matching a regex and aggregate the metrics on the remaining attributes, you can perform the following statement sequence:
+
+```yaml
+statements:
+   - delete_matching_keys(attributes, "(?i).*myRegex.*") where name == "system.memory.usage"
+   - aggregate_on_attributes("sum") where name == "system.memory.usage"
+```
+
+To aggregate only using a specified set of attributes, you can use `keep_matching_keys`.
+
 ## Examples
 
 ### Perform transformation if field does not exist

processor/transformprocessor/go.mod

@@ -4,6 +4,7 @@ go 1.21.0
 
 require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.105.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.105.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.105.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.105.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.105.0
@@ -49,7 +50,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.105.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.105.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect

processor/transformprocessor/internal/metrics/func_aggregate_on_attributes_metrics.go

@@ -0,0 +1,59 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package metrics // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor/internal/metrics"
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/collector/pdata/pmetric"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/aggregateutil"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/contexts/ottlmetric"
+)
+
+type aggregateOnAttributesArguments struct {
+	AggregationFunction string
+	Attributes          ottl.Optional[[]string]
+}
+
+func newAggregateOnAttributesFactory() ottl.Factory[ottlmetric.TransformContext] {
+	return ottl.NewFactory("aggregate_on_attributes", &aggregateOnAttributesArguments{}, createAggregateOnAttributesFunction)
+}
+
+func createAggregateOnAttributesFunction(_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[ottlmetric.TransformContext], error) {
+	args, ok := oArgs.(*aggregateOnAttributesArguments)
+
+	if !ok {
+		return nil, fmt.Errorf("AggregateOnAttributesFactory args must be of type *AggregateOnAttributesArguments")
+	}
+
+	t, err := aggregateutil.ConvertToAggregationFunction(args.AggregationFunction)
+	if err != nil {
+		return nil, fmt.Errorf("invalid aggregation function: '%s', valid options: %s", err.Error(), aggregateutil.GetSupportedAggregationFunctionsList())
+	}
+
+	return AggregateOnAttributes(t, args.Attributes)
+}
+
+func AggregateOnAttributes(aggregationFunction aggregateutil.AggregationType, attributes ottl.Optional[[]string]) (ottl.ExprFunc[ottlmetric.TransformContext], error) {
+	return func(_ context.Context, tCtx ottlmetric.TransformContext) (any, error) {
+		metric := tCtx.GetMetric()
+
+		if metric.Type() == pmetric.MetricTypeSummary {
+			return nil, fmt.Errorf("aggregate_on_attributes does not support aggregating Summary metrics")
+		}
+
+		ag := aggregateutil.AggGroups{}
+		aggregateutil.FilterAttrs(metric, attributes.Get())
+		newMetric := pmetric.NewMetric()
+		aggregateutil.CopyMetricDetails(metric, newMetric)
+		aggregateutil.GroupDataPoints(metric, &ag)
+		aggregateutil.MergeDataPoints(newMetric, aggregationFunction, ag)
+		newMetric.MoveTo(metric)
+
+		return nil, nil
+	}, nil
+}