[pkg/ottl] add ValueExpression to support extraction of values from…

… the signal context (#36883)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

This PR adds a new `ParseValueExpression` method to the OTTL parser
which allows users of this package to extract data from the context
using OTTL

<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes #35621 

<!--Describe what testing was performed and which tests were added.-->
#### Testing

Added unit and e2e tests 

<!--Describe the documentation added.-->
#### Documentation

Added godoc comments for the added methods and types

<!--Please delete paragraphs that you did not use before submitting.-->

---------

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>
Co-authored-by: Evan Bradley <11745660+evan-bradley@users.noreply.github.com>
Co-authored-by: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com>

.chloggen/ottl-value-expression.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/ottl
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add value expression parser that enables components using ottl to retrieve values from the output of an expression
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [35621]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: the expression can be either a literal value, a path value within the context, or the result of a converter and/or a mathematical expression.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

pkg/ottl/e2e/e2e_test.go

@@ -1186,6 +1186,61 @@ func Test_e2e_ottl_features(t *testing.T) {
 	}
 }
 
+func Test_e2e_ottl_value_expressions(t *testing.T) {
+	tests := []struct {
+		name      string
+		statement string
+		want      any
+	}{
+		{
+			name:      "string literal",
+			statement: `"foo"`,
+			want:      "foo",
+		},
+		{
+			name:      "attribute value",
+			statement: `resource.attributes["host.name"]`,
+			want:      "localhost",
+		},
+		{
+			name:      "accessing enum",
+			statement: `SEVERITY_NUMBER_TRACE`,
+			want:      int64(1),
+		},
+		{
+			name:      "Using converter",
+			statement: `TraceID(0x0102030405060708090a0b0c0d0e0f10)`,
+			want:      pcommon.TraceID{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10},
+		},
+		{
+			name:      "Adding results of two converter operations",
+			statement: `Len(attributes) + Len(attributes)`,
+			want:      int64(24),
+		},
+		{
+			name:      "Nested converter operations",
+			statement: `Hex(Len(attributes) + Len(attributes))`,
+			want:      "0000000000000018",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.statement, func(t *testing.T) {
+			settings := componenttest.NewNopTelemetrySettings()
+			logParser, err := ottllog.NewParser(ottlfuncs.StandardFuncs[ottllog.TransformContext](), settings)
+			assert.NoError(t, err)
+			valueExpr, err := logParser.ParseValueExpression(tt.statement)
+			assert.NoError(t, err)
+
+			tCtx := constructLogTransformContext()
+			val, err := valueExpr.Eval(context.Background(), tCtx)
+			assert.NoError(t, err)
+
+			assert.Equal(t, tt.want, val)
+		})
+	}
+}
+
 func Test_ProcessTraces_TraceContext(t *testing.T) {
 	tests := []struct {
 		statement string

pkg/ottl/grammar.go

@@ -245,6 +245,12 @@ type value struct {
 	List           *list            `parser:"| @@)"`
 }
 
+func (v *value) checkForCustomError() error {
+	validator := &grammarCustomErrorsVisitor{}
+	v.accept(validator)
+	return validator.join()
+}
+
 func (v *value) accept(vis grammarVisitor) {
 	vis.visitValue(v)
 	if v.Literal != nil {

pkg/ottl/parser.go

@@ -232,8 +232,9 @@ func (p *Parser[K]) prependContextToStatementPaths(context string, statement str
 }
 
 var (
-	parser          = newParser[parsedStatement]()
-	conditionParser = newParser[booleanExpression]()
+	parser                = newParser[parsedStatement]()
+	conditionParser       = newParser[booleanExpression]()
+	valueExpressionParser = newParser[value]()
 )
 
 func parseStatement(raw string) (*parsedStatement, error) {
@@ -262,6 +263,19 @@ func parseCondition(raw string) (*booleanExpression, error) {
 	return parsed, nil
 }
 
+func parseValueExpression(raw string) (*value, error) {
+	parsed, err := valueExpressionParser.ParseString("", raw)
+	if err != nil {
+		return nil, fmt.Errorf("expression has invalid syntax: %w", err)
+	}
+	err = parsed.checkForCustomError()
+	if err != nil {
+		return nil, err
+	}
+
+	return parsed, nil
+}
+
 func insertContextIntoStatementOffsets(context string, statement string, offsets []int) (string, error) {
 	if len(offsets) == 0 {
 		return statement, nil
@@ -439,3 +453,33 @@ func (c *ConditionSequence[K]) Eval(ctx context.Context, tCtx K) (bool, error) {
 	// It is not possible to get here if any condition during an AND explicitly failed.
 	return c.logicOp == And && atLeastOneMatch, nil
 }
+
+// ValueExpression represents an expression that resolves to a value. The returned value can be of any type,
+// and the expression can be either a literal value, a path value within the context, or the result of a converter and/or
+// a mathematical expression.
+// This allows other components using this library to extract data from the context of the incoming signal using OTTL.
+type ValueExpression[K any] struct {
+	getter Getter[K]
+}
+
+// Eval evaluates the given expression and returns the value the expression resolves to.
+func (e *ValueExpression[K]) Eval(ctx context.Context, tCtx K) (any, error) {
+	return e.getter.Get(ctx, tCtx)
+}
+
+// ParseValueExpression parses an expression string into a ValueExpression. The ValueExpression's Eval
+// method can then be used to extract the value from the context of the incoming signal.
+func (p *Parser[K]) ParseValueExpression(raw string) (*ValueExpression[K], error) {
+	parsed, err := parseValueExpression(raw)
+	if err != nil {
+		return nil, err
+	}
+	getter, err := p.newGetter(*parsed)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ValueExpression[K]{
+		getter: getter,
+	}, nil
+}

pkg/ottl/parser_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/pdata/pcommon"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottltest"
 )
@@ -2130,6 +2131,118 @@ func testParseEnum(val *EnumSymbol) (*Enum, error) {
 	return nil, fmt.Errorf("enum symbol not provided")
 }
 
+func Test_parseValueExpression_full(t *testing.T) {
+	time1 := time.Now()
+	time2 := time1.Add(5 * time.Second)
+	tests := []struct {
+		name            string
+		valueExpression string
+		tCtx            any
+		expected        func() any
+	}{
+		{
+			name:            "string value",
+			valueExpression: `"fido"`,
+			expected: func() any {
+				return "fido"
+			},
+		},
+		{
+			name:            "resolve context value",
+			valueExpression: `attributes`,
+			expected: func() any {
+				return map[string]any{
+					"attributes": map[string]any{
+						"foo": "bar",
+					},
+				}
+			},
+			tCtx: map[string]any{
+				"attributes": map[string]any{
+					"foo": "bar",
+				},
+			},
+		},
+		{
+			name:            "resolve math expression",
+			valueExpression: `time2 - time1`,
+			expected: func() any {
+				return 5 * time.Second
+			},
+			tCtx: map[string]time.Time{
+				"time1": time1,
+				"time2": time2,
+			},
+		},
+		{
+			name:            "nil",
+			valueExpression: `nil`,
+			expected: func() any {
+				return nil
+			},
+		},
+		{
+			name:            "string",
+			valueExpression: `"string"`,
+			expected: func() any {
+				return "string"
+			},
+		},
+		{
+			name:            "hex values",
+			valueExpression: `[0x0000000000000000, 0x0000000000000000]`,
+			expected: func() any {
+				return []any{
+					[]uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
+					[]uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
+				}
+			},
+		},
+		{
+			name:            "boolean",
+			valueExpression: `true`,
+			expected: func() any {
+				return true
+			},
+		},
+		{
+			name:            "map",
+			valueExpression: `{"map": 1}`,
+			expected: func() any {
+				m := pcommon.NewMap()
+				_ = m.FromRaw(map[string]any{
+					"map": 1,
+				})
+				return m
+			},
+		},
+		{
+			name:            "string list",
+			valueExpression: `["list", "of", "strings"]`,
+			expected: func() any {
+				return []any{"list", "of", "strings"}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.valueExpression, func(t *testing.T) {
+			p, _ := NewParser(
+				CreateFactoryMap[any](),
+				testParsePath[any],
+				componenttest.NewNopTelemetrySettings(),
+				WithEnumParser[any](testParseEnum),
+			)
+			parsed, err := p.ParseValueExpression(tt.valueExpression)
+			assert.NoError(t, err)
+
+			v, err := parsed.Eval(context.Background(), tt.tCtx)
+			require.NoError(t, err)
+			assert.Equal(t, tt.expected(), v)
+		})
+	}
+}
+
 func Test_ParseStatements_Error(t *testing.T) {
 	statements := []string{
 		`set(`,
@@ -2343,6 +2456,44 @@ func Test_parseCondition(t *testing.T) {
 	}
 }
 
+// This test doesn't validate parser results, simply checks whether the parse succeeds or not.
+// It's a fast way to check a large range of possible syntaxes.
+func Test_parseValueExpression(t *testing.T) {
+	converterNameErrorPrefix := "converter names must start with an uppercase letter"
+	editorWithIndexErrorPrefix := "only paths and converters may be indexed"
+
+	tests := []struct {
+		valueExpression   string
+		wantErr           bool
+		wantErrContaining string
+	}{
+		{valueExpression: `time_end - time_end`},
+		{valueExpression: `time_end - time_end - attributes["foo"]`},
+		{valueExpression: `Test("foo")`},
+		{valueExpression: `Test(Test("foo")) - attributes["bar"]`},
+		{valueExpression: `Test(Test("foo")) - attributes["bar"]"`, wantErr: true},
+		{valueExpression: `test("foo")`, wantErr: true, wantErrContaining: converterNameErrorPrefix},
+		{valueExpression: `test(animal)["kind"]`, wantErrContaining: editorWithIndexErrorPrefix},
+		{valueExpression: `Test("a"")foo"`, wantErr: true},
+		{valueExpression: `Test("a"") == 1"`, wantErr: true},
+	}
+	pat := regexp.MustCompile("[^a-zA-Z0-9]+")
+	for _, tt := range tests {
+		name := pat.ReplaceAllString(tt.valueExpression, "_")
+		t.Run(name, func(t *testing.T) {
+			ast, err := parseValueExpression(tt.valueExpression)
+			if (err != nil) != (tt.wantErr || tt.wantErrContaining != "") {
+				t.Errorf("parseCondition(%s) error = %v, wantErr %v", tt.valueExpression, err, tt.wantErr)
+				t.Errorf("AST: %+v", ast)
+				return
+			}
+			if tt.wantErrContaining != "" {
+				require.ErrorContains(t, err, tt.wantErrContaining)
+			}
+		})
+	}
+}
+
 func Test_Statement_Execute(t *testing.T) {
 	tests := []struct {
 		name              string