feat:(instrumentation-http) Capture Synthetic Source Type on Requests

CHANGELOG.md

@@ -144,6 +144,7 @@ For semantic convention package changes, see the [semconv CHANGELOG](packages/se
 * feat(core): add more scalable replacements for getEnv(), getEnvWithoutDefaults() [#5443](/~https://github.com/open-telemetry/opentelemetry-js/pull/5443) @pichlermarc
 * refactor(exporter-jaeger): migrate away from getEnv() [#5464](/~https://github.com/open-telemetry/opentelemetry-js/pull/5464) @pichlermarc
 * feat(core): add `diagLogLevelFromString` utility [#5475](/~https://github.com/open-telemetry/opentelemetry-js/pull/5475) @pichlermarc
+* feat(instrumentation-http): capture synthetic source type on requests [#5488](/~https://github.com/open-telemetry/opentelemetry-js/pull/5488) @JacksonWeber
 
 ### :bug: (Bug Fix)
 

experimental/packages/opentelemetry-instrumentation-http/src/http.ts

@@ -794,7 +794,8 @@ export class HttpInstrumentation extends InstrumentationBase<HttpInstrumentation
             instrumentation.getConfig().startOutgoingSpanHook
           ),
         },
-        instrumentation._semconvStability
+        instrumentation._semconvStability,
+        instrumentation.getConfig().enableSyntheticSourceDetection || false
       );
 
       const startTime = hrTime();

experimental/packages/opentelemetry-instrumentation-http/src/internal-types.ts

@@ -62,3 +62,13 @@ export const enum SemconvStability {
   /** Emit both stable and old semantic conventions*/
   DUPLICATE = 0x1 | 0x2,
 }
+
+/**
+ * Names of possible synthetic test sources.
+ */
+export const SYNTHETIC_TEST_NAMES = ['alwayson'];
+
+/**
+ * Names of possible synthetic bot sources.
+ */
+export const SYNTHETIC_BOT_NAMES = ['googlebot', 'bingbot'];

experimental/packages/opentelemetry-instrumentation-http/src/semconv.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This file contains a copy of unstable semantic convention definitions
+ * used by this package.
+ * @see /~https://github.com/open-telemetry/opentelemetry-js/tree/main/semantic-conventions#unstable-semconv
+ */
+
+/**
+ * Specifies the category of synthetic traffic, such as tests or bots.
+ *
+ * @note This attribute **MAY** be derived from the contents of the `user_agent.original` attribute. Components that populate the attribute are responsible for determining what they consider to be synthetic bot or test traffic. This attribute can either be set for self-identification purposes, or on telemetry detected to be generated as a result of a synthetic request. This attribute is useful for distinguishing between genuine client traffic and synthetic traffic generated by bots or tests.
+ *
+ * @experimental This attribute is experimental and is subject to breaking changes in minor releases of `@opentelemetry/semantic-conventions`.
+ */
+export const ATTR_USER_AGENT_SYNTHETIC_TYPE =
+  'user_agent.synthetic.type' as const;
+
+/**
+ * Enum value "bot" for attribute {@link ATTR_USER_AGENT_SYNTHETIC_TYPE}.
+ */
+export const USER_AGENT_SYNTHETIC_TYPE_VALUE_BOT = 'bot' as const;
+
+/**
+ * Enum value "test" for attribute {@link ATTR_USER_AGENT_SYNTHETIC_TYPE}.
+ */
+export const USER_AGENT_SYNTHETIC_TYPE_VALUE_TEST = 'test' as const;

experimental/packages/opentelemetry-instrumentation-http/src/types.ts

@@ -87,4 +87,6 @@ export interface HttpInstrumentationConfig extends InstrumentationConfig {
     client?: { requestHeaders?: string[]; responseHeaders?: string[] };
     server?: { requestHeaders?: string[]; responseHeaders?: string[] };
   };
+  /** Enable automatic population of synthetic source type based on the user-agent header */
+  enableSyntheticSourceDetection?: boolean;
 }

experimental/packages/opentelemetry-instrumentation-http/src/utils.ts

@@ -77,7 +77,14 @@ import {
   IgnoreMatcher,
   ParsedRequestOptions,
   SemconvStability,
+  SYNTHETIC_BOT_NAMES,
+  SYNTHETIC_TEST_NAMES,
 } from './internal-types';
+import {
+  ATTR_USER_AGENT_SYNTHETIC_TYPE,
+  USER_AGENT_SYNTHETIC_TYPE_VALUE_BOT,
+  USER_AGENT_SYNTHETIC_TYPE_VALUE_TEST,
+} from './semconv';
 import forwardedParse = require('forwarded-parse');
 
 /**
@@ -442,7 +449,8 @@ export const getOutgoingRequestAttributes = (
     port: string | number;
     hookAttributes?: Attributes;
   },
-  semconvStability: SemconvStability
+  semconvStability: SemconvStability,
+  enableSyntheticSourceDetection: boolean
 ): Attributes => {
   const hostname = options.hostname;
   const port = options.port;
@@ -485,6 +493,24 @@ export const getOutgoingRequestAttributes = (
     oldAttributes[SEMATTRS_HTTP_USER_AGENT] = userAgent;
   }
 
+  if (userAgent != null && enableSyntheticSourceDetection) {
+    const userAgentString: string = String(userAgent).toLowerCase();
+    for (const name of SYNTHETIC_TEST_NAMES) {
+      if (userAgentString.includes(name)) {
+        newAttributes[ATTR_USER_AGENT_SYNTHETIC_TYPE] =
+          USER_AGENT_SYNTHETIC_TYPE_VALUE_TEST;
+        break;
+      }
+    }
+    for (const name of SYNTHETIC_BOT_NAMES) {
+      if (userAgentString.includes(name)) {
+        newAttributes[ATTR_USER_AGENT_SYNTHETIC_TYPE] =
+          USER_AGENT_SYNTHETIC_TYPE_VALUE_BOT;
+        break;
+      }
+    }
+  }
+
   switch (semconvStability) {
     case SemconvStability.STABLE:
       return Object.assign(newAttributes, options.hookAttributes);
@@ -842,6 +868,24 @@ export const getIncomingRequestAttributes = (
     newAttributes[ATTR_HTTP_REQUEST_METHOD_ORIGINAL] = method;
   }
 
+  if (userAgent != null) {
+    const userAgentString: string = String(userAgent).toLowerCase();
+    for (const name of SYNTHETIC_TEST_NAMES) {
+      if (userAgentString.includes(name)) {
+        newAttributes[ATTR_USER_AGENT_SYNTHETIC_TYPE] =
+          USER_AGENT_SYNTHETIC_TYPE_VALUE_TEST;
+        break;
+      }
+    }
+    for (const name of SYNTHETIC_BOT_NAMES) {
+      if (userAgentString.includes(name)) {
+        newAttributes[ATTR_USER_AGENT_SYNTHETIC_TYPE] =
+          USER_AGENT_SYNTHETIC_TYPE_VALUE_BOT;
+        break;
+      }
+    }
+  }
+
   const oldAttributes: Attributes = {
     [SEMATTRS_HTTP_URL]: parsedUrl.toString(),
     [SEMATTRS_HTTP_HOST]: host,

experimental/packages/opentelemetry-instrumentation-http/test/functionals/utils.test.ts

@@ -28,6 +28,7 @@ import {
   SEMATTRS_HTTP_RESPONSE_CONTENT_LENGTH_UNCOMPRESSED,
   SEMATTRS_HTTP_ROUTE,
   SEMATTRS_HTTP_TARGET,
+  ATTR_USER_AGENT_ORIGINAL,
 } from '@opentelemetry/semantic-conventions';
 import * as assert from 'assert';
 import { IncomingMessage, ServerResponse } from 'http';
@@ -39,6 +40,10 @@ import {
   ParsedRequestOptions,
   SemconvStability,
 } from '../../src/internal-types';
+import {
+  ATTR_USER_AGENT_SYNTHETIC_TYPE,
+  USER_AGENT_SYNTHETIC_TYPE_VALUE_BOT,
+} from '../../src/semconv';
 import * as utils from '../../src/utils';
 import { AttributeNames } from '../../src/enums/AttributeNames';
 import { RPCType, setRPCMetadata } from '@opentelemetry/core';
@@ -463,6 +468,30 @@ describe('Utility', () => {
       );
       assert.strictEqual(attributes[SEMATTRS_HTTP_TARGET], '/user/?q=val');
     });
+
+    it('should set synthetic attributes on requests', () => {
+      const request = {
+        url: 'http://hostname/user/:id',
+        method: 'GET',
+        socket: {},
+      } as IncomingMessage;
+      request.headers = {
+        'user-agent': 'Googlebot',
+      };
+      const attributes = utils.getIncomingRequestAttributes(
+        request,
+        {
+          component: 'http',
+          semconvStability: SemconvStability.STABLE,
+        },
+        diag
+      );
+      assert.strictEqual(attributes[ATTR_USER_AGENT_ORIGINAL], 'Googlebot');
+      assert.strictEqual(
+        attributes[ATTR_USER_AGENT_SYNTHETIC_TYPE],
+        USER_AGENT_SYNTHETIC_TYPE_VALUE_BOT
+      );
+    });
   });
 
   describe('headers to span attributes capture', () => {