diff --git a/instrumentation/opentelemetry-instrumentation-openai/README.rst b/instrumentation/opentelemetry-instrumentation-openai/README.rst index 02e5922127..d3f2873622 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/README.rst +++ b/instrumentation/opentelemetry-instrumentation-openai/README.rst @@ -15,7 +15,7 @@ Installation :: - pip install opentelemetry-instrumentation-openai + pip install opentelemetry-instrumentation-openai-v2 References diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index ed3e1f53f7..9f08f11027 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -37,13 +37,13 @@ instruments = [ ] [project.entry-points.opentelemetry_instrumentor] -openai = "opentelemetry.instrumentation.openai:OpenAIInstrumentor" +openai = "opentelemetry.instrumentation.openai_v2:OpenAIInstrumentor" [project.urls] Homepage = "/~https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-openai" [tool.hatch.version] -path = "src/opentelemetry/instrumentation/openai/version.py" +path = "src/opentelemetry/instrumentation/openai_v2/version.py" [tool.hatch.build.targets.sdist] include = [ diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py similarity index 96% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py index 439f1e184c..85ab2fd4fd 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -46,7 +46,7 @@ from wrapt import wrap_function_wrapper from opentelemetry.instrumentation.instrumentor import BaseInstrumentor -from opentelemetry.instrumentation.openai.package import _instruments +from opentelemetry.instrumentation.openai_v2.package import _instruments from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import get_tracer diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/package.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/package.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/patch.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/patch.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/utils.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/utils.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml new file mode 100644 index 0000000000..34c5f81bef --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}], + "model": "gpt-4", "stream": false}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '112' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.47.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.47.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//hJBBS8NAEIXv+RXDntuSpLXV3BQPIihSxItI2CbTZO1md81MoFr632W3 + MakH8RLI+/Y93rxDBCBUKTIQRS25aJyeXt/cN+r26bH72q9f4v0Dzuu79a78KPQllWLiHXbzjgX/ + uGaFbZxGVtaccNGiZPSpySpdpcskThcBNLZE7W2V4+liGi+Tee+orSqQRAavEQDAIXx9N1PiXmQQ + T36UBolkhSIbHgGI1mqvCEmkiKVhMRlhYQ2jCXWfa0WgCCQwEs/gn//zlBa3HUnf3nRa9/pxqKVt + 5Vq7oZ4P+lYZRXXeoiRrfAVi60SgxwjgLZzf/bpIuNY2jnO2OzQ+MFmc4sQ49Bm86CFblnrU06u/ + THmJLJWmswnFqZ8y1RgQDyXDlYI+ibHJt8pU2LpWhUXDFsfoGwAA//8DAOtohWdIAgAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c8a6a3c3e5f11ac-MRS + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 25 Sep 2024 10:43:45 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=oXnQwjslhtdFXjG89Oureoj0ycU2U3.JD0YOmXVf7Oo-1727261025-1.0.1.1-8zkkMufvVyON_EWorQBeCtOhIav5dyIQ7s5UoEMu2gTW.uaDA3owAxnO_LwCkccXJhC56ryfDhKmS49nV855yA; + path=/; expires=Wed, 25-Sep-24 11:13:45 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=YSEnhEKHk_5duU.2X6td9ALnCbun1O0M.YvR4OF5DgU-1727261025973-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + openai-organization: + - scale3-1 + openai-processing-ms: + - '1074' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '1000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 1ms + x-request-id: + - req_c4bc5f6decb9ecc1d59dd8e9435531bf + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml new file mode 100644 index 0000000000..5859360bd7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml @@ -0,0 +1,157 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}], + "model": "gpt-4", "stream": true, "stream_options": {"include_usage": true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '154' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.47.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.47.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"This"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + test"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + This"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + test"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + This"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + test"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":14,"completion_tokens":15,"total_tokens":29,"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c8a8f963be411a4-MRS + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 25 Sep 2024 11:09:15 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=hvo156nahnyIUFXHe9iuYr0tn0dKzveWlQN7suEYz9Q-1727262555-1.0.1.1-L1wMbo_r0VTMdA..XHJ_8JDmEIjnuzOW_umwSN1y.LlARvkoK3fluYzgsPa5W1Wd_.Hx0yB__0kriyR1pszyDw; + path=/; expires=Wed, 25-Sep-24 11:39:15 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=imSGB7bQZURuXPnMTuFDyO6GqwZI2ELFF_y9AyZcOwk-1727262555002-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + openai-organization: + - scale3-1 + openai-processing-ms: + - '195' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '1000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 1ms + x-request-id: + - req_49e07a9a2909db3c1d58e009add4f3ad + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py new file mode 100644 index 0000000000..0ed9279c9d --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py @@ -0,0 +1,52 @@ +"""Unit tests configuration module.""" + +import os + +import pytest +from openai import OpenAI + +from opentelemetry import trace +from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +@pytest.fixture(scope="session") +def exporter(): + exporter = InMemorySpanExporter() + processor = SimpleSpanProcessor(exporter) + + provider = TracerProvider() + provider.add_span_processor(processor) + trace.set_tracer_provider(provider) + + return exporter + + +@pytest.fixture(autouse=True) +def clear_exporter(exporter): + exporter.clear() + + +@pytest.fixture(autouse=True) +def environment(): + if not os.getenv("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = "test-api-key" + + +@pytest.fixture +def openai_client(): + return OpenAI() + + +@pytest.fixture(scope="module") +def vcr_config(): + return {"filter_headers": ["authorization", "api-key"]} + + +@pytest.fixture(scope="session", autouse=True) +def instrument(): + OpenAIInstrumentor().instrument() diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py b/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py new file mode 100644 index 0000000000..f298a018a6 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py @@ -0,0 +1,162 @@ +import json + +import pytest + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + + +@pytest.mark.vcr() +def test_chat_completion(exporter, openai_client): + llm_model_value = "gpt-4" + messages_value = [ + {"role": "user", "content": "Say this is a test three times"} + ] + + kwargs = { + "model": llm_model_value, + "messages": messages_value, + "stream": False, + } + + response = openai_client.chat.completions.create(**kwargs) + spans = exporter.get_finished_spans() + chat_completion_span = spans[-1] + # assert that the span name is correct + assert chat_completion_span.name == f"chat {llm_model_value}" + + attributes = chat_completion_span.attributes + operation_name = attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + system = attributes[GenAIAttributes.GEN_AI_SYSTEM] + request_model = attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] + response_model = attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + response_id = attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + input_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + output_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + + # assert that the attributes are correct + assert ( + operation_name == GenAIAttributes.GenAiOperationNameValues.CHAT.value + ) + assert system == GenAIAttributes.GenAiSystemValues.OPENAI.value + assert request_model == llm_model_value + assert response_model == response.model + assert response_id == response.id + assert input_tokens == response.usage.prompt_tokens + assert output_tokens == response.usage.completion_tokens + + events = chat_completion_span.events + + # assert that the prompt and completion events are present + prompt_event = list( + filter( + lambda event: event.name == "gen_ai.content.prompt", + events, + ) + ) + completion_event = list( + filter( + lambda event: event.name == "gen_ai.content.completion", + events, + ) + ) + + assert prompt_event + assert completion_event + + # assert that the prompt and completion events have the correct attributes + assert prompt_event[0].attributes[ + GenAIAttributes.GEN_AI_PROMPT + ] == json.dumps(messages_value) + + assert ( + json.loads( + completion_event[0].attributes[GenAIAttributes.GEN_AI_COMPLETION] + )[0]["content"] + == response.choices[0].message.content + ) + + +@pytest.mark.vcr() +def test_chat_completion_streaming(exporter, openai_client): + llm_model_value = "gpt-4" + messages_value = [ + {"role": "user", "content": "Say this is a test three times"} + ] + + kwargs = { + "model": llm_model_value, + "messages": messages_value, + "stream": True, + "stream_options": {"include_usage": True}, + } + + response_stream_usage = None + response_stream_model = None + response_stream_id = None + response_stream_result = "" + response = openai_client.chat.completions.create(**kwargs) + for chunk in response: + if chunk.choices: + response_stream_result += chunk.choices[0].delta.content or "" + + # get the last chunk + if getattr(chunk, "usage", None): + response_stream_usage = chunk.usage + response_stream_model = chunk.model + response_stream_id = chunk.id + + spans = exporter.get_finished_spans() + streaming_span = spans[-1] + + assert streaming_span.name == f"chat {llm_model_value}" + attributes = streaming_span.attributes + + operation_name = attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + system = attributes[GenAIAttributes.GEN_AI_SYSTEM] + request_model = attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] + response_model = attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + response_id = attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + input_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + output_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + assert ( + operation_name == GenAIAttributes.GenAiOperationNameValues.CHAT.value + ) + assert system == GenAIAttributes.GenAiSystemValues.OPENAI.value + assert request_model == llm_model_value + assert response_model == response_stream_model + assert response_id == response_stream_id + assert input_tokens == response_stream_usage.prompt_tokens + assert output_tokens == response_stream_usage.completion_tokens + + events = streaming_span.events + + # assert that the prompt and completion events are present + prompt_event = list( + filter( + lambda event: event.name == "gen_ai.content.prompt", + events, + ) + ) + completion_event = list( + filter( + lambda event: event.name == "gen_ai.content.completion", + events, + ) + ) + + assert prompt_event + assert completion_event + + # assert that the prompt and completion events have the correct attributes + assert prompt_event[0].attributes[ + GenAIAttributes.GEN_AI_PROMPT + ] == json.dumps(messages_value) + + assert ( + json.loads( + completion_event[0].attributes[GenAIAttributes.GEN_AI_COMPLETION] + )[0]["content"] + == response_stream_result + )