From 776ca0040ed856a6291fb87ba0f3f0d31d1c2b47 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:14:19 +0200 Subject: [PATCH 1/8] feat(langchain): Record cached token usage values --- sentry_sdk/integrations/langchain.py | 76 +++++++++++-- .../integrations/langchain/test_langchain.py | 104 +++++++++++++++++- 2 files changed, 168 insertions(+), 12 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 61b3cda772..267f546578 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -4,7 +4,7 @@ import warnings from collections import OrderedDict from functools import wraps -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple import sentry_sdk from sentry_sdk.ai.utils import ( @@ -50,7 +50,7 @@ Callbacks, manager, ) - from langchain_core.messages import BaseMessage + from langchain_core.messages import AIMessage, BaseMessage from langchain_core.outputs import ( ChatGeneration, ChatGenerationChunk, @@ -63,6 +63,14 @@ raise DidNotEnable("langchain not installed") +class TokenUsage(NamedTuple): + input_tokens: "Optional[int]" + output_tokens: "Optional[int]" + total_tokens: "Optional[int]" + cache_read: "Optional[int]" + cache_creation: "Optional[int]" + + try: # >=v1 from langchain_classic.agents import AgentExecutor # type: ignore[import-not-found] @@ -720,7 +728,7 @@ def _extract_tokens( def _extract_tokens_from_generations( generations: "list[list[Generation | ChatGeneration | GenerationChunk | ChatGenerationChunk]]", -) -> "tuple[Optional[int], Optional[int], Optional[int]]": +) -> "TokenUsage": """Extract token usage from response.generations structure.""" if not generations: return None, None, None @@ -728,6 +736,8 @@ def _extract_tokens_from_generations( total_input = 0 total_output = 0 total_total = 0 + total_cache_read = None + total_cache_creation = None for gen_list in generations: if not gen_list: @@ -739,10 +749,36 @@ def _extract_tokens_from_generations( total_output += output_tokens if output_tokens is not None else 0 total_total += total_tokens if total_tokens is not None else 0 - return ( + if not isinstance(gen_list[0], ChatGeneration): + continue + + message = gen_list[0].message + + if not isinstance(message, AIMessage): + continue + + usage_metadata = message.usage_metadata + + input_token_details = usage_metadata.get("input_token_details") + if not isinstance(input_token_details, dict): + continue + + if "cache_read" in input_token_details: + total_cache_read = (total_cache_read or 0) + input_token_details[ + "cache_read" + ] + + if "cache_creation" in input_token_details: + total_cache_creation = (total_cache_creation or 0) + input_token_details[ + "cache_creation" + ] + + return TokenUsage( total_input if total_input > 0 else None, total_output if total_output > 0 else None, total_total if total_total > 0 else None, + total_cache_read, + total_cache_creation, ) @@ -777,13 +813,29 @@ def _get_token_usage(obj: "Any") -> "Optional[Dict[str, Any]]": def _record_token_usage( span: "Union[Span, StreamedSpan]", response: "LLMResult" ) -> None: + input_tokens = None + output_tokens = None + total_tokens = None + cache_read_tokens = None + cache_creation_tokens = None + + # Legacy that reads provider-specific token information. token_usage = _get_token_usage(response) if token_usage: input_tokens, output_tokens, total_tokens = _extract_tokens(token_usage) - else: - input_tokens, output_tokens, total_tokens = _extract_tokens_from_generations( - response.generations - ) + + # Prefer provider-agnostic UsageMetadata if available. + token_usage = _extract_tokens_from_generations(response.generations) + if token_usage.input_tokens is not None: + input_tokens = token_usage.input_tokens + if token_usage.output_tokens is not None: + output_tokens = token_usage.output_tokens + if token_usage.total_tokens is not None: + total_tokens = token_usage.total_tokens + if token_usage.cache_read is not None: + cache_read_tokens = token_usage.cache_read + if token_usage.cache_creation is not None: + cache_creation_tokens = token_usage.cache_creation set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data @@ -798,6 +850,14 @@ def _record_token_usage( if total_tokens is not None: set_on_span(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, total_tokens) + if cache_read_tokens is not None: + set_on_span(SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read_tokens) + + if cache_creation_tokens is not None: + set_on_span( + SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, cache_creation_tokens + ) + def _get_request_data( obj: "Any", args: "Any", kwargs: "Any" diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 892491bec4..212e0ae0c5 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -268,6 +268,13 @@ def nonstreaming_multi_candidate_google_genai_model_response(): ], model_version="gemini/gemini-pro", usage_metadata=google.genai.types.GenerateContentResponseUsageMetadata( + cache_tokens_details=[ + google.genai.types.ModalityTokenCount( + modality=google.genai.types.Modality.TEXT, + token_count=6, + ) + ], + cached_content_token_count=4, prompt_token_count=10, candidates_token_count=20, total_token_count=30, @@ -592,6 +599,11 @@ def test_langchain_multi_choice_response( assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 + + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == 4 + ) else: events = capture_events() @@ -613,6 +625,8 @@ def test_langchain_multi_choice_response( assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20 assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30 + assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @@ -798,6 +812,17 @@ def test_langchain_create_agent( assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == 4 + ) + assert ( + chat_spans[0]["attributes"][ + SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] + == 6 + ) + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert ( chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" @@ -874,6 +899,17 @@ def test_langchain_create_agent( assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == 4 + ) + assert ( + chat_spans[0]["attributes"][ + SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] + == 6 + ) + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert ( chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" @@ -943,6 +979,12 @@ def test_langchain_create_agent( assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20 assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30 + assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 + assert ( + chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 6 + ) + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" @@ -1013,8 +1055,8 @@ def test_tool_execution_span( ResponseUsage( input_tokens=142, input_tokens_details=InputTokensDetails( - cached_tokens=0, - cache_write_tokens=0, + cached_tokens=69, + cache_write_tokens=31, ), output_tokens=50, output_tokens_details=OutputTokensDetails( @@ -1025,8 +1067,8 @@ def test_tool_execution_span( ResponseUsage( input_tokens=89, input_tokens_details=InputTokensDetails( - cached_tokens=0, - cache_write_tokens=0, + cached_tokens=69, + cache_write_tokens=10, ), output_tokens=28, output_tokens_details=OutputTokensDetails( @@ -1107,11 +1149,31 @@ def test_tool_execution_span( assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == 69 + ) + assert ( + chat_spans[0]["attributes"][ + SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] + == 31 + ) assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == 69 + ) + assert ( + chat_spans[1]["attributes"][ + SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] + == 10 + ) assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): @@ -1220,11 +1282,31 @@ def test_tool_execution_span( assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == 69 + ) + assert ( + chat_spans[0]["attributes"][ + SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] + == 31 + ) assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] + == 69 + ) + assert ( + chat_spans[1]["attributes"][ + SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS + ] + == 10 + ) assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): @@ -1333,11 +1415,25 @@ def test_tool_execution_span( assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 + assert ( + chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 + ) + assert ( + chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 31 + ) assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 + assert ( + chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 + ) + assert ( + chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 10 + ) assert chat_spans[1]["data"]["gen_ai.system"] == "openai-chat" if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): From f0e811452522a111d04b91e1dd38ce3f211bfc7d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:16:13 +0200 Subject: [PATCH 2/8] clean up --- tests/integrations/langchain/test_langchain.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 212e0ae0c5..e0ca232302 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -268,12 +268,6 @@ def nonstreaming_multi_candidate_google_genai_model_response(): ], model_version="gemini/gemini-pro", usage_metadata=google.genai.types.GenerateContentResponseUsageMetadata( - cache_tokens_details=[ - google.genai.types.ModalityTokenCount( - modality=google.genai.types.Modality.TEXT, - token_count=6, - ) - ], cached_content_token_count=4, prompt_token_count=10, candidates_token_count=20, From 2ef53477a9ad99cc7eed6a6750ebf21b3adb0a37 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:18:36 +0200 Subject: [PATCH 3/8] make mypy happy --- sentry_sdk/consts.py | 14 ++++++++++++++ sentry_sdk/integrations/langchain.py | 3 +++ 2 files changed, 17 insertions(+) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index c6db94d780..ef468d9602 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -782,6 +782,20 @@ class SPANDATA: Example: "rainy, 57°F" """ + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read.input_tokens" + """ + The number of cached tokens used to process the AI input (prompt). + Example: 50 + """ + + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = ( + "gen_ai.usage.cache_creation.input_tokens" + ) + """ + The number of tokens written to the cache when processing the AI input (prompt). + Example: 100 + """ + GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens" """ The number of tokens in the input. diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 267f546578..465f6bd4f3 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -759,6 +759,9 @@ def _extract_tokens_from_generations( usage_metadata = message.usage_metadata + if not isinstance(usage_metadata, dict): + continue + input_token_details = usage_metadata.get("input_token_details") if not isinstance(input_token_details, dict): continue From a88a3df65935d0824b69cf436ed4e5eda74883e1 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:20:58 +0200 Subject: [PATCH 4/8] make mypy even happier --- sentry_sdk/integrations/langchain.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 465f6bd4f3..8741d4c28f 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -728,11 +728,8 @@ def _extract_tokens( def _extract_tokens_from_generations( generations: "list[list[Generation | ChatGeneration | GenerationChunk | ChatGenerationChunk]]", -) -> "TokenUsage": +) -> "Optional[TokenUsage]": """Extract token usage from response.generations structure.""" - if not generations: - return None, None, None - total_input = 0 total_output = 0 total_total = 0 @@ -828,17 +825,18 @@ def _record_token_usage( input_tokens, output_tokens, total_tokens = _extract_tokens(token_usage) # Prefer provider-agnostic UsageMetadata if available. - token_usage = _extract_tokens_from_generations(response.generations) - if token_usage.input_tokens is not None: - input_tokens = token_usage.input_tokens - if token_usage.output_tokens is not None: - output_tokens = token_usage.output_tokens - if token_usage.total_tokens is not None: - total_tokens = token_usage.total_tokens - if token_usage.cache_read is not None: - cache_read_tokens = token_usage.cache_read - if token_usage.cache_creation is not None: - cache_creation_tokens = token_usage.cache_creation + if response.generations is not None: + token_usage = _extract_tokens_from_generations(response.generations) + if token_usage.input_tokens is not None: + input_tokens = token_usage.input_tokens + if token_usage.output_tokens is not None: + output_tokens = token_usage.output_tokens + if token_usage.total_tokens is not None: + total_tokens = token_usage.total_tokens + if token_usage.cache_read is not None: + cache_read_tokens = token_usage.cache_read + if token_usage.cache_creation is not None: + cache_creation_tokens = token_usage.cache_creation set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data From 17498bbe96da452d82a6c075a0c3ca256ffcd457 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:22:03 +0200 Subject: [PATCH 5/8] mypy again --- sentry_sdk/integrations/langchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 8741d4c28f..714006629e 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -728,7 +728,7 @@ def _extract_tokens( def _extract_tokens_from_generations( generations: "list[list[Generation | ChatGeneration | GenerationChunk | ChatGenerationChunk]]", -) -> "Optional[TokenUsage]": +) -> "TokenUsage": """Extract token usage from response.generations structure.""" total_input = 0 total_output = 0 From 1d0bc01a85c71e9ecad392c84237b7faa7941fb1 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:49:04 +0200 Subject: [PATCH 6/8] . --- sentry_sdk/integrations/langchain.py | 10 +- tests/conftest.py | 4 +- .../integrations/langchain/test_langchain.py | 133 ------------------ 3 files changed, 7 insertions(+), 140 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 714006629e..a5c8713698 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -742,9 +742,9 @@ def _extract_tokens_from_generations( token_usage = _get_token_usage(gen_list[0]) input_tokens, output_tokens, total_tokens = _extract_tokens(token_usage) - total_input += input_tokens if input_tokens is not None else 0 - total_output += output_tokens if output_tokens is not None else 0 - total_total += total_tokens if total_tokens is not None else 0 + total_input += input_tokens if isinstance(input_tokens, int) else 0 + total_output += output_tokens if isinstance(output_tokens, int) else 0 + total_total += total_tokens if isinstance(total_tokens, int) else 0 if not isinstance(gen_list[0], ChatGeneration): continue @@ -763,12 +763,12 @@ def _extract_tokens_from_generations( if not isinstance(input_token_details, dict): continue - if "cache_read" in input_token_details: + if isinstance(input_token_details.get("cache_read"), int): total_cache_read = (total_cache_read or 0) + input_token_details[ "cache_read" ] - if "cache_creation" in input_token_details: + if isinstance(input_token_details.get("cache_creation"), int): total_cache_creation = (total_cache_creation or 0) + input_token_details[ "cache_creation" ] diff --git a/tests/conftest.py b/tests/conftest.py index f3ae302057..599c075224 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1439,8 +1439,8 @@ def nonstreaming_responses_model_response(): usage=openai.types.responses.ResponseUsage( input_tokens=10, input_tokens_details=openai.types.responses.response_usage.InputTokensDetails( - cached_tokens=0, - cache_write_tokens=0, + cached_tokens=4, + cache_write_tokens=6, ), output_tokens=20, output_tokens_details=openai.types.responses.response_usage.OutputTokensDetails( diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index e0ca232302..a6711f8157 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -5793,139 +5793,6 @@ def test_transform_google_file_data(self): } -@pytest.mark.parametrize("span_streaming", [True, False]) -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -@pytest.mark.parametrize( - "ai_type,expected_system", - [ - # Real LangChain _type values (from _llm_type properties) - # OpenAI - ("openai-chat", "openai-chat"), - ("openai", "openai"), - # Azure OpenAI - ("azure-openai-chat", "azure-openai-chat"), - ("azure", "azure"), - # Anthropic - ("anthropic-chat", "anthropic-chat"), - # Google - ("vertexai", "vertexai"), - ("chat-google-generative-ai", "chat-google-generative-ai"), - ("google_gemini", "google_gemini"), - # AWS Bedrock - ("amazon_bedrock_chat", "amazon_bedrock_chat"), - ("amazon_bedrock", "amazon_bedrock"), - # Cohere - ("cohere-chat", "cohere-chat"), - # Ollama - ("chat-ollama", "chat-ollama"), - ("ollama-llm", "ollama-llm"), - # Mistral - ("mistralai-chat", "mistralai-chat"), - # Fireworks - ("fireworks-chat", "fireworks-chat"), - ("fireworks", "fireworks"), - # HuggingFace - ("huggingface-chat-wrapper", "huggingface-chat-wrapper"), - # Groq - ("groq-chat", "groq-chat"), - # NVIDIA - ("chat-nvidia-ai-playground", "chat-nvidia-ai-playground"), - # xAI - ("xai-chat", "xai-chat"), - # DeepSeek - ("chat-deepseek", "chat-deepseek"), - # Edge cases - ("", None), - (None, None), - ], -) -def test_langchain_ai_system_detection( - sentry_init, - capture_events, - capture_items, - ai_type, - expected_system, - stream_gen_ai_spans, - span_streaming, -): - sentry_init( - integrations=[LangchainIntegration()], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - stream_gen_ai_spans=stream_gen_ai_spans, - trace_lifecycle="stream" if span_streaming else "static", - ) - - callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) - - run_id = "test-ai-system-uuid" - serialized = {"_type": ai_type} if ai_type is not None else {} - prompts = ["Test prompt"] - - if span_streaming or stream_gen_ai_spans: - items = capture_items("span") - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - invocation_params={"_type": ai_type, "model": "test-model"}, - ) - - generation = Mock(text="Test response", message=None) - response = Mock(generations=[[generation]]) - callback.on_llm_end(response=response, run_id=run_id) - - sentry_sdk.flush() - spans = [item.payload for item in items] - llm_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" - ] - - assert len(llm_spans) > 0 - llm_span = llm_spans[0] - - if expected_system is not None: - assert llm_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == expected_system - else: - assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("attributes", {}) - else: - events = capture_events() - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - invocation_params={"_type": ai_type, "model": "test-model"}, - ) - - generation = Mock(text="Test response", message=None) - response = Mock(generations=[[generation]]) - callback.on_llm_end(response=response, run_id=run_id) - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - llm_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" - ] - - assert len(llm_spans) > 0 - llm_span = llm_spans[0] - - if expected_system is not None: - assert llm_span["data"][SPANDATA.GEN_AI_SYSTEM] == expected_system - else: - assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("data", {}) - - class TestTransformLangchainMessageContent: """Tests for _transform_langchain_message_content function.""" From dfafde16ac2fd6900bc0cbd8efd9d25ee37fc3b2 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:50:33 +0200 Subject: [PATCH 7/8] restore test --- .../integrations/langchain/test_langchain.py | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index a6711f8157..e0ca232302 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -5793,6 +5793,139 @@ def test_transform_google_file_data(self): } +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "ai_type,expected_system", + [ + # Real LangChain _type values (from _llm_type properties) + # OpenAI + ("openai-chat", "openai-chat"), + ("openai", "openai"), + # Azure OpenAI + ("azure-openai-chat", "azure-openai-chat"), + ("azure", "azure"), + # Anthropic + ("anthropic-chat", "anthropic-chat"), + # Google + ("vertexai", "vertexai"), + ("chat-google-generative-ai", "chat-google-generative-ai"), + ("google_gemini", "google_gemini"), + # AWS Bedrock + ("amazon_bedrock_chat", "amazon_bedrock_chat"), + ("amazon_bedrock", "amazon_bedrock"), + # Cohere + ("cohere-chat", "cohere-chat"), + # Ollama + ("chat-ollama", "chat-ollama"), + ("ollama-llm", "ollama-llm"), + # Mistral + ("mistralai-chat", "mistralai-chat"), + # Fireworks + ("fireworks-chat", "fireworks-chat"), + ("fireworks", "fireworks"), + # HuggingFace + ("huggingface-chat-wrapper", "huggingface-chat-wrapper"), + # Groq + ("groq-chat", "groq-chat"), + # NVIDIA + ("chat-nvidia-ai-playground", "chat-nvidia-ai-playground"), + # xAI + ("xai-chat", "xai-chat"), + # DeepSeek + ("chat-deepseek", "chat-deepseek"), + # Edge cases + ("", None), + (None, None), + ], +) +def test_langchain_ai_system_detection( + sentry_init, + capture_events, + capture_items, + ai_type, + expected_system, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[LangchainIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + + callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) + + run_id = "test-ai-system-uuid" + serialized = {"_type": ai_type} if ai_type is not None else {} + prompts = ["Test prompt"] + + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(): + callback.on_llm_start( + serialized=serialized, + prompts=prompts, + run_id=run_id, + invocation_params={"_type": ai_type, "model": "test-model"}, + ) + + generation = Mock(text="Test response", message=None) + response = Mock(generations=[[generation]]) + callback.on_llm_end(response=response, run_id=run_id) + + sentry_sdk.flush() + spans = [item.payload for item in items] + llm_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.text_completion" + ] + + assert len(llm_spans) > 0 + llm_span = llm_spans[0] + + if expected_system is not None: + assert llm_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == expected_system + else: + assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("attributes", {}) + else: + events = capture_events() + + with start_transaction(): + callback.on_llm_start( + serialized=serialized, + prompts=prompts, + run_id=run_id, + invocation_params={"_type": ai_type, "model": "test-model"}, + ) + + generation = Mock(text="Test response", message=None) + response = Mock(generations=[[generation]]) + callback.on_llm_end(response=response, run_id=run_id) + + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" + + llm_spans = [ + span + for span in tx.get("spans", []) + if span.get("op") == "gen_ai.text_completion" + ] + + assert len(llm_spans) > 0 + llm_span = llm_spans[0] + + if expected_system is not None: + assert llm_span["data"][SPANDATA.GEN_AI_SYSTEM] == expected_system + else: + assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("data", {}) + + class TestTransformLangchainMessageContent: """Tests for _transform_langchain_message_content function.""" From b1c3907126fab22327ee6c00310ff159aac79d31 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Fri, 7 Aug 2026 11:59:36 +0200 Subject: [PATCH 8/8] . --- sentry_sdk/integrations/langchain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index a5c8713698..e53851a676 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -751,7 +751,8 @@ def _extract_tokens_from_generations( message = gen_list[0].message - if not isinstance(message, AIMessage): + # The property was added in https://github.com/langchain-ai/langchain/commit/fbfed65fb1ccff3eb8477c4f114450537a0510b2 + if not isinstance(message, AIMessage) or not hasattr(message, "usage_metadata"): continue usage_metadata = message.usage_metadata