FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fix(langchain): Use `gen_ai.tool.definitions` attribute when data col… · getsentry/sentry-python@8008b9c · GitHub

/ sentry-python Public

Commit 8008b9c

Browse files
ericapisani
authored and
sentrivana
committed
fix(langchain): Use gen_ai.tool.definitions attribute when data collection is enabled (#7204)
When `data_collection` is configured, tool definitions should be set on `GEN_AI_TOOL_DEFINITIONS` instead of the legacy `GEN_AI_REQUEST_AVAILABLE_TOOLS` attribute, matching the updated gen-ai semantic conventions. The legacy attribute is still used when `data_collection` is not configured. Also gate the response tool calls recorded in `on_llm_end` on the `outputs` setting rather than `inputs`, since they are part of the model's response. Refs PY-2734 Refs #7200
1 parent 7ca837a commit 8008b9c

2 files changed

Lines changed: 41 additions & 27 deletions

File tree

‎sentry_sdk/integrations/langchain.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ def _get_ai_system(all_params: "Dict[str, Any]") -> "Optional[str]":
149149

150150
DATA_FIELDS = {
151151
"frequency_penalty": SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY,
152+
# "function_call" is an OpenAI convention for the now-legacy Chat Completions API field
152153
"function_call": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
153154
"max_tokens": SPANDATA.GEN_AI_REQUEST_MAX_TOKENS,
154155
"presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY,
@@ -422,6 +423,12 @@ def on_llm_start(
422423

423424
for key, attribute in DATA_FIELDS.items():
424425
if key in all_params and all_params[key] is not None:
426+
# This is correctly gated on "inputs" at the moment because the
427+
# "on_llm_start" method is the start of a request.
428+
#
429+
# TODO: GEN_AI_RESPONSE_TOOL_CALLS will need to be
430+
# transitioned to non-deprecated tool call attributes
431+
425432
if (
426433
attribute == SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS
427434
and has_data_collection_enabled(client.options)
@@ -521,6 +528,11 @@ def on_chat_model_start(
521528
for key, attribute in DATA_FIELDS.items():
522529
if key in all_params and all_params[key] is not None:
523530
if (
531+
# This is correctly gated on "inputs" at the moment because the
532+
# "on_chat_model_start" method is the start of a request.
533+
#
534+
# TODO: GEN_AI_RESPONSE_TOOL_CALLS will need to be
535+
# transitioned to non-deprecated tool call attributes
524536
attribute == SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS
525537
and has_data_collection_enabled(client.options)
526538
and not client.options["data_collection"]["gen_ai"]["inputs"]
@@ -620,14 +632,11 @@ def on_llm_end(
620632

621633
client = sentry_sdk.get_client()
622634

623-
record_inputs = False
624635
record_outputs = False
625636
if has_data_collection_enabled(client.options):
626-
record_inputs = client.options["data_collection"]["gen_ai"]["inputs"]
627637
record_outputs = client.options["data_collection"]["gen_ai"]["outputs"]
628638
elif should_send_default_pii() and self.include_prompts:
629639
# TODO: Remove this branch once `send_default_pii` is deprecated
630-
record_inputs = True
631640
record_outputs = True
632641

633642
try:
@@ -652,7 +661,7 @@ def on_llm_end(
652661
if response_model is not None:
653662
set_on_span(SPANDATA.GEN_AI_RESPONSE_MODEL, response_model)
654663

655-
if record_inputs:
664+
if record_outputs:
656665
tool_calls = getattr(generation.message, "tool_calls", None)
657666
if tool_calls is not None and tool_calls != []:
658667
set_data_normalized(
@@ -1053,17 +1062,19 @@ def _set_tools_on_span(span: "Union[Span, StreamedSpan]", tools: "Any") -> None:
10531062
return
10541063

10551064
client = sentry_sdk.get_client()
1065+
attribute_name = SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS
10561066
if has_data_collection_enabled(client.options):
10571067
if not client.options["data_collection"]["gen_ai"]["inputs"]:
10581068
return
1059-
1069+
else:
1070+
attribute_name = SPANDATA.GEN_AI_TOOL_DEFINITIONS
10601071
# Before data collection was introduced this was set unconditionally, so it
10611072
# stays that way when data collection is not configured.
10621073
simplified_tools = _simplify_langchain_tools(tools)
10631074
if simplified_tools:
10641075
set_data_normalized(
10651076
span,
1066-
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
1077+
attribute_name,
10671078
simplified_tools,
10681079
unpack=False,
10691080
)

‎tests/integrations/langchain/test_langchain.py‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6497,10 +6497,10 @@ def test_langchain_text_completion_data_collection(
64976497
False,
64986498
False,
64996499
[
6500-
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
6500+
SPANDATA.GEN_AI_TOOL_DEFINITIONS,
65016501
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
65026502
],
6503-
[],
6503+
[SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS],
65046504
id="gen-ai-inputs-and-outputs-enabled-tools-collected",
65056505
),
65066506
pytest.param(
@@ -6509,6 +6509,7 @@ def test_langchain_text_completion_data_collection(
65096509
True,
65106510
[],
65116511
[
6512+
SPANDATA.GEN_AI_TOOL_DEFINITIONS,
65126513
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
65136514
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
65146515
],
@@ -6518,33 +6519,34 @@ def test_langchain_text_completion_data_collection(
65186519
{"gen_ai": {"inputs": True, "outputs": False}},
65196520
False,
65206521
False,
6522+
[SPANDATA.GEN_AI_TOOL_DEFINITIONS],
65216523
[
6524+
# REQUEST_AVAILABLE_TOOLS is the legacy value set when data collection is not enabled
65226525
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
65236526
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
65246527
],
6525-
[],
6526-
id="gen-ai-inputs-enabled-outputs-disabled-tools-collected",
6528+
id="gen-ai-inputs-enabled-outputs-disabled-only-tool-definitions-collected",
65276529
),
65286530
pytest.param(
65296531
{"gen_ai": {"inputs": False, "outputs": True}},
65306532
True,
65316533
True,
6532-
[],
6534+
[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS],
65336535
[
6536+
SPANDATA.GEN_AI_TOOL_DEFINITIONS,
65346537
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
6535-
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
65366538
],
6537-
id="gen-ai-outputs-enabled-inputs-disabled-tools-not-collected",
6539+
id="gen-ai-outputs-enabled-inputs-disabled-only-response-tool-calls-collected",
65386540
),
65396541
pytest.param(
65406542
{"gen_ai": {}},
65416543
False,
65426544
False,
65436545
[
6544-
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
6546+
SPANDATA.GEN_AI_TOOL_DEFINITIONS,
65456547
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
65466548
],
6547-
[],
6549+
[SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS],
65486550
id="gen-ai-inputs-and-outputs-omitted-default-to-enabled",
65496551
),
65506552
pytest.param(
@@ -6555,15 +6557,18 @@ def test_langchain_text_completion_data_collection(
65556557
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
65566558
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
65576559
],
6558-
[],
6560+
[SPANDATA.GEN_AI_TOOL_DEFINITIONS],
65596561
id="no-gen-ai-config-legacy-pii-and-include-prompts-enabled",
65606562
),
65616563
pytest.param(
65626564
None,
65636565
False,
65646566
False,
65656567
[SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS],
6566-
[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS],
6568+
[
6569+
SPANDATA.GEN_AI_TOOL_DEFINITIONS,
6570+
SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
6571+
],
65676572
id="no-gen-ai-config-available-tools-collected-regardless-of-pii",
65686573
),
65696574
],
@@ -6664,16 +6669,14 @@ def test_langchain_data_collection_tools(
66646669
for key in expected_absent:
66656670
assert key not in chat_spans[0], f"{key} should not have been collected"
66666671

6667-
# Available tools are gated the same way on every span they are set on
6668-
available_tools_collected = (
6669-
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in expected_present
6670-
)
6671-
assert (
6672-
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in chat_spans[1]
6673-
) is available_tools_collected
6674-
assert (
6675-
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in invoke_agent_span
6676-
) is available_tools_collected
6672+
# Tool definitions are gated the same way on every span they are set on
6673+
for key in (
6674+
SPANDATA.GEN_AI_TOOL_DEFINITIONS,
6675+
SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS,
6676+
):
6677+
collected = key in expected_present
6678+
assert (key in chat_spans[1]) is collected
6679+
assert (key in invoke_agent_span) is collected
66776680

66786681

66796682
@pytest.mark.parametrize("span_streaming", [True, False])

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL