2001 lines
67 KiB
Python
2001 lines
67 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from typing import Any, cast
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import httpx2
|
|
import pytest
|
|
|
|
import agents._debug as _debug
|
|
from agents.tracing import flush_traces, get_trace_provider, setup as tracing_setup
|
|
from agents.tracing.processor_interface import TracingExporter, TracingProcessor
|
|
from agents.tracing.processors import BackendSpanExporter, BatchTraceProcessor, ConsoleSpanExporter
|
|
from agents.tracing.provider import DefaultTraceProvider, TraceProvider
|
|
from agents.tracing.span_data import (
|
|
AgentSpanData,
|
|
CustomSpanData,
|
|
FunctionSpanData,
|
|
GenerationSpanData,
|
|
)
|
|
from agents.tracing.spans import Span, SpanImpl
|
|
from agents.tracing.traces import Trace, TraceImpl
|
|
|
|
|
|
def get_span(processor: TracingProcessor) -> SpanImpl[AgentSpanData]:
|
|
"""Create a minimal agent span for testing processors."""
|
|
return SpanImpl(
|
|
trace_id="test_trace_id",
|
|
span_id="test_span_id",
|
|
parent_id=None,
|
|
processor=processor,
|
|
span_data=AgentSpanData(name="test_agent"),
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
|
|
def get_trace(processor: TracingProcessor) -> TraceImpl:
|
|
"""Create a minimal trace."""
|
|
return TraceImpl(
|
|
name="test_trace",
|
|
trace_id="test_trace_id",
|
|
group_id="test_session_id",
|
|
metadata={},
|
|
processor=processor,
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("redacted", [True, False])
|
|
def test_console_span_exporter_respects_data_policy(monkeypatch, capsys, redacted: bool) -> None:
|
|
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
|
|
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted)
|
|
span = get_span(mock_processor())
|
|
span.span_data.name = "SECRET_CONSOLE_SPAN"
|
|
|
|
ConsoleSpanExporter().export([span])
|
|
|
|
output = capsys.readouterr().out
|
|
assert ("SECRET_CONSOLE_SPAN" not in output) is redacted
|
|
assert "Export span" in output
|
|
|
|
|
|
@pytest.fixture
|
|
def mocked_exporter():
|
|
exporter = MagicMock()
|
|
exporter.export = MagicMock()
|
|
return exporter
|
|
|
|
|
|
def test_batch_trace_processor_on_trace_start(mocked_exporter):
|
|
processor = BatchTraceProcessor(exporter=mocked_exporter, schedule_delay=0.1)
|
|
test_trace = get_trace(processor)
|
|
|
|
try:
|
|
with processor._export_lock:
|
|
processor.on_trace_start(test_trace)
|
|
assert processor._queue.qsize() == 1, "Trace should be added to the queue"
|
|
finally:
|
|
processor.shutdown()
|
|
|
|
|
|
def test_batch_trace_processor_on_span_end(mocked_exporter):
|
|
processor = BatchTraceProcessor(exporter=mocked_exporter, schedule_delay=0.1)
|
|
test_span = get_span(processor)
|
|
|
|
try:
|
|
with processor._export_lock:
|
|
processor.on_span_end(test_span)
|
|
assert processor._queue.qsize() == 1, "Span should be added to the queue"
|
|
finally:
|
|
processor.shutdown()
|
|
|
|
|
|
def test_batch_trace_processor_queue_full(mocked_exporter):
|
|
processor = BatchTraceProcessor(exporter=mocked_exporter, max_queue_size=2, schedule_delay=0.1)
|
|
try:
|
|
with processor._export_lock:
|
|
# Fill the queue.
|
|
processor.on_trace_start(get_trace(processor))
|
|
processor.on_trace_start(get_trace(processor))
|
|
assert processor._queue.full() is True
|
|
|
|
# Next item should not be queued.
|
|
processor.on_trace_start(get_trace(processor))
|
|
assert processor._queue.qsize() == 2, "Queue should not exceed max_queue_size"
|
|
|
|
processor.on_span_end(get_span(processor))
|
|
assert processor._queue.qsize() == 2, "Queue should not exceed max_queue_size"
|
|
finally:
|
|
processor.shutdown()
|
|
|
|
|
|
def test_batch_processor_doesnt_enqueue_on_trace_end_or_span_start(mocked_exporter):
|
|
processor = BatchTraceProcessor(exporter=mocked_exporter)
|
|
|
|
try:
|
|
with processor._export_lock:
|
|
processor.on_trace_start(get_trace(processor))
|
|
assert processor._queue.qsize() == 1, "Trace should be queued"
|
|
|
|
processor.on_span_start(get_span(processor))
|
|
assert processor._queue.qsize() == 1, "Span should not be queued"
|
|
|
|
processor.on_span_end(get_span(processor))
|
|
assert processor._queue.qsize() == 2, "Span should be queued"
|
|
|
|
processor.on_trace_end(get_trace(processor))
|
|
assert processor._queue.qsize() == 2, "Nothing new should be queued"
|
|
finally:
|
|
processor.shutdown()
|
|
|
|
|
|
def test_batch_trace_processor_force_flush(mocked_exporter):
|
|
processor = BatchTraceProcessor(exporter=mocked_exporter, max_batch_size=2, schedule_delay=5.0)
|
|
|
|
try:
|
|
with processor._export_lock:
|
|
processor.on_trace_start(get_trace(processor))
|
|
processor.on_span_end(get_span(processor))
|
|
processor.on_span_end(get_span(processor))
|
|
|
|
processor.force_flush()
|
|
|
|
# Ensure exporter.export was called with all items in batches respecting max_batch_size=2.
|
|
exported_batches = [call_args[0][0] for call_args in mocked_exporter.export.call_args_list]
|
|
total_exported = sum(len(batch) for batch in exported_batches)
|
|
|
|
# We pushed 3 items; ensure they all got exported across 2 batches (sizes 2 and 1).
|
|
assert total_exported == 3
|
|
assert [len(batch) for batch in exported_batches] == [2, 1]
|
|
finally:
|
|
processor.shutdown()
|
|
|
|
|
|
def test_batch_trace_processor_force_flush_waits_for_in_flight_background_export(monkeypatch):
|
|
export_started = threading.Event()
|
|
export_continue = threading.Event()
|
|
export_completed = threading.Event()
|
|
flush_blocked = threading.Event()
|
|
flush_completed = threading.Event()
|
|
|
|
class ObservedExportLock:
|
|
def __init__(self) -> None:
|
|
self.lock = threading.Lock()
|
|
|
|
def __enter__(self) -> None:
|
|
if not self.lock.acquire(blocking=False):
|
|
flush_blocked.set()
|
|
self.lock.acquire()
|
|
|
|
def __exit__(self, *args: object) -> None:
|
|
self.lock.release()
|
|
|
|
class BlockingExporter(TracingExporter):
|
|
def export(self, items: list[Trace | Span[Any]]) -> None:
|
|
export_started.set()
|
|
export_continue.wait()
|
|
export_completed.set()
|
|
|
|
processor = BatchTraceProcessor(exporter=BlockingExporter(), schedule_delay=0.01)
|
|
monkeypatch.setattr(processor, "_export_lock", ObservedExportLock())
|
|
|
|
def flush() -> None:
|
|
processor.force_flush()
|
|
flush_completed.set()
|
|
|
|
flush_thread = threading.Thread(target=flush)
|
|
try:
|
|
processor.on_trace_start(get_trace(processor))
|
|
assert export_started.wait(timeout=2.0)
|
|
|
|
flush_thread.start()
|
|
assert flush_blocked.wait(timeout=2.0)
|
|
assert not export_completed.is_set()
|
|
assert not flush_completed.is_set(), "force_flush() should wait for an in-flight export"
|
|
|
|
export_continue.set()
|
|
assert flush_completed.wait(timeout=2.0)
|
|
assert export_completed.is_set()
|
|
finally:
|
|
export_continue.set()
|
|
if flush_thread.ident is not None:
|
|
flush_thread.join(timeout=2.0)
|
|
processor.shutdown(timeout=2.0)
|
|
|
|
assert not flush_thread.is_alive()
|
|
assert processor._worker_thread is not None
|
|
assert not processor._worker_thread.is_alive()
|
|
|
|
|
|
def test_batch_trace_processor_shutdown_flushes(mocked_exporter):
|
|
processor = BatchTraceProcessor(exporter=mocked_exporter, schedule_delay=5.0)
|
|
try:
|
|
with processor._export_lock:
|
|
processor.on_trace_start(get_trace(processor))
|
|
processor.on_span_end(get_span(processor))
|
|
qsize_before = processor._queue.qsize()
|
|
assert qsize_before == 2
|
|
finally:
|
|
processor.shutdown()
|
|
|
|
# Ensure everything was exported after shutdown.
|
|
total_exported = 0
|
|
for call_args in mocked_exporter.export.call_args_list:
|
|
batch = call_args[0][0]
|
|
total_exported += len(batch)
|
|
|
|
assert total_exported == 2, "All items in the queue should be exported upon shutdown"
|
|
|
|
|
|
def test_batch_trace_processor_shutdown_timeout_returns_when_exporter_blocks(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
export_started = threading.Event()
|
|
release_export = threading.Event()
|
|
|
|
class BlockingExporter(TracingExporter):
|
|
def export(self, items: list[Trace | Span[Any]]) -> None:
|
|
export_started.set()
|
|
release_export.wait(timeout=5.0)
|
|
|
|
processor = BatchTraceProcessor(
|
|
exporter=BlockingExporter(),
|
|
max_queue_size=1,
|
|
schedule_delay=60.0,
|
|
export_trigger_ratio=1.0,
|
|
)
|
|
try:
|
|
processor.on_span_end(get_span(processor))
|
|
assert export_started.wait(timeout=2.0)
|
|
|
|
start = time.monotonic()
|
|
with caplog.at_level(logging.WARNING):
|
|
processor.shutdown(timeout=0.05)
|
|
elapsed = time.monotonic() - start
|
|
|
|
assert elapsed < 0.5
|
|
assert "shutdown timeout reached" in caplog.text
|
|
finally:
|
|
release_export.set()
|
|
processor.shutdown(timeout=2.0)
|
|
|
|
assert processor._worker_thread is not None
|
|
assert not processor._worker_thread.is_alive()
|
|
|
|
|
|
def test_batch_trace_processor_shutdown_uses_custom_exporter_public_method() -> None:
|
|
exported: list[Trace | Span[Any]] = []
|
|
|
|
class DeadlineExporter(TracingExporter):
|
|
def export(self, items: list[Trace | Span[Any]]) -> None:
|
|
exported.extend(items)
|
|
|
|
def _export_with_deadline(
|
|
self, items: list[Trace | Span[Any]], deadline: float | None
|
|
) -> None:
|
|
raise AssertionError("private deadline helpers are not an exporter extension API")
|
|
|
|
processor = BatchTraceProcessor(exporter=DeadlineExporter())
|
|
span = get_span(processor)
|
|
processor._queue.put_nowait(span)
|
|
|
|
processor.shutdown(timeout=1.0)
|
|
|
|
assert exported == [span]
|
|
|
|
|
|
@pytest.mark.parametrize("cleanup", ["flush", "shutdown", "timed_shutdown", "automatic"])
|
|
@pytest.mark.parametrize("reject_batch", [False, True], ids=["filter", "raise"])
|
|
def test_batch_trace_processor_cleanup_preserves_exporter_filter(
|
|
monkeypatch: pytest.MonkeyPatch, cleanup: str, reject_batch: bool
|
|
) -> None:
|
|
received: list[dict[str, Any]] = []
|
|
filtered_batches: list[list[Trace | Span[Any]]] = []
|
|
|
|
def handle_request(request: httpx2.Request) -> httpx2.Response:
|
|
received.extend(json.loads(request.content)["data"])
|
|
return httpx2.Response(200)
|
|
|
|
class FilteringExporter(BackendSpanExporter):
|
|
def export(self, items: list[Trace | Span[Any]]) -> None:
|
|
filtered_batches.append(items)
|
|
if reject_batch:
|
|
raise ValueError("batch rejected by application policy")
|
|
super().export([item for item in items if isinstance(item, Trace)])
|
|
|
|
exporter = FilteringExporter(api_key="test_key")
|
|
exporter._client.close()
|
|
exporter._client = httpx2.Client(transport=httpx2.MockTransport(handle_request))
|
|
processor = BatchTraceProcessor(exporter=exporter)
|
|
# Keep the batch queued until the chosen cleanup entry point drains it.
|
|
monkeypatch.setattr(processor, "_ensure_thread_started", lambda: None)
|
|
allowed_trace = get_trace(processor)
|
|
sensitive_span = SpanImpl(
|
|
trace_id=allowed_trace.trace_id,
|
|
span_id="test_sensitive_span",
|
|
parent_id=None,
|
|
processor=processor,
|
|
span_data=FunctionSpanData(
|
|
name="test_tool", input="SYNTHETIC_PRIVATE_INPUT", output="SYNTHETIC_PRIVATE_OUTPUT"
|
|
),
|
|
tracing_api_key=None,
|
|
)
|
|
processor.on_trace_start(allowed_trace)
|
|
processor.on_span_end(sensitive_span)
|
|
try:
|
|
if cleanup == "automatic":
|
|
provider = DefaultTraceProvider()
|
|
provider.register_processor(processor)
|
|
monkeypatch.setattr(tracing_setup, "GLOBAL_TRACE_PROVIDER", provider)
|
|
tracing_setup._shutdown_global_trace_provider()
|
|
elif cleanup == "timed_shutdown":
|
|
processor.shutdown(timeout=1.0)
|
|
elif cleanup == "shutdown":
|
|
processor.shutdown()
|
|
else:
|
|
processor.force_flush()
|
|
|
|
assert received == ([] if reject_batch else [allowed_trace.export()])
|
|
assert "SYNTHETIC_PRIVATE_INPUT" not in json.dumps(received)
|
|
assert "SYNTHETIC_PRIVATE_OUTPUT" not in json.dumps(received)
|
|
assert filtered_batches == [[allowed_trace, sensitive_span]]
|
|
finally:
|
|
processor.shutdown()
|
|
exporter.close()
|
|
|
|
|
|
@pytest.mark.parametrize("use_subclass", [False, True], ids=["default", "inherited-export"])
|
|
@patch("httpx2.Client")
|
|
def test_batch_trace_processor_timed_shutdown_retries_final_drain(
|
|
mock_client, use_subclass: bool
|
|
) -> None:
|
|
class InheritedExportBackend(BackendSpanExporter):
|
|
pass
|
|
|
|
transient = MagicMock(status_code=503, headers={})
|
|
success = MagicMock(status_code=200, headers={})
|
|
mock_client.return_value.post.side_effect = [transient, success]
|
|
|
|
exporter_type = InheritedExportBackend if use_subclass else BackendSpanExporter
|
|
exporter = exporter_type(api_key="test_key", max_retries=2, base_delay=0.001)
|
|
processor = BatchTraceProcessor(exporter=exporter)
|
|
processor._queue.put_nowait(get_span(processor))
|
|
|
|
processor.shutdown(timeout=1.0)
|
|
|
|
assert mock_client.return_value.post.call_count == 2
|
|
for request in mock_client.return_value.post.call_args_list:
|
|
timeout = request.kwargs["timeout"]
|
|
assert isinstance(timeout, httpx2.Timeout)
|
|
assert timeout.read is not None and 0 < timeout.read <= 1.0
|
|
exporter.close()
|
|
|
|
|
|
def test_batch_trace_processor_survives_exporter_exception():
|
|
"""A failing exporter must not kill the background worker thread.
|
|
|
|
Previously, an exception raised inside ``exporter.export`` propagated out of
|
|
``_export_batches`` and killed the ``_run`` thread, causing all subsequent
|
|
spans to silently accumulate in the queue until it filled up.
|
|
"""
|
|
|
|
first_export_started = threading.Event()
|
|
recovery_completed = threading.Event()
|
|
|
|
class FlakyExporter(TracingExporter):
|
|
def __init__(self) -> None:
|
|
self.call_count = 0
|
|
self.exported: list[Trace | Span[Any]] = []
|
|
|
|
def export(self, items: list[Trace | Span[Any]]) -> None:
|
|
self.call_count += 1
|
|
if self.call_count == 1:
|
|
first_export_started.set()
|
|
raise RuntimeError("simulated exporter failure")
|
|
self.exported.extend(items)
|
|
if len(self.exported) == 2:
|
|
recovery_completed.set()
|
|
|
|
exporter = FlakyExporter()
|
|
processor = BatchTraceProcessor(exporter, schedule_delay=0.05, max_batch_size=1)
|
|
try:
|
|
processor.on_span_end(get_span(processor))
|
|
assert first_export_started.wait(timeout=2.0)
|
|
worker = processor._worker_thread
|
|
|
|
later_spans = [get_span(processor), get_span(processor)]
|
|
for span in later_spans:
|
|
processor.on_span_end(span)
|
|
|
|
assert recovery_completed.wait(timeout=2.0)
|
|
assert worker is not None
|
|
assert processor._worker_thread is worker
|
|
assert worker.is_alive(), "Worker thread must survive an exporter exception"
|
|
|
|
# Recovery must happen on the worker before shutdown can drain the queue.
|
|
assert exporter.exported == later_spans
|
|
assert exporter.call_count == 3
|
|
finally:
|
|
processor.shutdown(timeout=2.0)
|
|
|
|
assert processor._worker_thread is not None
|
|
assert not processor._worker_thread.is_alive()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("adjusted_wall_time", "adjusted_monotonic_time", "expected_scheduled_exports"),
|
|
[
|
|
(2000.0, 100.5, 0),
|
|
(0.0, 101.5, 1),
|
|
],
|
|
)
|
|
def test_batch_trace_processor_schedule_uses_monotonic_clock(
|
|
mocked_exporter,
|
|
monkeypatch,
|
|
adjusted_wall_time: float,
|
|
adjusted_monotonic_time: float,
|
|
expected_scheduled_exports: int,
|
|
) -> None:
|
|
class ControlledTime:
|
|
def __init__(self) -> None:
|
|
self.wall_time = 1000.0
|
|
self.monotonic_time = 100.0
|
|
self.sleep_calls = 0
|
|
|
|
def time(self) -> float:
|
|
return self.wall_time
|
|
|
|
def monotonic(self) -> float:
|
|
return self.monotonic_time
|
|
|
|
def sleep(self, _seconds: float) -> None:
|
|
self.sleep_calls += 1
|
|
if self.sleep_calls == 1:
|
|
self.wall_time = adjusted_wall_time
|
|
self.monotonic_time = adjusted_monotonic_time
|
|
else:
|
|
processor._shutdown_event.set()
|
|
|
|
controlled_time = ControlledTime()
|
|
monkeypatch.setattr("agents.tracing.processors.time", controlled_time)
|
|
processor = BatchTraceProcessor(
|
|
exporter=mocked_exporter,
|
|
max_queue_size=100,
|
|
schedule_delay=1.0,
|
|
export_trigger_ratio=1.0,
|
|
)
|
|
processor._queue.put_nowait(get_span(processor))
|
|
scheduled_export = object()
|
|
export_deadlines: list[float | None | object] = []
|
|
|
|
def record_export(deadline: float | None | object = scheduled_export) -> None:
|
|
export_deadlines.append(deadline)
|
|
if sum(item is scheduled_export for item in export_deadlines) > 1:
|
|
processor._shutdown_event.set()
|
|
|
|
monkeypatch.setattr(processor, "_export_batches", record_export)
|
|
|
|
processor._run()
|
|
|
|
assert sum(deadline is scheduled_export for deadline in export_deadlines) == (
|
|
expected_scheduled_exports
|
|
)
|
|
assert export_deadlines[-1] is None
|
|
|
|
|
|
def test_flush_traces_delegates_to_default_trace_provider():
|
|
provider = DefaultTraceProvider()
|
|
mock_processor = MagicMock()
|
|
provider.register_processor(mock_processor)
|
|
|
|
with patch("agents.tracing.setup.GLOBAL_TRACE_PROVIDER", provider):
|
|
flush_traces()
|
|
|
|
mock_processor.force_flush.assert_called_once()
|
|
|
|
|
|
def test_flush_traces_is_importable_from_top_level_agents_package():
|
|
from agents import flush_traces as top_level_flush_traces
|
|
|
|
assert top_level_flush_traces is flush_traces
|
|
|
|
|
|
def test_default_trace_provider_force_flush_still_flushes_when_disabled():
|
|
provider = DefaultTraceProvider()
|
|
mock_processor = MagicMock()
|
|
provider.register_processor(mock_processor)
|
|
|
|
provider.set_disabled(True)
|
|
provider.force_flush()
|
|
|
|
mock_processor.force_flush.assert_called_once_with()
|
|
|
|
|
|
def test_trace_provider_force_flush_and_shutdown_default_to_noops():
|
|
class MinimalProvider(TraceProvider):
|
|
def register_processor(self, processor: TracingProcessor) -> None:
|
|
pass
|
|
|
|
def set_processors(self, processors: list[TracingProcessor]) -> None:
|
|
pass
|
|
|
|
def get_current_trace(self):
|
|
return None
|
|
|
|
def get_current_span(self):
|
|
return None
|
|
|
|
def set_disabled(self, disabled: bool) -> None:
|
|
pass
|
|
|
|
def time_iso(self) -> str:
|
|
return ""
|
|
|
|
def gen_trace_id(self) -> str:
|
|
return "trace_123"
|
|
|
|
def gen_span_id(self) -> str:
|
|
return "span_123"
|
|
|
|
def gen_group_id(self) -> str:
|
|
return "group_123"
|
|
|
|
def create_trace(
|
|
self,
|
|
name,
|
|
trace_id=None,
|
|
group_id=None,
|
|
metadata=None,
|
|
disabled=False,
|
|
tracing=None,
|
|
):
|
|
raise NotImplementedError
|
|
|
|
def create_span(self, span_data, span_id=None, parent=None, disabled=False):
|
|
raise NotImplementedError
|
|
|
|
provider = MinimalProvider()
|
|
provider.force_flush()
|
|
provider.shutdown()
|
|
|
|
|
|
def test_get_trace_provider_force_flush_flushes_default_processor(mocked_exporter):
|
|
provider = DefaultTraceProvider()
|
|
processor = BatchTraceProcessor(exporter=mocked_exporter, schedule_delay=60.0)
|
|
provider.register_processor(processor)
|
|
|
|
try:
|
|
with patch("agents.tracing.setup.GLOBAL_TRACE_PROVIDER", provider):
|
|
processor.on_trace_start(get_trace(processor))
|
|
processor.on_span_end(get_span(processor))
|
|
|
|
get_trace_provider().force_flush()
|
|
|
|
total_exported = sum(
|
|
len(call_args[0][0]) for call_args in mocked_exporter.export.call_args_list
|
|
)
|
|
assert total_exported == 2
|
|
finally:
|
|
processor.shutdown()
|
|
|
|
|
|
def mock_processor():
|
|
processor = MagicMock()
|
|
processor.on_trace_start = MagicMock()
|
|
processor.on_span_end = MagicMock()
|
|
return processor
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_no_items(mock_client):
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
exporter.export([])
|
|
# No calls should be made if there are no items
|
|
mock_client.return_value.post.assert_not_called()
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_no_api_key(mock_client):
|
|
# Ensure that os.environ is empty (sometimes devs have the openai api key set in their env)
|
|
|
|
with patch.dict(os.environ, {}, clear=True):
|
|
exporter = BackendSpanExporter(api_key=None)
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
# Should log an error and return without calling post
|
|
mock_client.return_value.post.assert_not_called()
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_2xx_success(mock_client):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
exporter.export([get_span(mock_processor()), get_trace(mock_processor())])
|
|
|
|
# Should have called post exactly once
|
|
mock_client.return_value.post.assert_called_once()
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
@pytest.mark.parametrize("redacted", [True, False])
|
|
def test_backend_span_exporter_4xx_client_error(mock_client, monkeypatch, caplog, redacted: bool):
|
|
monkeypatch.setattr(_debug, "DONT_LOG_MODEL_DATA", redacted)
|
|
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", redacted)
|
|
|
|
class Response:
|
|
status_code = 400
|
|
text_reads = 0
|
|
|
|
@property
|
|
def text(self) -> str:
|
|
self.text_reads += 1
|
|
return "SECRET_TRACE_RESPONSE_BODY"
|
|
|
|
mock_response = Response()
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
with caplog.at_level(logging.ERROR, logger="openai.agents"):
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
# 4xx should not be retried
|
|
mock_client.return_value.post.assert_called_once()
|
|
assert ("SECRET_TRACE_RESPONSE_BODY" not in caplog.text) is redacted
|
|
assert mock_response.text_reads == (0 if redacted else 1)
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
@pytest.mark.parametrize("status_code", [408, 409, 429])
|
|
def test_backend_span_exporter_retries_transient_client_errors(mock_client, status_code: int):
|
|
"""A rate limit, timeout, or conflict is transient, as the OpenAI client treats it;
|
|
dropping the batch on the first response loses every trace in it."""
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = status_code
|
|
mock_response.headers = {}
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
|
|
with patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry:
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
assert mock_client.return_value.post.call_count == 3
|
|
assert wait_for_retry.call_count == 2
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_rate_limit_obeys_x_should_retry_false(mock_client, caplog):
|
|
"""An explicit `x-should-retry: false` wins over the status classification, as in the
|
|
OpenAI client."""
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 429
|
|
mock_response.headers = {"x-should-retry": "false", "retry-after": "1"}
|
|
mock_response.text = "rate limited"
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
|
|
with (
|
|
patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry,
|
|
caplog.at_level(logging.ERROR, logger="openai.agents"),
|
|
):
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
mock_client.return_value.post.assert_called_once()
|
|
wait_for_retry.assert_not_called()
|
|
assert "Tracing client error 429" in caplog.text
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_rate_limit_recovers_after_retry(mock_client):
|
|
ok_response = MagicMock()
|
|
ok_response.status_code = 200
|
|
limited_response = MagicMock()
|
|
limited_response.status_code = 429
|
|
limited_response.headers = {}
|
|
mock_client.return_value.post.side_effect = [limited_response, ok_response]
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
|
|
with patch.object(exporter._shutdown_event, "wait", return_value=False):
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
assert mock_client.return_value.post.call_count == 2
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
@pytest.mark.parametrize(
|
|
("headers", "expected_wait"),
|
|
[
|
|
({"retry-after": "3"}, 3.0),
|
|
({"retry-after-ms": "1500"}, 1.5),
|
|
# Larger than max_delay: bounded so the export thread cannot stall.
|
|
({"retry-after": "600"}, 5.0),
|
|
# Unparsable or negative values fall back to exponential backoff.
|
|
({"retry-after": "soon"}, None),
|
|
({"retry-after": "-1"}, None),
|
|
],
|
|
)
|
|
def test_backend_span_exporter_rate_limit_honours_retry_after(
|
|
mock_client, headers: dict[str, str], expected_wait: float | None
|
|
):
|
|
limited_response = MagicMock()
|
|
limited_response.status_code = 429
|
|
limited_response.headers = headers
|
|
ok_response = MagicMock()
|
|
ok_response.status_code = 200
|
|
mock_client.return_value.post.side_effect = [limited_response, ok_response]
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=5.0)
|
|
with patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry:
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
assert mock_client.return_value.post.call_count == 2
|
|
waited = wait_for_retry.call_args.args[0]
|
|
if expected_wait is None:
|
|
assert 0.1 <= waited <= 0.11
|
|
else:
|
|
assert waited == pytest.approx(expected_wait)
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_5xx_retry(mock_client):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 500
|
|
|
|
# Make post() return 500 every time
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
|
|
with patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry:
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
# Should retry up to max_retries times
|
|
assert mock_client.return_value.post.call_count == 3
|
|
assert wait_for_retry.call_count == 2
|
|
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_5xx_obeys_x_should_retry_false(mock_client, caplog):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 503
|
|
mock_response.headers = {"x-should-retry": "false"}
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
|
|
with (
|
|
patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry,
|
|
caplog.at_level(logging.ERROR, logger="openai.agents"),
|
|
):
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
mock_client.return_value.post.assert_called_once()
|
|
wait_for_retry.assert_not_called()
|
|
assert "server forbade retry for 503" in caplog.text
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_deadline_stops_during_5xx_retry_backoff(mock_client):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 504
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=1.0)
|
|
with patch("agents.tracing.processors.time") as clock:
|
|
# Spend the deadline budget only during backoff, independent of machine load.
|
|
clock.monotonic.return_value = 100.0
|
|
|
|
def advance_clock(delay: float) -> None:
|
|
clock.monotonic.return_value += delay
|
|
|
|
clock.sleep.side_effect = advance_clock
|
|
exporter._export_with_deadline([get_span(mock_processor())], deadline=100.01)
|
|
|
|
assert mock_client.return_value.post.call_count == 1
|
|
clock.sleep.assert_called_once_with(pytest.approx(0.01))
|
|
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_batch_trace_processor_shutdown_interrupts_exporter_retry_backoff(mock_client):
|
|
post_called = threading.Event()
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 504
|
|
|
|
def post(**kwargs: Any) -> Any:
|
|
post_called.set()
|
|
return mock_response
|
|
|
|
mock_client.return_value.post.side_effect = post
|
|
|
|
exporter = BackendSpanExporter(
|
|
api_key="test_key",
|
|
max_retries=100,
|
|
base_delay=10.0,
|
|
max_delay=10.0,
|
|
)
|
|
processor = BatchTraceProcessor(
|
|
exporter=exporter,
|
|
max_queue_size=1,
|
|
max_batch_size=1,
|
|
schedule_delay=60.0,
|
|
export_trigger_ratio=1.0,
|
|
)
|
|
|
|
try:
|
|
processor.on_span_end(get_span(processor))
|
|
assert post_called.wait(timeout=2.0)
|
|
|
|
start = time.monotonic()
|
|
processor.shutdown(timeout=1.0)
|
|
elapsed = time.monotonic() - start
|
|
|
|
assert elapsed < 0.5
|
|
assert processor._worker_thread is not None
|
|
assert not processor._worker_thread.is_alive()
|
|
assert mock_client.return_value.post.call_count == 1
|
|
|
|
finally:
|
|
processor.shutdown(timeout=2.0)
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_batch_trace_processor_shutdown_without_timeout_preserves_export_retries(mock_client):
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 504
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(
|
|
api_key="test_key",
|
|
max_retries=3,
|
|
base_delay=0.1,
|
|
max_delay=0.2,
|
|
)
|
|
processor = BatchTraceProcessor(exporter=exporter)
|
|
processor._queue.put_nowait(get_span(processor))
|
|
|
|
with patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry:
|
|
processor.shutdown(timeout=None)
|
|
|
|
assert mock_client.return_value.post.call_count == 3
|
|
assert wait_for_retry.call_count == 2
|
|
|
|
exporter.close()
|
|
|
|
|
|
@pytest.mark.serial
|
|
@pytest.mark.review_optional
|
|
def test_tracing_atexit_cleanup_timeout_preserves_process_exit_code_on_504() -> None:
|
|
script = textwrap.dedent(
|
|
"""
|
|
import sys
|
|
import threading
|
|
import time
|
|
from unittest.mock import patch
|
|
|
|
from agents.tracing import custom_span, trace
|
|
from agents.tracing.processors import BackendSpanExporter, BatchTraceProcessor
|
|
from agents.tracing.provider import DefaultTraceProvider
|
|
from agents.tracing import setup as tracing_setup
|
|
|
|
tracing_setup._DEFAULT_SHUTDOWN_TIMEOUT = 0.2
|
|
|
|
class Always504Response:
|
|
status_code = 504
|
|
text = "gateway timeout"
|
|
|
|
class Always504Client:
|
|
def __init__(self):
|
|
self.request_seen = threading.Event()
|
|
|
|
def post(self, **kwargs):
|
|
self.request_seen.set()
|
|
return Always504Response()
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
client = Always504Client()
|
|
with patch("agents.tracing.processors.httpx2.Client", return_value=client):
|
|
exporter = BackendSpanExporter(
|
|
api_key="test_key",
|
|
max_retries=100,
|
|
base_delay=10.0,
|
|
max_delay=10.0,
|
|
)
|
|
processor = BatchTraceProcessor(
|
|
exporter=exporter,
|
|
max_queue_size=1,
|
|
max_batch_size=1,
|
|
schedule_delay=60.0,
|
|
export_trigger_ratio=1.0,
|
|
)
|
|
provider = DefaultTraceProvider()
|
|
provider.register_processor(processor)
|
|
original_shutdown = provider.shutdown
|
|
|
|
def timed_shutdown(*args, **kwargs):
|
|
shutdown_started = time.monotonic()
|
|
try:
|
|
return original_shutdown(*args, **kwargs)
|
|
finally:
|
|
print(
|
|
f"shutdown_elapsed={time.monotonic() - shutdown_started:.6f}",
|
|
flush=True,
|
|
)
|
|
|
|
provider.shutdown = timed_shutdown
|
|
tracing_setup.set_trace_provider(provider)
|
|
|
|
with trace("probe"):
|
|
with custom_span("probe-span"):
|
|
pass
|
|
|
|
assert client.request_seen.wait(timeout=5.0)
|
|
sys.exit(7)
|
|
"""
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", script],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10.0,
|
|
)
|
|
|
|
assert result.returncode == 7
|
|
shutdown_elapsed_prefix = "shutdown_elapsed="
|
|
shutdown_elapsed_lines = [
|
|
line for line in result.stdout.splitlines() if line.startswith(shutdown_elapsed_prefix)
|
|
]
|
|
assert len(shutdown_elapsed_lines) == 1
|
|
assert float(shutdown_elapsed_lines[0][len(shutdown_elapsed_prefix) :]) < 0.5
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_request_error(mock_client):
|
|
# Make post() raise a RequestError each time
|
|
mock_client.return_value.post.side_effect = httpx2.RequestError("Network error")
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", max_retries=2, base_delay=0.1, max_delay=0.2)
|
|
with patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry:
|
|
exporter.export([get_span(mock_processor())])
|
|
|
|
# Should retry up to max_retries times
|
|
assert mock_client.return_value.post.call_count == 2
|
|
wait_for_retry.assert_called_once()
|
|
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_close(mock_client):
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
exporter.close()
|
|
|
|
# Ensure underlying http client is closed
|
|
mock_client.return_value.close.assert_called_once()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_sanitizes_generation_usage_for_openai_tracing(mock_client):
|
|
"""Unsupported usage keys should be stripped before POSTing to OpenAI tracing."""
|
|
|
|
class DummyItem:
|
|
tracing_api_key = None
|
|
|
|
def __init__(self):
|
|
self.exported_payload: dict[str, Any] = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"requests": 1,
|
|
"input_tokens": 10,
|
|
"output_tokens": 5,
|
|
"total_tokens": 15,
|
|
"input_tokens_details": {"cached_tokens": 1},
|
|
"output_tokens_details": {"reasoning_tokens": 2},
|
|
},
|
|
},
|
|
}
|
|
|
|
def export(self):
|
|
return self.exported_payload
|
|
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
item = DummyItem()
|
|
exporter.export([cast(Any, item)])
|
|
|
|
sent_payload = mock_client.return_value.post.call_args.kwargs["json"]["data"][0]
|
|
sent_usage = sent_payload["span_data"]["usage"]
|
|
assert "requests" not in sent_usage
|
|
assert "total_tokens" not in sent_usage
|
|
assert "input_tokens_details" not in sent_usage
|
|
assert "output_tokens_details" not in sent_usage
|
|
assert sent_usage["input_tokens"] == 10
|
|
assert sent_usage["output_tokens"] == 5
|
|
assert sent_usage["details"] == {
|
|
"requests": 1,
|
|
"total_tokens": 15,
|
|
"input_tokens_details": {"cached_tokens": 1},
|
|
"output_tokens_details": {"reasoning_tokens": 2},
|
|
}
|
|
|
|
# Ensure the original exported object has not been mutated.
|
|
assert "requests" in item.exported_payload["span_data"]["usage"]
|
|
assert item.exported_payload["span_data"]["usage"]["total_tokens"] == 15
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_truncates_large_input_for_openai_tracing(mock_client):
|
|
class DummyItem:
|
|
tracing_api_key = None
|
|
|
|
def __init__(self):
|
|
self.exported_payload: dict[str, Any] = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"input": "x" * (BackendSpanExporter._OPENAI_TRACING_MAX_FIELD_BYTES + 5_000),
|
|
},
|
|
}
|
|
|
|
def export(self):
|
|
return self.exported_payload
|
|
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
item = DummyItem()
|
|
exporter.export([cast(Any, item)])
|
|
|
|
sent_payload = mock_client.return_value.post.call_args.kwargs["json"]["data"][0]
|
|
sent_input = sent_payload["span_data"]["input"]
|
|
assert isinstance(sent_input, str)
|
|
assert sent_input.endswith(exporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX)
|
|
assert exporter._value_json_size_bytes(sent_input) <= exporter._OPENAI_TRACING_MAX_FIELD_BYTES
|
|
assert item.exported_payload["span_data"]["input"] != sent_input
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_truncates_large_structured_input_without_stringifying(mock_client):
|
|
class NoStringifyDict(dict[str, Any]):
|
|
def __str__(self) -> str:
|
|
raise AssertionError("__str__ should not be called for oversized non-string previews")
|
|
|
|
class DummyItem:
|
|
tracing_api_key = None
|
|
|
|
def __init__(self):
|
|
payload_input = NoStringifyDict(
|
|
blob="x" * (BackendSpanExporter._OPENAI_TRACING_MAX_FIELD_BYTES + 5_000)
|
|
)
|
|
self.exported_payload: dict[str, Any] = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"input": payload_input,
|
|
},
|
|
}
|
|
|
|
def export(self):
|
|
return self.exported_payload
|
|
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
exporter.export([cast(Any, DummyItem())])
|
|
|
|
sent_payload = mock_client.return_value.post.call_args.kwargs["json"]["data"][0]
|
|
sent_input = sent_payload["span_data"]["input"]
|
|
assert isinstance(sent_input, dict)
|
|
assert isinstance(sent_input["blob"], str)
|
|
assert sent_input["blob"].endswith(exporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX)
|
|
assert exporter._value_json_size_bytes(sent_input) <= exporter._OPENAI_TRACING_MAX_FIELD_BYTES
|
|
exporter.close()
|
|
|
|
|
|
def _exporter_capturing_posts(received: list[dict[str, Any]], **kwargs: Any) -> BackendSpanExporter:
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
received.extend(json.loads(request.content)["data"])
|
|
return httpx2.Response(200)
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key", **kwargs)
|
|
exporter._client.close()
|
|
exporter._client = httpx2.Client(transport=httpx2.MockTransport(handler))
|
|
return exporter
|
|
|
|
|
|
@pytest.mark.parametrize("bad_value", [uuid.uuid4(), float("nan")], ids=["uuid", "nan"])
|
|
def test_backend_span_exporter_keeps_batch_when_trace_metadata_is_not_json(bad_value: Any):
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received)
|
|
clean_trace = get_trace(mock_processor())
|
|
clean_span = get_span(mock_processor())
|
|
bad_trace = TraceImpl(
|
|
name="bad_trace",
|
|
trace_id="bad_trace_id",
|
|
group_id=None,
|
|
metadata={"request_id": bad_value, "ok": "x"},
|
|
processor=mock_processor(),
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
exporter.export([clean_trace, clean_span, bad_trace])
|
|
|
|
assert received == [
|
|
clean_trace.export(),
|
|
clean_span.export(),
|
|
{**cast(dict[str, Any], bad_trace.export()), "metadata": {"ok": "x"}},
|
|
]
|
|
exporter.close()
|
|
|
|
|
|
def test_backend_span_exporter_repair_sends_dict_keys_the_way_json_does():
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received)
|
|
encodable_keys: dict[Any, Any] = {200: "int", 1.5: "float", True: "bool", None: "none"}
|
|
metadata: dict[Any, Any] = {
|
|
**encodable_keys,
|
|
float("nan"): "nan key",
|
|
"at": datetime.now(timezone.utc),
|
|
}
|
|
bad_trace = TraceImpl(
|
|
name="bad_trace",
|
|
trace_id="bad_trace_id",
|
|
group_id=None,
|
|
metadata=metadata,
|
|
processor=mock_processor(),
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
exporter.export([bad_trace])
|
|
|
|
assert received == [
|
|
{
|
|
**cast(dict[str, Any], bad_trace.export()),
|
|
"metadata": json.loads(json.dumps(encodable_keys)),
|
|
},
|
|
]
|
|
assert received[0]["metadata"] == {"200": "int", "1.5": "float", "true": "bool", "null": "none"}
|
|
exporter.close()
|
|
|
|
|
|
def test_backend_span_exporter_keeps_batch_when_a_string_has_an_unpaired_surrogate():
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received)
|
|
clean_trace = get_trace(mock_processor())
|
|
bad_trace = TraceImpl(
|
|
name="bad_trace",
|
|
trace_id="bad_trace_id",
|
|
group_id=None,
|
|
metadata={"note": "x\ud800y", "ok": "x"},
|
|
processor=mock_processor(),
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
exporter.export([clean_trace, bad_trace])
|
|
|
|
assert received == [
|
|
clean_trace.export(),
|
|
{**cast(dict[str, Any], bad_trace.export()), "metadata": {"note": "x?y", "ok": "x"}},
|
|
]
|
|
exporter.close()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("span_data", "repaired"),
|
|
[
|
|
(
|
|
FunctionSpanData(name="lookup", input="x\ud800y", output="x\ud800y"),
|
|
{"input": "x?y", "output": "x?y"},
|
|
),
|
|
(
|
|
GenerationSpanData(input=[{"content": "x\ud800y"}], output=[{"content": "x\ud800y"}]),
|
|
{"input": [{"content": "x?y"}], "output": [{"content": "x?y"}]},
|
|
),
|
|
],
|
|
ids=["function", "generation"],
|
|
)
|
|
def test_backend_span_exporter_keeps_openai_batch_when_span_io_has_an_unpaired_surrogate(
|
|
span_data: Any, repaired: dict[str, Any]
|
|
):
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received)
|
|
clean_trace = get_trace(mock_processor())
|
|
span = SpanImpl(
|
|
trace_id="test_trace_id",
|
|
span_id="io_span_id",
|
|
parent_id=None,
|
|
processor=mock_processor(),
|
|
span_data=span_data,
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
exporter.export([clean_trace, span])
|
|
|
|
exported_span = cast(dict[str, Any], span.export())
|
|
assert received == [
|
|
clean_trace.export(),
|
|
{**exported_span, "span_data": {**exported_span["span_data"], **repaired}},
|
|
]
|
|
exporter.close()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"endpoint",
|
|
[BackendSpanExporter._OPENAI_TRACING_INGEST_ENDPOINT, "https://example.test/traces"],
|
|
ids=["openai", "custom"],
|
|
)
|
|
def test_backend_span_exporter_keeps_valid_custom_span_data_when_repairing(endpoint: str):
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received, endpoint=endpoint)
|
|
clean_trace = get_trace(mock_processor())
|
|
custom_span = SpanImpl(
|
|
trace_id="test_trace_id",
|
|
span_id="custom_span_id",
|
|
parent_id="parent_span_id",
|
|
processor=mock_processor(),
|
|
span_data=CustomSpanData(
|
|
name="lookup",
|
|
data={"status_counts": {200: 3}, "at": datetime.now(timezone.utc)},
|
|
),
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
exporter.export([clean_trace, custom_span])
|
|
|
|
exported_span = cast(dict[str, Any], custom_span.export())
|
|
assert received == [
|
|
clean_trace.export(),
|
|
{
|
|
**exported_span,
|
|
"span_data": {
|
|
"type": "custom",
|
|
"name": "lookup",
|
|
"data": {"status_counts": {"200": 3}},
|
|
},
|
|
},
|
|
]
|
|
assert received[1]["id"] == "custom_span_id"
|
|
assert received[1]["trace_id"] == "test_trace_id"
|
|
assert received[1]["parent_id"] == "parent_span_id"
|
|
exporter.close()
|
|
|
|
|
|
def test_backend_span_exporter_keeps_int_keys_in_openai_sanitized_span_fields():
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received)
|
|
oversized_input: list[dict[Any, Any]] = [
|
|
{200: 3, "blob": "x" * (BackendSpanExporter._OPENAI_TRACING_MAX_FIELD_BYTES + 5_000)}
|
|
]
|
|
span = SpanImpl(
|
|
trace_id="test_trace_id",
|
|
span_id="generation_span_id",
|
|
parent_id=None,
|
|
processor=mock_processor(),
|
|
span_data=GenerationSpanData(
|
|
input=oversized_input,
|
|
usage={"input_tokens": 1, "output_tokens": 2, "details": {"by_status": {200: 3}}},
|
|
),
|
|
tracing_api_key=None,
|
|
)
|
|
|
|
exporter.export([span])
|
|
|
|
[sent] = received
|
|
assert sent["id"] == "generation_span_id"
|
|
assert sent["span_data"]["usage"] == {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"details": {"by_status": {"200": 3}},
|
|
}
|
|
[sent_input] = sent["span_data"]["input"]
|
|
assert sent_input["200"] == 3
|
|
assert sent_input["blob"].endswith(BackendSpanExporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX)
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_keeps_generation_usage_for_custom_endpoint(mock_client):
|
|
class DummyItem:
|
|
tracing_api_key = None
|
|
|
|
def __init__(self):
|
|
self.exported_payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"requests": 1,
|
|
"input_tokens": 10,
|
|
"output_tokens": 5,
|
|
},
|
|
},
|
|
}
|
|
|
|
def export(self):
|
|
return self.exported_payload
|
|
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(
|
|
api_key="test_key",
|
|
endpoint="https://example.com/v1/traces/ingest",
|
|
)
|
|
exporter.export([cast(Any, DummyItem())])
|
|
|
|
sent_payload = mock_client.return_value.post.call_args.kwargs["json"]["data"][0]
|
|
assert sent_payload["span_data"]["usage"]["requests"] == 1
|
|
assert sent_payload["span_data"]["usage"]["input_tokens"] == 10
|
|
assert sent_payload["span_data"]["usage"]["output_tokens"] == 5
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_drops_non_generation_usage_for_openai_endpoint(mock_client):
|
|
class DummyItem:
|
|
tracing_api_key = None
|
|
|
|
def export(self):
|
|
return {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "function",
|
|
"usage": {"requests": 1},
|
|
},
|
|
}
|
|
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
exporter.export([cast(Any, DummyItem())])
|
|
|
|
sent_payload = mock_client.return_value.post.call_args.kwargs["json"]["data"][0]
|
|
assert "usage" not in sent_payload["span_data"]
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_keeps_non_generation_usage_for_custom_endpoint(mock_client):
|
|
class DummyItem:
|
|
tracing_api_key = None
|
|
|
|
def export(self):
|
|
return {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "function",
|
|
"usage": {"requests": 1},
|
|
},
|
|
}
|
|
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(
|
|
api_key="test_key",
|
|
endpoint="https://example.com/v1/traces/ingest",
|
|
)
|
|
exporter.export([cast(Any, DummyItem())])
|
|
|
|
sent_payload = mock_client.return_value.post.call_args.kwargs["json"]["data"][0]
|
|
assert sent_payload["span_data"]["usage"] == {"requests": 1}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_keeps_allowed_generation_usage():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
},
|
|
},
|
|
}
|
|
assert exporter._sanitize_for_openai_tracing_api(payload) is payload
|
|
exporter.close()
|
|
|
|
|
|
@patch("httpx2.Client")
|
|
def test_backend_span_exporter_keeps_large_input_for_custom_endpoint(mock_client):
|
|
class DummyItem:
|
|
tracing_api_key = None
|
|
|
|
def __init__(self):
|
|
self.exported_payload: dict[str, Any] = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"input": "x" * (BackendSpanExporter._OPENAI_TRACING_MAX_FIELD_BYTES + 5_000),
|
|
},
|
|
}
|
|
|
|
def export(self):
|
|
return self.exported_payload
|
|
|
|
mock_response = MagicMock()
|
|
mock_response.status_code = 200
|
|
mock_client.return_value.post.return_value = mock_response
|
|
|
|
exporter = BackendSpanExporter(
|
|
api_key="test_key",
|
|
endpoint="https://example.com/v1/traces/ingest",
|
|
)
|
|
item = DummyItem()
|
|
exporter.export([cast(Any, item)])
|
|
|
|
sent_payload: dict[str, Any] = mock_client.return_value.post.call_args.kwargs["json"]["data"][0]
|
|
assert sent_payload["span_data"]["input"] == item.exported_payload["span_data"]["input"]
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_moves_unsupported_generation_usage_to_details():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"total_tokens": 3,
|
|
"input_tokens_details": {"cached_tokens": 0},
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
"details": {"provider": "litellm"},
|
|
},
|
|
},
|
|
}
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"]["usage"] == {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"details": {
|
|
"provider": "litellm",
|
|
"total_tokens": 3,
|
|
"input_tokens_details": {"cached_tokens": 0},
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
},
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_filters_non_json_values_in_usage_details():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
non_json = object()
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"input_tokens_details": {
|
|
"cached_tokens": 0,
|
|
"bad": non_json,
|
|
},
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
"provider_usage": [1, non_json, {"ok": True, "bad": non_json}],
|
|
"details": {
|
|
"provider": "litellm",
|
|
"bad": non_json,
|
|
"nested": {"keep": 1, "bad": non_json},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"]["usage"] == {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"details": {
|
|
"provider": "litellm",
|
|
"nested": {"keep": 1},
|
|
"input_tokens_details": {"cached_tokens": 0},
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
"provider_usage": [1, {"ok": True}],
|
|
},
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_keeps_json_encodable_usage_detail_keys():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"details": {200: 3, True: 4, None: 5, 1.5: 6, "ok": 7},
|
|
"provider_usage": {404: "missing"},
|
|
},
|
|
},
|
|
}
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"]["usage"]["details"] == {
|
|
"200": 3,
|
|
"true": 4,
|
|
"null": 5,
|
|
"1.5": 6,
|
|
"ok": 7,
|
|
"provider_usage": {"404": "missing"},
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_handles_cyclic_usage_values():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
cyclic_dict: dict[str, Any] = {}
|
|
cyclic_dict["self"] = cyclic_dict
|
|
cyclic_list: list[Any] = []
|
|
cyclic_list.append(cyclic_list)
|
|
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"input_tokens_details": cyclic_dict,
|
|
"details": {
|
|
"provider": "litellm",
|
|
"cycle": cyclic_list,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"]["usage"] == {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"details": {
|
|
"provider": "litellm",
|
|
"cycle": [],
|
|
"input_tokens_details": {},
|
|
},
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_drops_non_dict_generation_usage_details():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
"details": "invalid",
|
|
},
|
|
},
|
|
}
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"]["usage"] == {
|
|
"input_tokens": 1,
|
|
"output_tokens": 2,
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_drops_generation_usage_missing_required_tokens():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": 1,
|
|
"total_tokens": 3,
|
|
"input_tokens_details": {"cached_tokens": 0},
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
},
|
|
},
|
|
}
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"] == {
|
|
"type": "generation",
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_rejects_boolean_token_counts():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": {
|
|
"input_tokens": True,
|
|
"output_tokens": False,
|
|
"input_tokens_details": {"cached_tokens": 0},
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
},
|
|
},
|
|
}
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"] == {
|
|
"type": "generation",
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_skips_non_dict_generation_usage():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"usage": None,
|
|
},
|
|
}
|
|
assert exporter._sanitize_for_openai_tracing_api(payload) is payload
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_keeps_small_input_without_mutation():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"input": "short input",
|
|
"usage": {"input_tokens": 1, "output_tokens": 2},
|
|
},
|
|
}
|
|
|
|
assert exporter._sanitize_for_openai_tracing_api(payload) is payload
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_truncates_oversized_output():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload: dict[str, Any] = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "function",
|
|
"output": "x" * (BackendSpanExporter._OPENAI_TRACING_MAX_FIELD_BYTES + 5_000),
|
|
},
|
|
}
|
|
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized is not payload
|
|
assert sanitized["span_data"]["output"].endswith(
|
|
exporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX
|
|
)
|
|
assert (
|
|
exporter._value_json_size_bytes(sanitized["span_data"]["output"])
|
|
<= exporter._OPENAI_TRACING_MAX_FIELD_BYTES
|
|
)
|
|
assert payload["span_data"]["output"] != sanitized["span_data"]["output"]
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_preserves_generation_input_list_shape():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"input": [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "input_audio",
|
|
"input_audio": {
|
|
"data": "x"
|
|
* (BackendSpanExporter._OPENAI_TRACING_MAX_FIELD_BYTES + 5_000),
|
|
"format": "wav",
|
|
},
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"usage": {"input_tokens": 1, "output_tokens": 1},
|
|
},
|
|
}
|
|
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
sanitized_input = sanitized["span_data"]["input"]
|
|
assert isinstance(sanitized_input, list)
|
|
assert isinstance(sanitized_input[0], dict)
|
|
assert sanitized_input[0]["role"] == "user"
|
|
assert (
|
|
exporter._value_json_size_bytes(sanitized_input) <= exporter._OPENAI_TRACING_MAX_FIELD_BYTES
|
|
)
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_replaces_unserializable_output():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload: dict[str, Any] = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "function",
|
|
"output": b"x" * 10,
|
|
},
|
|
}
|
|
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
assert sanitized["span_data"]["output"] == {
|
|
"truncated": True,
|
|
"original_type": "bytes",
|
|
"preview": "<bytes bytes=10 truncated>",
|
|
}
|
|
exporter.close()
|
|
|
|
|
|
def test_truncate_json_value_for_limit_terminates_preview_dict_under_zero_budget():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
preview = exporter._truncated_preview(None)
|
|
|
|
truncated = exporter._truncate_json_value_for_limit(preview, 0)
|
|
|
|
assert truncated == {}
|
|
exporter.close()
|
|
|
|
|
|
def test_sanitize_for_openai_tracing_api_handles_none_content_under_tight_budget():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
payload: dict[str, Any] = {
|
|
"object": "trace.span",
|
|
"span_data": {
|
|
"type": "generation",
|
|
"output": [
|
|
{
|
|
"role": "assistant",
|
|
"content": None,
|
|
"name": "a" * 25_000,
|
|
"tool_calls": [],
|
|
}
|
|
for _ in range(8)
|
|
],
|
|
"usage": {"input_tokens": 1, "output_tokens": 1},
|
|
},
|
|
}
|
|
|
|
sanitized = exporter._sanitize_for_openai_tracing_api(payload)
|
|
sanitized_output = cast(list[Any], sanitized["span_data"]["output"])
|
|
|
|
assert isinstance(sanitized_output, list)
|
|
assert sanitized_output != payload["span_data"]["output"]
|
|
assert (
|
|
exporter._value_json_size_bytes(sanitized_output)
|
|
<= exporter._OPENAI_TRACING_MAX_FIELD_BYTES
|
|
)
|
|
assert any(item == {} for item in sanitized_output)
|
|
exporter.close()
|
|
|
|
|
|
def test_truncate_string_for_json_limit_returns_original_when_within_limit():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
value = "hello"
|
|
max_bytes = exporter._value_json_size_bytes(value)
|
|
|
|
assert exporter._truncate_string_for_json_limit(value, max_bytes) == value
|
|
exporter.close()
|
|
|
|
|
|
def test_truncate_string_for_json_limit_returns_suffix_when_limit_equals_suffix():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
max_bytes = exporter._value_json_size_bytes(exporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX)
|
|
|
|
assert (
|
|
exporter._truncate_string_for_json_limit("x" * 100, max_bytes)
|
|
== exporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX
|
|
)
|
|
exporter.close()
|
|
|
|
|
|
def test_truncate_string_for_json_limit_returns_empty_when_suffix_too_large():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
max_bytes = (
|
|
exporter._value_json_size_bytes(exporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX) - 1
|
|
)
|
|
|
|
assert exporter._truncate_string_for_json_limit("x" * 100, max_bytes) == ""
|
|
exporter.close()
|
|
|
|
|
|
def test_truncate_string_for_json_limit_handles_escape_heavy_input():
|
|
exporter = BackendSpanExporter(api_key="test_key")
|
|
value = ('\\"' * 40_000) + "tail"
|
|
max_bytes = exporter._OPENAI_TRACING_MAX_FIELD_BYTES
|
|
|
|
truncated = exporter._truncate_string_for_json_limit(value, max_bytes)
|
|
|
|
assert truncated.endswith(exporter._OPENAI_TRACING_STRING_TRUNCATION_SUFFIX)
|
|
assert exporter._value_json_size_bytes(truncated) <= max_bytes
|
|
exporter.close()
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_backend_span_exporter_bounds_multipart_normalization_work(monkeypatch):
|
|
from openai import AsyncOpenAI
|
|
|
|
from agents import Agent, OpenAIChatCompletionsModel, Runner
|
|
from tests.testing_processor import fetch_ordered_spans
|
|
|
|
provider_messages: list[Any] = []
|
|
|
|
def provider_response(request: httpx2.Request) -> httpx2.Response:
|
|
provider_messages.append(json.loads(request.content)["messages"])
|
|
return httpx2.Response(
|
|
200,
|
|
json={
|
|
"id": "synthetic-completion",
|
|
"object": "chat.completion",
|
|
"created": 0,
|
|
"model": "test-model",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"finish_reason": "stop",
|
|
"message": {"role": "assistant", "content": "ok"},
|
|
}
|
|
],
|
|
},
|
|
)
|
|
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received)
|
|
original_size = exporter._value_json_size_bytes
|
|
work = 0
|
|
work_limit = 0
|
|
|
|
def counted_size(value: Any) -> int:
|
|
nonlocal work
|
|
size = original_size(value)
|
|
work += size
|
|
# Bound regression runtime as well as work; the old implementation fails
|
|
# after a small number of rescans instead of completing a quadratic run.
|
|
assert work <= work_limit
|
|
return size
|
|
|
|
monkeypatch.setattr(exporter, "_value_json_size_bytes", counted_size)
|
|
measured_work: list[int] = []
|
|
try:
|
|
async with AsyncOpenAI(
|
|
api_key="test_key",
|
|
http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(provider_response)),
|
|
) as client:
|
|
agent = Agent(
|
|
name="multipart-test", model=OpenAIChatCompletionsModel("test-model", client)
|
|
)
|
|
for width in (1_000, 2_000):
|
|
parts = [{"type": "input_text", "text": "synthetic " * 20} for _ in range(width)]
|
|
result = await Runner.run(agent, [{"role": "user", "content": parts}])
|
|
assert result.final_output == "ok"
|
|
span = [s for s in fetch_ordered_spans() if s.span_data.type == "generation"][-1]
|
|
original = span.export()
|
|
assert original is not None
|
|
assert original["span_data"]["input"] == provider_messages[-1]
|
|
assert len(provider_messages[-1][0]["content"]) == width
|
|
work = 0
|
|
work_limit = 30 * original_size(provider_messages[-1])
|
|
exporter.export([span])
|
|
measured_work.append(work)
|
|
sent_input = received[-1]["span_data"]["input"]
|
|
assert (
|
|
len(json.dumps(sent_input, ensure_ascii=False, separators=(",", ":")).encode())
|
|
<= 100_000
|
|
)
|
|
assert sent_input[0]["role"] == "user"
|
|
assert isinstance(sent_input[0]["content"], list)
|
|
assert any(part.get("text") for part in sent_input[0]["content"])
|
|
assert span.export() == original
|
|
assert measured_work[1] <= 3 * measured_work[0]
|
|
|
|
# An ordinary trace still leaves the same processor after the large
|
|
# span, in a separate batch. Mock ingest makes the completion bounded.
|
|
received.clear()
|
|
work = 0
|
|
processor = BatchTraceProcessor(exporter, max_batch_size=1, schedule_delay=0.01)
|
|
normal = get_span(mock_processor())
|
|
processor.on_span_end(span)
|
|
processor.on_span_end(normal)
|
|
try:
|
|
processor.force_flush()
|
|
assert len(received) == 2
|
|
assert received[-1] == normal.export()
|
|
assert received[-2]["id"] == span.span_id
|
|
assert processor._queue.empty()
|
|
finally:
|
|
processor.shutdown(timeout=1)
|
|
finally:
|
|
exporter.close()
|
|
|
|
|
|
def test_backend_span_exporter_bounds_wide_mapping_normalization_work(monkeypatch):
|
|
received: list[dict[str, Any]] = []
|
|
exporter = _exporter_capturing_posts(received)
|
|
original_size = exporter._value_json_size_bytes
|
|
measured_work: list[int] = []
|
|
work = 0
|
|
work_limit = 0
|
|
|
|
def counted_size(value: Any) -> int:
|
|
nonlocal work
|
|
size = original_size(value)
|
|
work += size
|
|
assert work <= work_limit
|
|
return size
|
|
|
|
monkeypatch.setattr(exporter, "_value_json_size_bytes", counted_size)
|
|
try:
|
|
for width in (1_000, 2_000):
|
|
# Public generation spans accept structured mappings; escaped Unicode
|
|
# keys exercise key/colon/comma accounting when entries are removed.
|
|
mapping = {f'{index}:"\\雪' + "k" * 120: "value" for index in range(width)}
|
|
span = SpanImpl(
|
|
trace_id="test_trace_id",
|
|
span_id="generation_span_id",
|
|
parent_id=None,
|
|
processor=mock_processor(),
|
|
span_data=GenerationSpanData(input=[mapping]),
|
|
tracing_api_key=None,
|
|
)
|
|
work = 0
|
|
work_limit = 30 * original_size(mapping)
|
|
|
|
exporter.export([span])
|
|
measured_work.append(work)
|
|
[sent] = received[-1]["span_data"]["input"]
|
|
assert 0 < len(sent) < width
|
|
assert list(sent) == list(mapping)[-len(sent) :]
|
|
assert set(sent.values()) == {""}
|
|
assert (
|
|
len(json.dumps([sent], ensure_ascii=False, separators=(",", ":")).encode())
|
|
<= 100_000
|
|
)
|
|
assert len(mapping) == width
|
|
assert measured_work[1] <= 3 * measured_work[0]
|
|
finally:
|
|
exporter.close()
|