* Align GPT-5 and newer model defaults with TypeScript * Preserve explicit request settings when applying model defaults
2548 lines
89 KiB
Python
2548 lines
89 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import copy
|
|
import importlib
|
|
import json
|
|
import sys
|
|
import types as pytypes
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any, Literal, cast
|
|
|
|
import httpx2
|
|
import pytest
|
|
from openai.types.chat import (
|
|
ChatCompletion,
|
|
ChatCompletionChunk,
|
|
ChatCompletionMessage,
|
|
ChatCompletionMessageFunctionToolCall,
|
|
)
|
|
from openai.types.chat.chat_completion import Choice
|
|
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice, ChoiceDelta
|
|
from openai.types.completion_usage import (
|
|
CompletionTokensDetails,
|
|
CompletionUsage,
|
|
PromptTokensDetails,
|
|
)
|
|
from openai.types.responses import (
|
|
Response,
|
|
ResponseCompletedEvent,
|
|
ResponseOutputMessage,
|
|
ResponseOutputRefusal,
|
|
)
|
|
from openai.types.responses.response import IncompleteDetails
|
|
from openai.types.responses.response_created_event import ResponseCreatedEvent
|
|
from openai.types.responses.response_error_event import ResponseErrorEvent
|
|
from openai.types.responses.response_failed_event import ResponseFailedEvent
|
|
from openai.types.responses.response_incomplete_event import ResponseIncompleteEvent
|
|
from openai.types.responses.response_output_text import ResponseOutputText
|
|
from openai.types.responses.response_usage import (
|
|
InputTokensDetails,
|
|
OutputTokensDetails,
|
|
ResponseUsage,
|
|
)
|
|
from openai.types.shared import Reasoning
|
|
from pydantic import BaseModel
|
|
|
|
from agents import (
|
|
Agent,
|
|
Handoff,
|
|
ModelBehaviorError,
|
|
ModelSettings,
|
|
ModelTracing,
|
|
RunConfig,
|
|
Runner,
|
|
Tool,
|
|
TResponseInputItem,
|
|
__version__,
|
|
function_tool,
|
|
handoff,
|
|
trace,
|
|
)
|
|
from agents.exceptions import UserError
|
|
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
|
|
from agents.models.fake_id import FAKE_RESPONSES_ID
|
|
from agents.tracing.processors import BackendSpanExporter
|
|
from tests.testing_processor import fetch_ordered_spans
|
|
|
|
|
|
class FakeAnyLLMProvider:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
supports_responses: bool,
|
|
chat_response: Any | None = None,
|
|
responses_response: Any | None = None,
|
|
) -> None:
|
|
self.SUPPORTS_RESPONSES = supports_responses
|
|
self.chat_response = chat_response
|
|
self.responses_response = responses_response
|
|
self.chat_calls: list[dict[str, Any]] = []
|
|
self.responses_calls: list[dict[str, Any]] = []
|
|
self.private_responses_calls: list[dict[str, Any]] = []
|
|
|
|
async def acompletion(self, **kwargs: Any) -> Any:
|
|
self.chat_calls.append(kwargs)
|
|
return self.chat_response
|
|
|
|
async def aresponses(self, **kwargs: Any) -> Any:
|
|
self.responses_calls.append(kwargs)
|
|
return self.responses_response
|
|
|
|
async def _aresponses(self, params: Any, **kwargs: Any) -> Any:
|
|
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
|
|
return self.responses_response
|
|
|
|
|
|
def _import_any_llm_module(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
provider: FakeAnyLLMProvider,
|
|
) -> tuple[Any, list[dict[str, Any]]]:
|
|
create_calls: list[dict[str, Any]] = []
|
|
|
|
class FakeAnyLLMFactory:
|
|
@staticmethod
|
|
def create(provider_name: str, api_key: str | None = None, api_base: str | None = None):
|
|
create_calls.append(
|
|
{
|
|
"provider_name": provider_name,
|
|
"api_key": api_key,
|
|
"api_base": api_base,
|
|
}
|
|
)
|
|
return provider
|
|
|
|
fake_any_llm: Any = pytypes.ModuleType("any_llm")
|
|
fake_any_llm.AnyLLM = FakeAnyLLMFactory
|
|
|
|
# Importing the submodule fresh replaces both bindings that other code resolves
|
|
# it through: the ``sys.modules`` entry and the ``any_llm_model`` attribute on the
|
|
# parent package. Route both through ``monkeypatch`` so teardown restores them
|
|
# together; otherwise a later ``from agents.extensions.models import any_llm_model``
|
|
# would still see the stub-backed module while ``sys.modules`` has the original.
|
|
parent_package = importlib.import_module("agents.extensions.models")
|
|
monkeypatch.delitem(sys.modules, "agents.extensions.models.any_llm_model", raising=False)
|
|
monkeypatch.delattr(parent_package, "any_llm_model", raising=False)
|
|
monkeypatch.setitem(sys.modules, "any_llm", fake_any_llm)
|
|
|
|
module = importlib.import_module("agents.extensions.models.any_llm_model")
|
|
monkeypatch.setattr(module, "AnyLLM", FakeAnyLLMFactory, raising=True)
|
|
return module, create_calls
|
|
|
|
|
|
def test_import_any_llm_module_restores_module_bindings_on_teardown() -> None:
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
parent_package = importlib.import_module("agents.extensions.models")
|
|
original = importlib.import_module("agents.extensions.models.any_llm_model")
|
|
|
|
with pytest.MonkeyPatch.context() as monkeypatch:
|
|
module, _ = _import_any_llm_module(
|
|
monkeypatch, FakeAnyLLMProvider(supports_responses=False)
|
|
)
|
|
assert module is not original
|
|
assert sys.modules["agents.extensions.models.any_llm_model"] is module
|
|
assert parent_package.any_llm_model is module
|
|
|
|
assert sys.modules["agents.extensions.models.any_llm_model"] is original
|
|
assert parent_package.any_llm_model is original
|
|
|
|
|
|
def _chat_completion(text: str) -> ChatCompletion:
|
|
return ChatCompletion(
|
|
id="chatcmpl_123",
|
|
created=0,
|
|
model="fake-model",
|
|
object="chat.completion",
|
|
choices=[
|
|
Choice(
|
|
index=0,
|
|
finish_reason="stop",
|
|
message=ChatCompletionMessage(role="assistant", content=text),
|
|
)
|
|
],
|
|
usage=CompletionUsage(
|
|
completion_tokens=5,
|
|
prompt_tokens=7,
|
|
total_tokens=12,
|
|
prompt_tokens_details=PromptTokensDetails.model_validate(
|
|
{"cached_tokens": 2, "cache_write_tokens": 4}
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
def _responses_output(text: str) -> list[Any]:
|
|
return [
|
|
ResponseOutputMessage(
|
|
id="msg_123",
|
|
role="assistant",
|
|
status="completed",
|
|
type="message",
|
|
content=[
|
|
ResponseOutputText(
|
|
text=text,
|
|
type="output_text",
|
|
annotations=[],
|
|
logprobs=[],
|
|
)
|
|
],
|
|
)
|
|
]
|
|
|
|
|
|
def _response(text: str, response_id: str = "resp_123") -> Response:
|
|
return Response(
|
|
id=response_id,
|
|
created_at=123,
|
|
model="fake-model",
|
|
object="response",
|
|
output=_responses_output(text),
|
|
tool_choice="none",
|
|
tools=[],
|
|
parallel_tool_calls=False,
|
|
usage=ResponseUsage(
|
|
input_tokens=11,
|
|
output_tokens=13,
|
|
total_tokens=24,
|
|
input_tokens_details=InputTokensDetails.model_validate(
|
|
{"cache_write_tokens": 0, "cached_tokens": 0}
|
|
),
|
|
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
|
|
),
|
|
)
|
|
|
|
|
|
def _responses_response_with_terminal_status(status: str) -> Response:
|
|
return Response(
|
|
id="resp_terminal",
|
|
created_at=123,
|
|
model="fake-model",
|
|
object="response",
|
|
output=[],
|
|
tool_choice="none",
|
|
tools=[],
|
|
parallel_tool_calls=False,
|
|
usage=None,
|
|
status=status,
|
|
incomplete_details=(
|
|
IncompleteDetails(reason="max_output_tokens") if status == "incomplete" else None
|
|
),
|
|
)
|
|
|
|
|
|
def _chat_completion_with_tool_call(*, thought_signature: str) -> ChatCompletion:
|
|
return ChatCompletion(
|
|
id="chatcmpl_tool_123",
|
|
created=0,
|
|
model="fake-model",
|
|
object="chat.completion",
|
|
choices=[
|
|
Choice(
|
|
index=0,
|
|
finish_reason="tool_calls",
|
|
message=ChatCompletionMessage(
|
|
role="assistant",
|
|
content="Calling a tool.",
|
|
tool_calls=[
|
|
ChatCompletionMessageFunctionToolCall.model_validate(
|
|
{
|
|
"id": "call_123",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"arguments": '{"city":"Paris"}',
|
|
},
|
|
"extra_content": {
|
|
"google": {"thought_signature": thought_signature}
|
|
},
|
|
}
|
|
)
|
|
],
|
|
),
|
|
)
|
|
],
|
|
usage=CompletionUsage(
|
|
completion_tokens=5,
|
|
prompt_tokens=7,
|
|
total_tokens=12,
|
|
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
|
|
),
|
|
)
|
|
|
|
|
|
class GenericChatCompletionPayload(BaseModel):
|
|
id: str
|
|
created: int
|
|
model: str
|
|
object: str
|
|
choices: list[Any]
|
|
usage: Any
|
|
|
|
|
|
class GenericResponsesPayload(BaseModel):
|
|
id: str
|
|
created_at: float
|
|
model: str
|
|
object: str
|
|
output: list[Any]
|
|
parallel_tool_calls: bool
|
|
tool_choice: Any
|
|
tools: list[Any]
|
|
usage: Any
|
|
|
|
|
|
async def _empty_chat_stream() -> AsyncIterator[ChatCompletionChunk]:
|
|
if False:
|
|
yield ChatCompletionChunk(
|
|
id="chunk_123",
|
|
created=0,
|
|
model="fake-model",
|
|
object="chat.completion.chunk",
|
|
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason=None)],
|
|
)
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("api", ["chat_completions", "responses"])
|
|
@pytest.mark.parametrize(
|
|
("counts", "expected"),
|
|
[
|
|
((None, 5, 12), (0, 5, 12)),
|
|
((7, None, 12), (7, 0, 12)),
|
|
((7, 5, None), (7, 5, 0)),
|
|
((None, None, None), (0, 0, 0)),
|
|
((7, 5, 12), (7, 5, 12)),
|
|
(None, (0, 0, 0)),
|
|
],
|
|
ids=["null-input", "null-output", "null-total", "all-null", "valid", "no-usage"],
|
|
)
|
|
async def test_any_llm_runner_normalizes_nullable_usage(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
api: str,
|
|
counts: tuple[int | None, int | None, int | None] | None,
|
|
expected: tuple[int, int, int],
|
|
) -> None:
|
|
usage = (
|
|
{
|
|
"prompt_tokens": counts[0],
|
|
"completion_tokens": counts[1],
|
|
"total_tokens": counts[2],
|
|
"prompt_tokens_details": {"cached_tokens": 3},
|
|
"completion_tokens_details": {"reasoning_tokens": 2},
|
|
}
|
|
if counts is not None
|
|
else None
|
|
)
|
|
# Provider SDK parsing can construct typed responses without validating null counts.
|
|
if api == "responses":
|
|
usage = (
|
|
{
|
|
"input_tokens": counts[0],
|
|
"output_tokens": counts[1],
|
|
"total_tokens": counts[2],
|
|
"input_tokens_details": {"cached_tokens": 3, "cache_write_tokens": 0},
|
|
"output_tokens_details": {"reasoning_tokens": 2},
|
|
}
|
|
if counts is not None
|
|
else None
|
|
)
|
|
response = _response("Hello")
|
|
response.usage = ResponseUsage.model_construct(**usage) if usage is not None else None
|
|
if response.usage is not None:
|
|
response.usage.input_tokens_details = InputTokensDetails.model_validate(
|
|
{"cached_tokens": 3, "cache_write_tokens": 0}
|
|
)
|
|
response.usage.output_tokens_details = OutputTokensDetails(reasoning_tokens=2)
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response)
|
|
else:
|
|
chat = _chat_completion("Hello")
|
|
chat.usage = CompletionUsage.model_construct(**usage) if usage is not None else None
|
|
if chat.usage is not None:
|
|
chat.usage.prompt_tokens_details = PromptTokensDetails(cached_tokens=3)
|
|
chat.usage.completion_tokens_details = CompletionTokensDetails(reasoning_tokens=2)
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=chat)
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
agent = Agent(
|
|
name="test",
|
|
model=module.AnyLLMModel(model="openai/fake", api=api),
|
|
model_settings=ModelSettings(preserve_raw_usage=True),
|
|
)
|
|
result = await Runner.run(agent, "hi")
|
|
|
|
assert result.final_output == "Hello"
|
|
normalized = result.context_wrapper.usage
|
|
assert normalized.requests == 1
|
|
assert (normalized.input_tokens, normalized.output_tokens, normalized.total_tokens) == expected
|
|
assert normalized.input_tokens_details.cached_tokens == (3 if counts is not None else 0)
|
|
assert normalized.output_tokens_details.reasoning_tokens == (2 if counts is not None else 0)
|
|
assert result.raw_responses[0].raw_usage == usage
|
|
if expected == (0, 0, 0):
|
|
assert normalized.request_usage_entries == []
|
|
else:
|
|
assert len(normalized.request_usage_entries) == 1
|
|
entry = normalized.request_usage_entries[0]
|
|
assert (entry.input_tokens, entry.output_tokens, entry.total_tokens) == expected
|
|
assert entry.input_tokens_details.cached_tokens == 3
|
|
assert entry.output_tokens_details.reasoning_tokens == 2
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_routed_any_llm_chat_preserves_explicit_reasoning_effort(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
any_llm = pytest.importorskip("any_llm")
|
|
# Record the arguments delivered by the real adapter to the provider's Chat Completions API.
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
monkeypatch.setattr(any_llm.AnyLLM, "create", lambda *args, **kwargs: provider)
|
|
|
|
result = await Runner.run(
|
|
Agent(name="test", model="any-llm/openai/gpt-6-astra"),
|
|
"hi",
|
|
run_config=RunConfig(
|
|
model_settings=ModelSettings(extra_args={"reasoning_effort": "high"}),
|
|
tracing_disabled=True,
|
|
),
|
|
)
|
|
|
|
assert result.final_output == "Hello"
|
|
(call,) = provider.chat_calls
|
|
assert call["model"] == "gpt-6-astra"
|
|
assert call["reasoning_effort"] == "high"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
|
|
async def test_user_agent_header_any_llm_chat(override_ua: str | None, monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
expected_ua = override_ua or f"Agents/Python {__version__}"
|
|
|
|
if override_ua is not None:
|
|
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
|
|
else:
|
|
token = None
|
|
try:
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
finally:
|
|
if token is not None:
|
|
HEADERS_OVERRIDE.reset(token)
|
|
|
|
assert provider.chat_calls[0]["extra_headers"]["User-Agent"] == expected_ua
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("parallel_tool_calls", [True, False])
|
|
@pytest.mark.parametrize("tool_source", ["none", "function", "handoff"])
|
|
async def test_any_llm_chat_only_forwards_parallel_tool_calls_with_converted_tools(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
parallel_tool_calls: bool,
|
|
tool_source: str,
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(
|
|
model="openrouter/openai/gpt-5.4-mini",
|
|
api="chat_completions",
|
|
)
|
|
|
|
tools: list[Tool] = (
|
|
[function_tool(lambda: "ok", name_override="test_tool")]
|
|
if tool_source == "function"
|
|
else []
|
|
)
|
|
handoffs = [handoff(Agent(name="handoff"))] if tool_source == "handoff" else []
|
|
|
|
with trace("any-llm-offered-tools"):
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(parallel_tool_calls=parallel_tool_calls),
|
|
tools=tools,
|
|
output_schema=None,
|
|
handoffs=handoffs,
|
|
tracing=ModelTracing.ENABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
expected_parallel_tool_calls = parallel_tool_calls if tool_source != "none" else None
|
|
call = provider.chat_calls[0]
|
|
assert call["parallel_tool_calls"] is expected_parallel_tool_calls
|
|
assert (call["tools"] is not None) is (tool_source != "none")
|
|
(generation,) = [span for span in fetch_ordered_spans() if span.span_data.type == "generation"]
|
|
assert generation.span_data.tools == call["tools"]
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_preserves_falsy_reasoning(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
class FalsyReasoning(Reasoning):
|
|
def __bool__(self) -> bool:
|
|
return False
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(reasoning=FalsyReasoning(effort="low")),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
assert provider.chat_calls[0]["reasoning_effort"] == "low"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_nests_extra_body_instead_of_flattening(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
extra_body = {"cached_content": "some_cache", "foo": 123, "temperature": 0.9}
|
|
settings = ModelSettings(
|
|
temperature=0.1,
|
|
extra_body=extra_body,
|
|
extra_query={},
|
|
metadata={},
|
|
)
|
|
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=settings,
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
call = provider.chat_calls[0]
|
|
assert call["temperature"] == 0.1
|
|
assert call["extra_body"] == extra_body
|
|
assert call["extra_body"] is not extra_body
|
|
assert call["extra_query"] == {}
|
|
assert call["metadata"] == {}
|
|
assert "cached_content" not in call
|
|
assert "foo" not in call
|
|
extra_body["foo"] = 999
|
|
assert call["extra_body"]["foo"] == 123
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("provider_name", ["gemini", "vertexai"])
|
|
@pytest.mark.parametrize("options_type", ["unset", "dictionary", "model"])
|
|
async def test_any_llm_google_chat_headers_use_http_options(
|
|
monkeypatch: pytest.MonkeyPatch, provider_name: str, options_type: str
|
|
) -> None:
|
|
class HttpOptions(BaseModel):
|
|
headers: dict[str, str]
|
|
timeout: int
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model=f"{provider_name}/gemini-2.5-flash")
|
|
|
|
extra_args: dict[str, Any] = {}
|
|
configured_options: dict[str, Any] | HttpOptions | None = None
|
|
if options_type == "dictionary":
|
|
configured_options = {"headers": {"X-Existing": "existing"}, "timeout": 1000}
|
|
extra_args["http_options"] = configured_options
|
|
elif options_type == "model":
|
|
configured_options = HttpOptions(headers={"X-Existing": "existing"}, timeout=1000)
|
|
extra_args["http_options"] = configured_options
|
|
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(
|
|
extra_args=extra_args,
|
|
extra_headers={"X-Test-Header": "test"},
|
|
),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
call = provider.chat_calls[0]
|
|
assert "extra_headers" not in call
|
|
http_options = call["http_options"]
|
|
if isinstance(http_options, BaseModel):
|
|
http_options = http_options.model_dump()
|
|
assert http_options["headers"]["User-Agent"] == f"Agents/Python {__version__}"
|
|
assert http_options["headers"]["X-Test-Header"] == "test"
|
|
if configured_options is not None:
|
|
assert http_options["headers"]["X-Existing"] == "existing"
|
|
assert http_options["timeout"] == 1000
|
|
if isinstance(configured_options, BaseModel):
|
|
assert configured_options.headers == {"X-Existing": "existing"}
|
|
else:
|
|
assert configured_options["headers"] == {"X-Existing": "existing"}
|
|
|
|
|
|
@pytest.mark.parametrize("provider_name", ["gemini", "vertexai"])
|
|
@pytest.mark.parametrize("content_type", ["model", "dictionary"])
|
|
def test_any_llm_google_provider_normalizes_function_result_roles(
|
|
monkeypatch: pytest.MonkeyPatch, provider_name: str, content_type: str
|
|
) -> None:
|
|
class GoogleContent(BaseModel):
|
|
role: str
|
|
parts: list[dict[str, Any]]
|
|
|
|
tool_result: dict[str, Any] = {
|
|
"role": "function",
|
|
"parts": [{"function_response": {"name": "get_weather", "response": {"result": "sunny"}}}],
|
|
}
|
|
original_tool_result: GoogleContent | dict[str, Any]
|
|
if content_type == "model":
|
|
original_tool_result = GoogleContent.model_validate(tool_result)
|
|
else:
|
|
original_tool_result = tool_result
|
|
|
|
class GoogleProvider(FakeAnyLLMProvider):
|
|
@staticmethod
|
|
def _convert_completion_params(*args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
return {
|
|
"model": "gemini-3.6-flash",
|
|
"contents": [
|
|
GoogleContent(role="user", parts=[{"text": "Check the weather."}]),
|
|
original_tool_result,
|
|
GoogleContent(role="model", parts=[{"text": "Done."}]),
|
|
],
|
|
}
|
|
|
|
provider = GoogleProvider(supports_responses=False)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model=f"{provider_name}/gemini-3.6-flash")
|
|
|
|
converted = model._get_provider()._convert_completion_params(object())
|
|
contents = converted["contents"]
|
|
|
|
assert [
|
|
item.role if isinstance(item, GoogleContent) else item["role"] for item in contents
|
|
] == [
|
|
"user",
|
|
"user",
|
|
"model",
|
|
]
|
|
normalized_tool_result = contents[1]
|
|
if isinstance(normalized_tool_result, BaseModel):
|
|
normalized_tool_result = normalized_tool_result.model_dump()
|
|
assert normalized_tool_result["parts"] == tool_result["parts"]
|
|
assert (
|
|
original_tool_result.role
|
|
if isinstance(original_tool_result, GoogleContent)
|
|
else original_tool_result["role"]
|
|
) == "function"
|
|
|
|
|
|
def test_any_llm_non_google_provider_does_not_normalize_function_result_roles(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
class NonGoogleProvider(FakeAnyLLMProvider):
|
|
@staticmethod
|
|
def _convert_completion_params(*args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
return {"contents": [{"role": "function", "parts": [{"result": "ok"}]}]}
|
|
|
|
provider = NonGoogleProvider(supports_responses=False)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openrouter/google/gemini-3.6-flash")
|
|
|
|
converted = model._get_provider()._convert_completion_params(object())
|
|
|
|
assert converted["contents"][0]["role"] == "function"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_path_is_used_when_responses_are_unsupported(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="router-key")
|
|
response = await model.get_response(
|
|
system_instructions="You are terse.",
|
|
input="hi",
|
|
model_settings=ModelSettings(preserve_raw_usage=True),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id="resp_prev",
|
|
conversation_id="conv_123",
|
|
prompt=None,
|
|
)
|
|
|
|
assert create_calls == [
|
|
{
|
|
"provider_name": "openrouter",
|
|
"api_key": "router-key",
|
|
"api_base": None,
|
|
}
|
|
]
|
|
assert len(provider.chat_calls) == 1
|
|
assert provider.responses_calls == []
|
|
assert provider.chat_calls[0]["model"] == "openai/gpt-5.4-mini"
|
|
assert response.response_id is None
|
|
assert response.output[0].content[0].text == "Hello"
|
|
assert response.usage.input_tokens_details.cached_tokens == 2
|
|
assert getattr(response.usage.input_tokens_details, "cache_write_tokens", None) == 4
|
|
assert response.raw_usage is not None
|
|
assert response.raw_usage["prompt_tokens_details"] == {
|
|
"cached_tokens": 2,
|
|
"cache_write_tokens": 4,
|
|
}
|
|
|
|
|
|
def _content_filtered_chat_completion(content: str) -> ChatCompletion:
|
|
completion = _chat_completion(content)
|
|
completion.choices[0].finish_reason = "content_filter"
|
|
return completion
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_path_surfaces_content_filter_refusal(monkeypatch) -> None:
|
|
"""A filtered turn must become a refusal instead of an empty output."""
|
|
provider = FakeAnyLLMProvider(
|
|
supports_responses=False,
|
|
chat_response=_content_filtered_chat_completion(""),
|
|
)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
|
|
model = module.AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
response = await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
refusals = [
|
|
content
|
|
for item in response.output
|
|
if isinstance(item, ResponseOutputMessage)
|
|
for content in item.content
|
|
if isinstance(content, ResponseOutputRefusal)
|
|
]
|
|
assert refusals, f"expected a refusal item, got: {response.output}"
|
|
assert refusals[0].refusal
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_path_content_filter_keeps_real_content(monkeypatch) -> None:
|
|
"""A filtered turn that still carries text keeps the text and gains no refusal."""
|
|
provider = FakeAnyLLMProvider(
|
|
supports_responses=False,
|
|
chat_response=_content_filtered_chat_completion("here is the answer"),
|
|
)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
|
|
model = module.AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
response = await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
refusals = [
|
|
content
|
|
for item in response.output
|
|
if isinstance(item, ResponseOutputMessage)
|
|
for content in item.content
|
|
if isinstance(content, ResponseOutputRefusal)
|
|
]
|
|
assert not refusals
|
|
assert response.output[0].content[0].text == "here is the answer"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"chat_response",
|
|
[
|
|
pytest.param(_chat_completion("Hello").model_dump(), id="dict"),
|
|
pytest.param(
|
|
GenericChatCompletionPayload.model_validate(_chat_completion("Hello").model_dump()),
|
|
id="basemodel",
|
|
),
|
|
],
|
|
)
|
|
async def test_any_llm_chat_path_normalizes_non_stream_payloads(
|
|
monkeypatch,
|
|
chat_response: Any,
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=chat_response)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
response = await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
assert response.response_id is None
|
|
assert response.output[0].content[0].text == "Hello"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_path_preserves_plaintext_reasoning_for_replay(monkeypatch) -> None:
|
|
chat_response = _chat_completion("The answer is 42.")
|
|
chat_response.choices[0].message = ChatCompletionMessage.model_validate(
|
|
{
|
|
"role": "assistant",
|
|
"content": "The answer is 42.",
|
|
"reasoning_content": "",
|
|
"reasoning": "I should calculate this carefully.",
|
|
}
|
|
)
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=chat_response)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openrouter/reasoning-model")
|
|
|
|
model_settings = ModelSettings(reasoning=Reasoning(effort="high"))
|
|
response = await model.get_response(
|
|
system_instructions=None,
|
|
input="What is six times seven?",
|
|
model_settings=model_settings,
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input=response.to_input_items(),
|
|
model_settings=model_settings,
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
replayed_messages = provider.chat_calls[1]["messages"]
|
|
assert replayed_messages == [
|
|
{
|
|
"role": "assistant",
|
|
"content": "The answer is 42.",
|
|
"reasoning": "I should calculate this carefully.",
|
|
}
|
|
]
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_path_preserves_gemini_tool_call_metadata(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(
|
|
supports_responses=False,
|
|
chat_response=_chat_completion_with_tool_call(thought_signature="sig_123"),
|
|
)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="gemini/gemini-2.0-flash")
|
|
response = await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
function_calls = [
|
|
item for item in response.output if getattr(item, "type", None) == "function_call"
|
|
]
|
|
assert len(function_calls) == 1
|
|
provider_data = function_calls[0].model_dump()["provider_data"]
|
|
assert provider_data["model"] == "gemini/gemini-2.0-flash"
|
|
assert provider_data["response_id"] == "chatcmpl_tool_123"
|
|
assert provider_data["thought_signature"] == "sig_123"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_path_is_used_when_supported(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
|
module, create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="gpt-5.4-mini", api_key="openai-key")
|
|
response = await model.get_response(
|
|
system_instructions="You are terse.",
|
|
input="hi",
|
|
model_settings=ModelSettings(store=True),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id="resp_prev",
|
|
conversation_id="conv_123",
|
|
prompt=None,
|
|
)
|
|
|
|
assert create_calls == [
|
|
{
|
|
"provider_name": "openai",
|
|
"api_key": "openai-key",
|
|
"api_base": None,
|
|
}
|
|
]
|
|
assert provider.chat_calls == []
|
|
assert provider.responses_calls == []
|
|
assert len(provider.private_responses_calls) == 1
|
|
params = provider.private_responses_calls[0]["params"]
|
|
kwargs = provider.private_responses_calls[0]["kwargs"]
|
|
assert params.model == "gpt-5.4-mini"
|
|
assert params.previous_response_id == "resp_prev"
|
|
assert params.conversation == "conv_123"
|
|
assert kwargs["extra_headers"]["User-Agent"] == f"Agents/Python {__version__}"
|
|
assert response.response_id == "resp_123"
|
|
assert response.output[0].content[0].text == "Hello"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("status", ["incomplete", "failed"])
|
|
async def test_any_llm_responses_path_rejects_failed_terminal_status(
|
|
monkeypatch, status: str
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(
|
|
supports_responses=True,
|
|
responses_response=_responses_response_with_terminal_status(status),
|
|
)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
|
|
model = module.AnyLLMModel(model="openai/gpt-5.4-mini", api_key="openai-key")
|
|
with pytest.raises(ModelBehaviorError, match=f"response.{status}"):
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("parallel_tool_calls", [True, False, None])
|
|
@pytest.mark.parametrize("tool_source", ["none", "function", "handoff"])
|
|
async def test_any_llm_responses_parallel_tool_calls_follow_converted_tools(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
parallel_tool_calls: bool | None,
|
|
tool_source: str,
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openai/gpt-5.4-mini", api="responses")
|
|
tools: list[Tool] = (
|
|
[function_tool(lambda: "ok", name_override="test_tool")]
|
|
if tool_source == "function"
|
|
else []
|
|
)
|
|
handoffs = [handoff(Agent(name="handoff"))] if tool_source == "handoff" else []
|
|
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(parallel_tool_calls=parallel_tool_calls),
|
|
tools=tools,
|
|
output_schema=None,
|
|
handoffs=handoffs,
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
params = provider.private_responses_calls[0]["params"]
|
|
expected_parallel_tool_calls = parallel_tool_calls if tool_source != "none" else None
|
|
assert params.parallel_tool_calls is expected_parallel_tool_calls
|
|
assert bool(params.tools) is (tool_source != "none")
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("payload_type", ["dict", "basemodel"])
|
|
async def test_any_llm_responses_path_defaults_missing_cache_write_tokens(
|
|
monkeypatch: pytest.MonkeyPatch, payload_type: str
|
|
) -> None:
|
|
response_payload = _response("Hello").model_dump()
|
|
response_payload["usage"]["input_tokens_details"].pop("cache_write_tokens")
|
|
response: Any = response_payload
|
|
if payload_type == "basemodel":
|
|
response = GenericResponsesPayload.model_validate(response_payload)
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
|
|
normalized = await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(preserve_raw_usage=True),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
assert normalized.output[0].content[0].text == "Hello"
|
|
assert normalized.usage.input_tokens_details.cache_write_tokens == 0
|
|
assert "cache_write_tokens" not in response_payload["usage"]["input_tokens_details"]
|
|
assert normalized.raw_usage is not None
|
|
assert "cache_write_tokens" not in normalized.raw_usage["input_tokens_details"]
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_can_force_chat_completions_when_responses_are_supported(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(
|
|
supports_responses=True,
|
|
chat_response=_chat_completion("Hello from chat"),
|
|
responses_response=_response("Hello from responses"),
|
|
)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openai/gpt-4.1-mini", api="chat_completions")
|
|
response = await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id="resp_prev",
|
|
conversation_id="conv_123",
|
|
prompt=None,
|
|
)
|
|
|
|
assert len(provider.chat_calls) == 1
|
|
assert provider.responses_calls == []
|
|
assert response.response_id is None
|
|
assert response.output[0].content[0].text == "Hello from chat"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_forced_responses_errors_when_provider_does_not_support_it(
|
|
monkeypatch,
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openrouter/openai/gpt-4.1-mini", api="responses")
|
|
with pytest.raises(UserError, match="does not support the Responses API"):
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_stream_uses_chat_handler_when_responses_are_unsupported(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_empty_chat_stream())
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
completed = ResponseCompletedEvent(
|
|
type="response.completed",
|
|
response=_response("Hello from stream"),
|
|
sequence_number=1,
|
|
)
|
|
|
|
async def fake_handle_stream(response, stream, model=None):
|
|
assert model == "openrouter/openai/gpt-5.4-mini"
|
|
async for _chunk in stream:
|
|
pass
|
|
yield completed
|
|
|
|
monkeypatch.setattr(module.ChatCmplStreamHandler, "handle_stream", fake_handle_stream)
|
|
|
|
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
events = [
|
|
event
|
|
async for event in model.stream_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
]
|
|
|
|
assert [event.type for event in events] == ["response.completed"]
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_stream_passthrough_uses_responses_when_supported(monkeypatch) -> None:
|
|
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
|
|
yield ResponseCompletedEvent(
|
|
type="response.completed",
|
|
response=_response("Hello from responses stream"),
|
|
sequence_number=1,
|
|
)
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
events = [
|
|
event
|
|
async for event in model.stream_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id="resp_prev",
|
|
conversation_id="conv_123",
|
|
prompt=None,
|
|
)
|
|
]
|
|
|
|
assert [event.type for event in events] == ["response.completed"]
|
|
assert provider.responses_calls == []
|
|
assert provider.private_responses_calls[0]["params"].previous_response_id == "resp_prev"
|
|
assert provider.private_responses_calls[0]["params"].conversation == "conv_123"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("terminal_event_type", "terminal_event_cls"),
|
|
[
|
|
("response.incomplete", ResponseIncompleteEvent),
|
|
("response.failed", ResponseFailedEvent),
|
|
],
|
|
)
|
|
async def test_any_llm_responses_stream_rejects_failed_terminal_events(
|
|
monkeypatch,
|
|
terminal_event_type: str,
|
|
terminal_event_cls: type[Any],
|
|
) -> None:
|
|
async def response_stream() -> AsyncIterator[Any]:
|
|
yield terminal_event_cls(
|
|
type=terminal_event_type,
|
|
response=_response("partial", response_id="resp-terminal"),
|
|
sequence_number=1,
|
|
)
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
events = []
|
|
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
|
|
async for event in model.stream_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
):
|
|
events.append(event)
|
|
|
|
assert len(events) == 1
|
|
assert events[0].type == terminal_event_type
|
|
assert events[0].response.id == "resp-terminal"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_rejects_error_event(monkeypatch) -> None:
|
|
async def response_stream() -> AsyncIterator[ResponseErrorEvent]:
|
|
yield ResponseErrorEvent(
|
|
type="error",
|
|
code="invalid_request_error",
|
|
message="bad request",
|
|
param=None,
|
|
sequence_number=1,
|
|
)
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
events = []
|
|
with pytest.raises(ModelBehaviorError, match="invalid_request_error"):
|
|
async for event in model.stream_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
):
|
|
events.append(event)
|
|
|
|
assert len(events) == 1
|
|
assert events[0].type == "error"
|
|
assert events[0].code == "invalid_request_error"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_path_passes_transport_kwargs_via_private_provider_api(
|
|
monkeypatch,
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(
|
|
extra_headers={"X-Test-Header": "test"},
|
|
extra_query={"trace": "1"},
|
|
extra_body={"foo": "bar"},
|
|
),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
assert provider.responses_calls == []
|
|
assert len(provider.private_responses_calls) == 1
|
|
call = provider.private_responses_calls[0]
|
|
assert call["kwargs"]["extra_headers"]["X-Test-Header"] == "test"
|
|
assert call["kwargs"]["extra_query"] == {"trace": "1"}
|
|
assert call["kwargs"]["extra_body"] == {"foo": "bar"}
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_prompt_requests_fail_fast(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
with pytest.raises(Exception, match="prompt-managed requests"):
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt={"id": "pmpt_123"},
|
|
)
|
|
|
|
|
|
def test_any_llm_responses_input_sanitizer_strips_none_fields_from_reasoning_items() -> None:
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
from agents.extensions.models.any_llm_model import AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
raw_input = [
|
|
{
|
|
"id": "rid1",
|
|
"summary": [{"text": "why", "type": "summary_text"}],
|
|
"type": "reasoning",
|
|
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
|
"status": None,
|
|
"encrypted_content": None,
|
|
}
|
|
]
|
|
|
|
cleaned = model._sanitize_any_llm_responses_input(raw_input)
|
|
|
|
assert cleaned == [
|
|
{
|
|
"id": "rid1",
|
|
"summary": [{"text": "why", "type": "summary_text"}],
|
|
"type": "reasoning",
|
|
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
|
}
|
|
]
|
|
|
|
ResponsesParams = importlib.import_module("any_llm.types.responses").ResponsesParams
|
|
params = ResponsesParams(model="dummy", input=cleaned)
|
|
assert isinstance(params.input, list)
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_path_sanitizes_replayed_items_before_validation() -> None:
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
from agents.extensions.models.any_llm_model import AnyLLMModel
|
|
|
|
class ValidatingProvider:
|
|
SUPPORTS_RESPONSES = True
|
|
|
|
def __init__(self) -> None:
|
|
self.private_responses_calls: list[dict[str, Any]] = []
|
|
|
|
async def aresponses(self, **kwargs: Any) -> Any:
|
|
raise AssertionError("public aresponses path should not be used in this test")
|
|
|
|
async def _aresponses(self, params: Any, **kwargs: Any) -> Response:
|
|
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
|
|
return _response("Hello from sanitized replay")
|
|
|
|
class TestAnyLLMModel(AnyLLMModel):
|
|
def __init__(self, provider: ValidatingProvider) -> None:
|
|
super().__init__(model="openai/gpt-5.4-mini", api="responses")
|
|
self._provider = provider
|
|
|
|
def _get_provider(self) -> Any:
|
|
return self._provider
|
|
|
|
provider = ValidatingProvider()
|
|
model = TestAnyLLMModel(provider)
|
|
tools: list[Tool] = []
|
|
handoffs: list[Handoff[Any, Agent[Any]]] = []
|
|
stream_flag: Literal[False] = False
|
|
|
|
replay_input = cast(
|
|
list[TResponseInputItem],
|
|
[
|
|
{"role": "user", "content": "What's the weather in Tokyo?"},
|
|
{
|
|
"id": FAKE_RESPONSES_ID,
|
|
"summary": [
|
|
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
|
],
|
|
"type": "reasoning",
|
|
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
|
"status": None,
|
|
"provider_data": {"model": "anthropic/fake-responses-model"},
|
|
},
|
|
{
|
|
"id": FAKE_RESPONSES_ID,
|
|
"arguments": '{"city": "Tokyo"}',
|
|
"call_id": "call_weather_123",
|
|
"name": "get_weather",
|
|
"type": "function_call",
|
|
"status": None,
|
|
"provider_data": {"model": "anthropic/fake-responses-model"},
|
|
},
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_weather_123",
|
|
"output": "The weather in Tokyo is sunny and 22°C.",
|
|
},
|
|
],
|
|
)
|
|
|
|
response = await model._fetch_responses_response(
|
|
system_instructions=None,
|
|
input=replay_input,
|
|
model_settings=ModelSettings(),
|
|
tools=tools,
|
|
output_schema=None,
|
|
handoffs=handoffs,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
stream=stream_flag,
|
|
prompt=None,
|
|
)
|
|
|
|
assert response.id == "resp_123"
|
|
assert len(provider.private_responses_calls) == 1
|
|
params = provider.private_responses_calls[0]["params"]
|
|
assert params.input == [
|
|
{"role": "user", "content": "What's the weather in Tokyo?"},
|
|
{
|
|
"arguments": '{"city": "Tokyo"}',
|
|
"call_id": "call_weather_123",
|
|
"name": "get_weather",
|
|
"type": "function_call",
|
|
},
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_weather_123",
|
|
"output": "The weather in Tokyo is sunny and 22°C.",
|
|
},
|
|
]
|
|
|
|
|
|
class _RecordingResponsesProvider:
|
|
"""Provider stub that records the params any-llm's private responses API receives."""
|
|
|
|
SUPPORTS_RESPONSES = True
|
|
|
|
def __init__(self, response: Any) -> None:
|
|
self._response = response
|
|
self.private_responses_calls: list[dict[str, Any]] = []
|
|
|
|
async def aresponses(self, **kwargs: Any) -> Any:
|
|
raise AssertionError("public aresponses path should not be used in this test")
|
|
|
|
async def _aresponses(self, params: Any, **kwargs: Any) -> Any:
|
|
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
|
|
return self._response
|
|
|
|
|
|
def _model_bound_to_provider(provider: Any) -> Any:
|
|
from agents.extensions.models.any_llm_model import AnyLLMModel
|
|
|
|
class _BoundAnyLLMModel(AnyLLMModel):
|
|
def _get_provider(self) -> Any:
|
|
return provider
|
|
|
|
return _BoundAnyLLMModel(model="openai/gpt-5.4-mini", api="responses")
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("stream", [False, True])
|
|
async def test_any_llm_responses_path_sends_reasoning_as_a_mapping(stream: bool) -> None:
|
|
"""any-llm types `ResponsesParams.reasoning` as a mapping, so it must not be a pair list."""
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
|
|
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
|
|
yield ResponseCompletedEvent(
|
|
type="response.completed",
|
|
response=_response("Hello"),
|
|
sequence_number=1,
|
|
)
|
|
|
|
provider = _RecordingResponsesProvider(response_stream() if stream else _response("Hello"))
|
|
model = _model_bound_to_provider(provider)
|
|
|
|
# Building `ResponsesParams` validates the payload, so a pair list fails before the
|
|
# provider is reached.
|
|
result = await cast(Any, model)._fetch_responses_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(reasoning=Reasoning(effort="low", summary="concise")),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
stream=stream,
|
|
prompt=None,
|
|
)
|
|
if stream:
|
|
await result.aclose()
|
|
|
|
assert len(provider.private_responses_calls) == 1
|
|
params = provider.private_responses_calls[0]["params"]
|
|
# Unset reasoning fields are dropped, matching how this adapter sanitizes replayed input.
|
|
assert params.reasoning == {"effort": "low", "summary": "concise"}
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_path_omits_reasoning_when_unset() -> None:
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
|
|
provider = _RecordingResponsesProvider(_response("Hello"))
|
|
model = _model_bound_to_provider(provider)
|
|
|
|
await cast(Any, model)._fetch_responses_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
stream=False,
|
|
prompt=None,
|
|
)
|
|
|
|
assert len(provider.private_responses_calls) == 1
|
|
assert provider.private_responses_calls[0]["params"].reasoning is None
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("stream", [False, True])
|
|
async def test_any_llm_responses_path_forwards_prompt_cache_retention(stream: bool) -> None:
|
|
"""`ModelSettings.prompt_cache_retention` must reach the any-llm Responses request."""
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
|
|
provider = _RecordingResponsesProvider(_response("Hello"))
|
|
model = _model_bound_to_provider(provider)
|
|
|
|
await cast(Any, model)._fetch_responses_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(prompt_cache_retention="24h"),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
stream=stream,
|
|
prompt=None,
|
|
)
|
|
|
|
assert len(provider.private_responses_calls) == 1
|
|
assert provider.private_responses_calls[0]["params"].prompt_cache_retention == "24h"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_path_omits_prompt_cache_retention_when_unset() -> None:
|
|
"""An unset retention stays unset instead of pinning a default on the request."""
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
|
|
provider = _RecordingResponsesProvider(_response("Hello"))
|
|
model = _model_bound_to_provider(provider)
|
|
|
|
await cast(Any, model)._fetch_responses_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
stream=False,
|
|
prompt=None,
|
|
)
|
|
|
|
assert len(provider.private_responses_calls) == 1
|
|
assert provider.private_responses_calls[0]["params"].prompt_cache_retention is None
|
|
|
|
|
|
def test_any_llm_provider_passes_api_override() -> None:
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
from agents.extensions.models.any_llm_model import AnyLLMModel
|
|
from agents.extensions.models.any_llm_provider import AnyLLMProvider
|
|
|
|
provider = AnyLLMProvider(api="chat_completions")
|
|
model = provider.get_model("openai/gpt-4.1-mini")
|
|
|
|
assert isinstance(model, AnyLLMModel)
|
|
assert model.api == "chat_completions"
|
|
|
|
|
|
def test_any_llm_provider_reads_default_model_at_call_time(monkeypatch: Any) -> None:
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
from agents.extensions.models.any_llm_provider import AnyLLMProvider
|
|
|
|
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", "gpt-4.1")
|
|
provider = AnyLLMProvider()
|
|
|
|
assert cast(Any, provider.get_model(None)).model == "openai/gpt-4.1"
|
|
|
|
|
|
def test_any_llm_reasoning_objects_prefer_content_attributes_over_iterable_pairs() -> None:
|
|
pytest.importorskip(
|
|
"any_llm",
|
|
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
|
)
|
|
from any_llm.types.completion import Reasoning
|
|
|
|
from agents.extensions.models.any_llm_model import _extract_any_llm_reasoning_text
|
|
|
|
delta = pytypes.SimpleNamespace(reasoning=Reasoning(content="用户"))
|
|
|
|
assert _extract_any_llm_reasoning_text(delta) == "用户"
|
|
|
|
|
|
def test_any_llm_stream_flattens_reasoning_object_when_reasoning_content_is_empty(
|
|
monkeypatch,
|
|
) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
delta = ChoiceDelta.model_construct(
|
|
reasoning_content="",
|
|
reasoning=pytypes.SimpleNamespace(content="Plaintext reasoning"),
|
|
)
|
|
chunk = ChatCompletionChunk(
|
|
id="chunk-id",
|
|
created=1,
|
|
model="fake",
|
|
object="chat.completion.chunk",
|
|
choices=[ChunkChoice(index=0, delta=delta)],
|
|
)
|
|
|
|
normalized = module.AnyLLMModel(model="openrouter/reasoning-model")._normalize_chat_chunk(chunk)
|
|
|
|
assert normalized.choices[0].delta.reasoning == "Plaintext reasoning"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
async def test_any_llm_thinking_is_omitted_from_default_trace_export(monkeypatch) -> None:
|
|
secret = "PRIVATE_ANY_LLM_THINKING_SENTINEL"
|
|
completion = _chat_completion("Visible answer")
|
|
completion.choices[0].message = ChatCompletionMessage.model_validate(
|
|
{"role": "assistant", "content": "Visible answer", "thinking": secret}
|
|
)
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=completion)
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openrouter/test-model", api="chat_completions")
|
|
result = await Runner.run(
|
|
Agent(name="Test", model=model),
|
|
"Question",
|
|
run_config=RunConfig(trace_include_sensitive_data=True),
|
|
)
|
|
generation = next(span for span in fetch_ordered_spans() if span.span_data.type == "generation")
|
|
original = copy.deepcopy(generation.export())
|
|
assert original is not None
|
|
assert original["span_data"]["output"][0]["thinking"] == secret
|
|
replay = result.to_input_list()
|
|
assert secret in json.dumps(replay)
|
|
assert result.final_output == "Visible answer"
|
|
|
|
payloads: list[dict[str, Any]] = []
|
|
|
|
def post(self, url, **kwargs):
|
|
payloads.append(copy.deepcopy(kwargs["json"]))
|
|
return httpx2.Response(200)
|
|
|
|
monkeypatch.setattr(httpx2.Client, "post", post)
|
|
exporter = BackendSpanExporter(api_key="test-key", max_retries=0)
|
|
try:
|
|
exporter.export([generation])
|
|
sent = payloads[-1]["data"][0]["span_data"]
|
|
assert secret not in json.dumps(sent)
|
|
assert sent["output"][0]["content"] == "Visible answer"
|
|
assert sent["usage"]["output_tokens"] == 5
|
|
assert generation.export() == original
|
|
assert result.to_input_list() == replay
|
|
finally:
|
|
exporter.close()
|
|
await model.close()
|
|
|
|
|
|
def test_any_llm_nonstream_preserves_native_reasoning_content_field(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
message = ChatCompletionMessage.model_validate(
|
|
{
|
|
"role": "assistant",
|
|
"content": "Answer",
|
|
"reasoning": "Plaintext reasoning",
|
|
"reasoning_content": "Native reasoning content",
|
|
}
|
|
)
|
|
|
|
normalized = module._normalize_any_llm_message(message)
|
|
|
|
assert normalized.reasoning_content == "Native reasoning content"
|
|
assert normalized.reasoning == ""
|
|
|
|
|
|
def test_any_llm_split_does_not_duplicate_content_or_thinking(monkeypatch) -> None:
|
|
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
|
|
|
|
Anthropic's extended thinking API rejects requests that include the same signed
|
|
thinking block more than once, and duplicated assistant text corrupts conversation
|
|
history. Only the first split should retain content, thinking_blocks, and
|
|
reasoning_content; subsequent splits should carry the tool_call alone.
|
|
"""
|
|
provider = FakeAnyLLMProvider(supports_responses=False)
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="anthropic/claude-3-5-sonnet")
|
|
messages: list[Any] = [
|
|
{"role": "user", "content": "Search both"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "Looking up both queries.",
|
|
"thinking_blocks": [{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}],
|
|
"reasoning_content": "internal plan",
|
|
"reasoning": "plaintext plan",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"type": "function",
|
|
"function": {"name": "s", "arguments": "{}"},
|
|
},
|
|
{
|
|
"id": "call_2",
|
|
"type": "function",
|
|
"function": {"name": "s", "arguments": "{}"},
|
|
},
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
|
|
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
|
|
]
|
|
|
|
result = model._fix_tool_message_ordering(messages)
|
|
|
|
assistants = [m for m in result if m.get("role") == "assistant"]
|
|
assert len(assistants) == 2
|
|
# First split keeps the shared fields.
|
|
assert assistants[0].get("content") == "Looking up both queries."
|
|
assert "thinking_blocks" in assistants[0]
|
|
assert "reasoning_content" in assistants[0]
|
|
assert "reasoning" in assistants[0]
|
|
# Second split must NOT duplicate them.
|
|
assert "content" not in assistants[1]
|
|
assert "thinking_blocks" not in assistants[1]
|
|
assert "reasoning_content" not in assistants[1]
|
|
assert "reasoning" not in assistants[1]
|
|
# Tool calls are still split one-per-message.
|
|
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
|
|
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_sets_logprobs_when_top_logprobs_set(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="k")
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(top_logprobs=2),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
# The Chat Completions API rejects top_logprobs unless logprobs is True.
|
|
assert provider.chat_calls[0]["top_logprobs"] == 2
|
|
assert provider.chat_calls[0]["logprobs"] is True
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_omits_logprobs_when_top_logprobs_unset(monkeypatch) -> None:
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
AnyLLMModel = module.AnyLLMModel
|
|
|
|
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="k")
|
|
await model.get_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
assert "logprobs" not in provider.chat_calls[0]
|
|
|
|
|
|
class _ClosableStream:
|
|
"""Stands in for the iterator any-llm returns, recording how often it was closed.
|
|
|
|
These fakes deliberately cover only the SDK's own boundary. any-llm's exception and
|
|
provider wrappers (`utils/exception_handler.py::_wrap_async_iterator`, the per-provider
|
|
`chunk_iterator` / `_stream_completion_async` generators) delegate with bare
|
|
`async for ... yield` as of 1.11.0 and do not forward `aclose()` to the underlying
|
|
transport, so no assertion here should be read as proving the transport was closed.
|
|
"""
|
|
|
|
def __init__(self, chunks: list[Any]) -> None:
|
|
self._chunks = list(chunks)
|
|
self.aclose_calls = 0
|
|
|
|
def __aiter__(self) -> _ClosableStream:
|
|
return self
|
|
|
|
async def __anext__(self) -> Any:
|
|
if not self._chunks:
|
|
raise StopAsyncIteration
|
|
return self._chunks.pop(0)
|
|
|
|
async def aclose(self) -> None:
|
|
self.aclose_calls += 1
|
|
|
|
|
|
class _FailingCloseStream(_ClosableStream):
|
|
"""Raises from `aclose` after recording the cleanup attempt."""
|
|
|
|
async def aclose(self) -> None:
|
|
self.aclose_calls += 1
|
|
raise RuntimeError("close-failure")
|
|
|
|
|
|
class _BlockingStream(_ClosableStream):
|
|
"""Blocks forever after its chunks are exhausted so the consumer can be cancelled."""
|
|
|
|
def __init__(self, chunks: list[Any], blocked: asyncio.Event) -> None:
|
|
super().__init__(chunks)
|
|
self._blocked = blocked
|
|
|
|
async def __anext__(self) -> Any:
|
|
if self._chunks:
|
|
return self._chunks.pop(0)
|
|
self._blocked.set()
|
|
await asyncio.Event().wait()
|
|
raise AssertionError("unreachable")
|
|
|
|
|
|
class _SlowCloseStream(_BlockingStream):
|
|
"""Blocks in `aclose` until released, mirroring a close that waits on transport I/O."""
|
|
|
|
def __init__(self, chunks: list[Any], blocked: asyncio.Event, release: asyncio.Event) -> None:
|
|
super().__init__(chunks, blocked)
|
|
self._release = release
|
|
self.aclose_completed = 0
|
|
|
|
async def aclose(self) -> None:
|
|
self.aclose_calls += 1
|
|
await self._release.wait()
|
|
self.aclose_completed += 1
|
|
|
|
|
|
def _chat_chunk(text: str) -> ChatCompletionChunk:
|
|
return ChatCompletionChunk(
|
|
id="chunk_123",
|
|
created=0,
|
|
model="fake-model",
|
|
object="chat.completion.chunk",
|
|
choices=[ChunkChoice(index=0, delta=ChoiceDelta(content=text))],
|
|
)
|
|
|
|
|
|
def _completed_event() -> ResponseCompletedEvent:
|
|
return ResponseCompletedEvent(
|
|
type="response.completed",
|
|
response=_response("Hello"),
|
|
sequence_number=1,
|
|
)
|
|
|
|
|
|
def _stream_events(model: Any, tracing: ModelTracing = ModelTracing.DISABLED) -> Any:
|
|
return model.stream_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=tracing,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
|
|
|
|
def _chat_module_and_model_with_stream(monkeypatch, stream: _ClosableStream) -> tuple[Any, Any]:
|
|
"""Build an AnyLLMModel whose Chat Completions path yields one completed event."""
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=stream)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
|
|
async def fake_handle_stream(response, chunk_stream, model=None):
|
|
async for _chunk in chunk_stream:
|
|
pass
|
|
yield _completed_event()
|
|
|
|
monkeypatch.setattr(module.ChatCmplStreamHandler, "handle_stream", fake_handle_stream)
|
|
return module, module.AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
|
|
|
|
def _chat_model_with_stream(monkeypatch, stream: _ClosableStream) -> Any:
|
|
_module, model = _chat_module_and_model_with_stream(monkeypatch, stream)
|
|
return model
|
|
|
|
|
|
def _responses_module_and_model_with_stream(
|
|
monkeypatch, stream: _ClosableStream
|
|
) -> tuple[Any, Any]:
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=stream)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
return module, module.AnyLLMModel(model="openai/gpt-5.4-mini")
|
|
|
|
|
|
def _responses_model_with_stream(monkeypatch, stream: _ClosableStream) -> Any:
|
|
_module, model = _responses_module_and_model_with_stream(monkeypatch, stream)
|
|
return model
|
|
|
|
|
|
def _capture_spans(monkeypatch, module: Any, factory_name: str) -> list[Any]:
|
|
"""Record the live span objects the module creates so tests can read them mid-stream."""
|
|
original = getattr(module, factory_name)
|
|
captured: list[Any] = []
|
|
|
|
def recording_factory(*args: Any, **kwargs: Any) -> Any:
|
|
span = original(*args, **kwargs)
|
|
captured.append(span)
|
|
return span
|
|
|
|
monkeypatch.setattr(module, factory_name, recording_factory)
|
|
return captured
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_stream_ignores_close_failure_after_terminal_event(monkeypatch) -> None:
|
|
"""A completed Chat Completions response stays successful when provider cleanup fails."""
|
|
stream = _FailingCloseStream([_chat_chunk("Hello")])
|
|
model = _chat_model_with_stream(monkeypatch, stream)
|
|
|
|
events = [event async for event in _stream_events(model)]
|
|
|
|
assert [event.type for event in events] == ["response.completed"]
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_stream_ignores_close_failure_when_closed_at_terminal_event(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""Terminal state must be recorded before the completed event is yielded."""
|
|
stream = _FailingCloseStream([_chat_chunk("Hello")])
|
|
model = _chat_model_with_stream(monkeypatch, stream)
|
|
stream_agen = cast(Any, _stream_events(model))
|
|
|
|
async for event in stream_agen:
|
|
if event.type == "response.completed":
|
|
break
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_stream_closes_provider_stream_after_cancellation(monkeypatch) -> None:
|
|
"""Cancelling the consumer must still release the provider stream."""
|
|
blocked = asyncio.Event()
|
|
stream = _BlockingStream([_chat_chunk("He")], blocked)
|
|
model = _chat_model_with_stream(monkeypatch, stream)
|
|
stream_agen = cast(Any, _stream_events(model))
|
|
|
|
async def consume() -> None:
|
|
async for _event in stream_agen:
|
|
pass
|
|
|
|
task = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(blocked.wait(), timeout=5)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
finally:
|
|
task.cancel()
|
|
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_stream_does_not_block_cancellation_on_slow_close(monkeypatch) -> None:
|
|
"""A provider close that waits on transport I/O must not delay cancellation."""
|
|
blocked = asyncio.Event()
|
|
release = asyncio.Event()
|
|
stream = _SlowCloseStream([_chat_chunk("He")], blocked, release)
|
|
model = _chat_model_with_stream(monkeypatch, stream)
|
|
stream_agen = cast(Any, _stream_events(model))
|
|
|
|
async def consume() -> None:
|
|
async for _event in stream_agen:
|
|
pass
|
|
|
|
task = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(blocked.wait(), timeout=5)
|
|
task.cancel()
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await asyncio.wait_for(task, timeout=5)
|
|
assert stream.aclose_calls == 1
|
|
assert stream.aclose_completed == 0
|
|
|
|
release.set()
|
|
for _ in range(200):
|
|
if stream.aclose_completed == 1:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert stream.aclose_completed == 1
|
|
finally:
|
|
release.set()
|
|
task.cancel()
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_stream_propagates_close_failure_before_terminal_event(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""Cleanup failures before completion stay observable by the caller."""
|
|
stream = _FailingCloseStream([_chat_chunk("Hello")])
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=stream)
|
|
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
|
|
|
async def fake_handle_stream(response, chunk_stream, model=None):
|
|
yield ResponseCreatedEvent(
|
|
type="response.created",
|
|
response=_response("partial"),
|
|
sequence_number=0,
|
|
)
|
|
async for _chunk in chunk_stream:
|
|
pass
|
|
yield _completed_event()
|
|
|
|
monkeypatch.setattr(module.ChatCmplStreamHandler, "handle_stream", fake_handle_stream)
|
|
model = module.AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
stream_agen = cast(Any, _stream_events(model))
|
|
|
|
first_event = await anext(stream_agen)
|
|
assert first_event.type == "response.created"
|
|
|
|
with pytest.raises(RuntimeError, match="close-failure"):
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_ignores_close_failure_after_terminal_event(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""A completed Responses stream stays successful when provider cleanup fails."""
|
|
stream = _FailingCloseStream([_completed_event()])
|
|
model = _responses_model_with_stream(monkeypatch, stream)
|
|
|
|
events = [event async for event in _stream_events(model)]
|
|
|
|
assert [event.type for event in events] == ["response.completed"]
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_ignores_close_failure_after_terminal_failure(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""A cleanup failure must not replace the terminal failure the caller should see."""
|
|
stream = _FailingCloseStream(
|
|
[
|
|
ResponseFailedEvent(
|
|
type="response.failed",
|
|
response=_response("partial", response_id="resp-terminal"),
|
|
sequence_number=1,
|
|
)
|
|
]
|
|
)
|
|
model = _responses_model_with_stream(monkeypatch, stream)
|
|
|
|
with pytest.raises(ModelBehaviorError, match="response.failed"):
|
|
async for _event in _stream_events(model):
|
|
pass
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_propagates_close_failure_before_terminal_event(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""Cleanup failures before completion stay observable by the caller."""
|
|
stream = _FailingCloseStream(
|
|
[
|
|
ResponseCreatedEvent(
|
|
type="response.created",
|
|
response=_response("partial"),
|
|
sequence_number=0,
|
|
),
|
|
_completed_event(),
|
|
]
|
|
)
|
|
model = _responses_model_with_stream(monkeypatch, stream)
|
|
stream_agen = cast(Any, _stream_events(model))
|
|
|
|
first_event = await anext(stream_agen)
|
|
assert first_event.type == "response.created"
|
|
|
|
with pytest.raises(RuntimeError, match="close-failure"):
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_closes_provider_stream_after_cancellation(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""Cancelling the consumer must still release the provider Responses stream."""
|
|
blocked = asyncio.Event()
|
|
stream = _BlockingStream(
|
|
[
|
|
ResponseCreatedEvent(
|
|
type="response.created",
|
|
response=_response("partial"),
|
|
sequence_number=0,
|
|
)
|
|
],
|
|
blocked,
|
|
)
|
|
model = _responses_model_with_stream(monkeypatch, stream)
|
|
stream_agen = cast(Any, _stream_events(model))
|
|
|
|
async def consume() -> None:
|
|
async for _event in stream_agen:
|
|
pass
|
|
|
|
task = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(blocked.wait(), timeout=5)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
finally:
|
|
task.cancel()
|
|
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
class _CloseSignalingStream(_ClosableStream):
|
|
"""Signals when `aclose` starts and blocks until released, so a test can cancel mid-close."""
|
|
|
|
def __init__(
|
|
self,
|
|
chunks: list[Any],
|
|
close_started: asyncio.Event,
|
|
release: asyncio.Event,
|
|
) -> None:
|
|
super().__init__(chunks)
|
|
self._close_started = close_started
|
|
self._release = release
|
|
self.aclose_completed = 0
|
|
|
|
async def aclose(self) -> None:
|
|
self.aclose_calls += 1
|
|
self._close_started.set()
|
|
await self._release.wait()
|
|
self.aclose_completed += 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("with_usage", [False, True], ids=["no-usage", "detailed-usage"])
|
|
@pytest.mark.parametrize("tracing", [ModelTracing.ENABLED, ModelTracing.ENABLED_WITHOUT_DATA])
|
|
async def test_any_llm_chat_stream_preserves_trace_data_at_completed(
|
|
monkeypatch, with_usage: bool, tracing: ModelTracing
|
|
) -> None:
|
|
"""Exercise real chat stream conversion while the adapter is suspended at its terminal yield."""
|
|
chunk = _chat_chunk("Hello")
|
|
if with_usage:
|
|
chunk.usage = CompletionUsage(
|
|
completion_tokens=5,
|
|
prompt_tokens=7,
|
|
total_tokens=12,
|
|
prompt_tokens_details=PromptTokensDetails(cached_tokens=2),
|
|
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=3),
|
|
)
|
|
stream = _ClosableStream([chunk])
|
|
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=stream)
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
model = module.AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
|
spans = _capture_spans(monkeypatch, module, "generation_span")
|
|
|
|
with trace(workflow_name="any-llm-chat-terminal-span"):
|
|
stream_agen = _stream_events(model, tracing)
|
|
try:
|
|
async for event in stream_agen:
|
|
if event.type == "response.completed":
|
|
[span] = spans
|
|
assert span.span_data.usage == {
|
|
"requests": 1,
|
|
"input_tokens": 7 if with_usage else 0,
|
|
"output_tokens": 5 if with_usage else 0,
|
|
"total_tokens": 12 if with_usage else 0,
|
|
"input_tokens_details": {
|
|
"cache_write_tokens_details": None,
|
|
"cached_tokens": 2 if with_usage else 0,
|
|
"cache_write_tokens": 0,
|
|
},
|
|
"output_tokens_details": {"reasoning_tokens": 3 if with_usage else 0},
|
|
}
|
|
assert span.span_data.output == (
|
|
[event.response.model_dump()] if tracing == ModelTracing.ENABLED else None
|
|
)
|
|
assert (event.response.usage is not None) == with_usage
|
|
break
|
|
else:
|
|
pytest.fail("The stream did not yield a completed response.")
|
|
finally:
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_stream_populates_span_before_yielding_completed(monkeypatch) -> None:
|
|
"""The generation span must carry output and usage when the completed event is delivered."""
|
|
stream = _ClosableStream([_chat_chunk("Hello")])
|
|
module, model = _chat_module_and_model_with_stream(monkeypatch, stream)
|
|
spans = _capture_spans(monkeypatch, module, "generation_span")
|
|
|
|
with trace(workflow_name="any-llm-chat-span"):
|
|
stream_agen = cast(Any, _stream_events(model, ModelTracing.ENABLED))
|
|
async for event in stream_agen:
|
|
if event.type == "response.completed":
|
|
# Assert while the generator is suspended at the terminal yield.
|
|
[span] = spans
|
|
assert span.span_data.output == [_response("Hello").model_dump()]
|
|
assert span.span_data.usage == {
|
|
"requests": 1,
|
|
"input_tokens": 11,
|
|
"output_tokens": 13,
|
|
"total_tokens": 24,
|
|
"input_tokens_details": {
|
|
"cache_write_tokens_details": None,
|
|
"cached_tokens": 0,
|
|
"cache_write_tokens": 0,
|
|
},
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
}
|
|
break
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_populates_span_before_yielding_completed(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""The response span must carry the response and input when the completed event arrives."""
|
|
stream = _ClosableStream([_completed_event()])
|
|
module, model = _responses_module_and_model_with_stream(monkeypatch, stream)
|
|
spans = _capture_spans(monkeypatch, module, "response_span")
|
|
|
|
with trace(workflow_name="any-llm-responses-span"):
|
|
stream_agen = cast(Any, _stream_events(model, ModelTracing.ENABLED))
|
|
async for event in stream_agen:
|
|
if event.type == "response.completed":
|
|
[span] = spans
|
|
assert span.span_data.response is not None
|
|
assert span.span_data.response.id == "resp_123"
|
|
assert span.span_data.input == "hi"
|
|
break
|
|
await stream_agen.aclose()
|
|
|
|
assert stream.aclose_calls == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_chat_stream_lets_in_flight_close_finish_after_cancellation(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""Cancelling during `aclose` continues that close instead of starting a second one."""
|
|
close_started = asyncio.Event()
|
|
release = asyncio.Event()
|
|
stream = _CloseSignalingStream([_chat_chunk("Hello")], close_started, release)
|
|
model = _chat_model_with_stream(monkeypatch, stream)
|
|
|
|
async def consume() -> None:
|
|
async for _event in _stream_events(model):
|
|
pass
|
|
|
|
task = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(close_started.wait(), timeout=5)
|
|
assert stream.aclose_calls == 1
|
|
assert stream.aclose_completed == 0
|
|
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await asyncio.wait_for(task, timeout=5)
|
|
|
|
# The cancelled consumer must not have started a second close.
|
|
assert stream.aclose_calls == 1
|
|
assert stream.aclose_completed == 0
|
|
|
|
release.set()
|
|
for _ in range(200):
|
|
if stream.aclose_completed == 1:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
|
|
assert stream.aclose_calls == 1
|
|
assert stream.aclose_completed == 1
|
|
finally:
|
|
release.set()
|
|
task.cancel()
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_lets_in_flight_close_finish_after_cancellation(
|
|
monkeypatch,
|
|
) -> None:
|
|
"""Cancelling during `aclose` continues that close instead of starting a second one."""
|
|
close_started = asyncio.Event()
|
|
release = asyncio.Event()
|
|
stream = _CloseSignalingStream([_completed_event()], close_started, release)
|
|
model = _responses_model_with_stream(monkeypatch, stream)
|
|
|
|
async def consume() -> None:
|
|
async for _event in _stream_events(model):
|
|
pass
|
|
|
|
task = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(close_started.wait(), timeout=5)
|
|
assert stream.aclose_calls == 1
|
|
assert stream.aclose_completed == 0
|
|
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await asyncio.wait_for(task, timeout=5)
|
|
|
|
assert stream.aclose_calls == 1
|
|
assert stream.aclose_completed == 0
|
|
|
|
release.set()
|
|
for _ in range(200):
|
|
if stream.aclose_completed == 1:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
|
|
assert stream.aclose_calls == 1
|
|
assert stream.aclose_completed == 1
|
|
finally:
|
|
release.set()
|
|
task.cancel()
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_counts_request_when_usage_is_absent(monkeypatch) -> None:
|
|
"""The AnyLLM Responses stream must count its request like the non-streaming path.
|
|
|
|
`get_response` already reports one request when the provider omits usage. The streaming
|
|
path went through the run loop's usage-less fallback and reported zero, so the same call
|
|
was counted differently depending only on whether it was streamed.
|
|
"""
|
|
completed = _response("Hello")
|
|
completed.usage = None
|
|
|
|
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
|
|
yield ResponseCompletedEvent(
|
|
type="response.completed", response=completed, sequence_number=1
|
|
)
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
|
|
events = [
|
|
event
|
|
async for event in module.AnyLLMModel(model="openai/gpt-5.4-mini").stream_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
]
|
|
|
|
from agents.usage import _requests_for_response_without_usage
|
|
|
|
terminal = events[-1]
|
|
assert isinstance(terminal, ResponseCompletedEvent)
|
|
# No usage payload is synthesized, so token counts are not reported as real zeros.
|
|
assert terminal.response.usage is None
|
|
assert _requests_for_response_without_usage(terminal.response) == 1
|
|
|
|
|
|
@pytest.mark.allow_call_model_methods
|
|
@pytest.mark.asyncio
|
|
async def test_any_llm_responses_stream_with_usage_is_not_marked(monkeypatch) -> None:
|
|
"""A response that did report usage must not also be counted by the usage-less path."""
|
|
|
|
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
|
|
yield ResponseCompletedEvent(
|
|
type="response.completed", response=_response("Hello"), sequence_number=1
|
|
)
|
|
|
|
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
|
module, _ = _import_any_llm_module(monkeypatch, provider)
|
|
|
|
events = [
|
|
event
|
|
async for event in module.AnyLLMModel(model="openai/gpt-5.4-mini").stream_response(
|
|
system_instructions=None,
|
|
input="hi",
|
|
model_settings=ModelSettings(),
|
|
tools=[],
|
|
output_schema=None,
|
|
handoffs=[],
|
|
tracing=ModelTracing.DISABLED,
|
|
previous_response_id=None,
|
|
conversation_id=None,
|
|
prompt=None,
|
|
)
|
|
]
|
|
|
|
from agents.usage import _requests_for_response_without_usage
|
|
|
|
terminal = events[-1]
|
|
assert isinstance(terminal, ResponseCompletedEvent)
|
|
assert terminal.response.usage is not None
|
|
assert _requests_for_response_without_usage(terminal.response) == 0
|