openai-agents-python/tests/models/test_openai_responses.py
Kazuhiro Sera d5287b741d
fix: align model defaults with TS SDK (#6064)
* Align GPT-5 and newer model defaults with TypeScript

* Preserve explicit request settings when applying model defaults
2026-11-17 21:40:15 +00:00

6057 lines
194 KiB
Python

from __future__ import annotations
import asyncio
import copy
import json
import logging
from collections.abc import AsyncIterator
from contextlib import nullcontext
from dataclasses import asdict, fields, replace
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import AsyncMock
import httpx2
import pytest
from openai import NOT_GIVEN, APIConnectionError, AsyncOpenAI, RateLimitError, omit
from openai.types.responses import Response, ResponseCompletedEvent, ResponseErrorEvent
from openai.types.responses.response import IncompleteDetails
from openai.types.responses.response_create_params import ContextManagement, PromptCacheOptions
from openai.types.responses.response_usage import ResponseUsage
from openai.types.responses.tool_param import ImageGeneration
from openai.types.shared.reasoning import Reasoning
from agents import (
Agent,
ApplyPatchTool,
AsyncComputer,
Computer,
ComputerTool,
ImageGenerationTool,
ModelSettings,
ModelTracing,
ProgrammaticToolCallingTool,
RunConfig,
Runner,
ShellTool,
Tool,
ToolSearchTool,
WebSearchTool,
__version__,
function_tool,
handoff,
tool_namespace,
trace,
)
from agents.exceptions import ModelBehaviorError, UserError
from agents.models._retry_runtime import (
provider_managed_retries_disabled,
websocket_pre_event_retries_disabled,
)
from agents.models.openai_provider import OpenAIProvider
from agents.models.openai_responses import (
_HEADERS_OVERRIDE as RESP_HEADERS,
ConvertedTools,
Converter,
OpenAIResponsesModel,
OpenAIResponsesWSModel,
ResponsesWebSocketError,
_ResponseStreamWithRequestId,
_should_retry_pre_event_websocket_disconnect,
)
from agents.retry import ModelRetryAdviceRequest
from agents.usage import Usage
from tests.model_test_helpers import get_response_obj
from tests.test_responses import get_text_message
from tests.testing_processor import fetch_ordered_spans
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
@pytest.mark.parametrize(
("counts", "expected"),
[
((None, 5, 12), (0, 5, 12)),
((7, None, 12), (7, 0, 12)),
((7, 5, None), (7, 5, 0)),
((None, None, None), (0, 0, 0)),
((0, 0, 0), (0, 0, 0)),
((7, 5, 12), (7, 5, 12)),
],
ids=["null-input", "null-output", "null-total", "all-null", "zero", "valid"],
)
async def test_runner_normalizes_nullable_responses_usage(
stream: bool,
counts: tuple[int | None, int | None, int | None],
expected: tuple[int, int, int],
) -> None:
usage = {
"input_tokens": counts[0],
"output_tokens": counts[1],
"total_tokens": counts[2],
"input_tokens_details": {"cached_tokens": 3},
"output_tokens_details": {"reasoning_tokens": 2},
}
# Exercise provider parsing and usage conversion instead of a normalized scripted model.
async def handler(request: httpx2.Request) -> httpx2.Response:
payload = get_response_obj([get_text_message("Hello")]).model_dump()
payload["usage"] = usage
if stream:
event = {"type": "response.completed", "response": payload, "sequence_number": 0}
return httpx2.Response(
200,
content=f"event: response.completed\ndata: {json.dumps(event)}\n\n",
headers={"content-type": "text/event-stream"},
)
return httpx2.Response(200, json=payload)
async with AsyncOpenAI(
api_key="test-key",
base_url="https://provider.example/v1",
http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)),
) as client:
agent = Agent(
name="test",
model=OpenAIResponsesModel(model="fake", openai_client=client),
model_settings=ModelSettings(preserve_raw_usage=True),
)
if stream:
result = Runner.run_streamed(agent, "hi")
async for _ in result.stream_events():
pass
else:
result = await Runner.run(agent, "hi")
assert result.final_output == "Hello"
normalized = result.context_wrapper.usage
assert normalized.requests == 1
assert (normalized.input_tokens, normalized.output_tokens, normalized.total_tokens) == expected
assert normalized.input_tokens_details.cached_tokens == 3
assert normalized.output_tokens_details.reasoning_tokens == 2
assert result.raw_responses[0].raw_usage == usage
if expected == (0, 0, 0):
assert normalized.request_usage_entries == []
else:
assert len(normalized.request_usage_entries) == 1
entry = normalized.request_usage_entries[0]
assert (entry.input_tokens, entry.output_tokens, entry.total_tokens) == expected
assert entry.input_tokens_details.cached_tokens == 3
assert entry.output_tokens_details.reasoning_tokens == 2
async def _run_responses_model_with_custom_base_url(
model_settings: ModelSettings | dict[str, Any] | None = None,
) -> dict[str, Any]:
class DummyResponses:
def __init__(self) -> None:
self.kwargs: dict[str, Any] = {}
async def create(self, **kwargs: Any) -> Any:
self.kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self, responses: DummyResponses) -> None:
self.responses = responses
self.base_url = httpx2.URL("https://custom.example.test/v1/")
responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=DummyResponsesClient(responses), # type: ignore[arg-type]
)
agent = Agent(name="test", model=model, model_settings=model_settings or ModelSettings())
await Runner.run(agent, "hi")
return responses.kwargs
async def _run_responses_model_with_official_client(
model_settings: ModelSettings | None = None,
) -> list[httpx2.Request]:
requests: list[httpx2.Request] = []
async def handler(request: httpx2.Request) -> httpx2.Response:
requests.append(request)
return httpx2.Response(
200,
content=get_response_obj([]).model_dump_json(),
headers={"content-type": "application/json"},
request=request,
)
http_client = httpx2.AsyncClient(transport=httpx2.MockTransport(handler))
try:
client = AsyncOpenAI(
api_key="test-key",
base_url="https://example.test/v1",
http_client=http_client,
)
model = OpenAIResponsesModel(model="gpt-4", openai_client=client)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=model_settings or ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
finally:
await http_client.aclose()
return requests
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("call_id_fields", [{}, {"call_id": None}], ids=["omitted", "null"])
@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"])
async def test_unpaired_function_output_preserved_in_responses_request(
call_id_fields: dict[str, None], stream: bool
) -> None:
"""Responses keeps external-context output and its source without inventing a call ID."""
request_bodies: list[dict[str, Any]] = []
async def handler(request: httpx2.Request) -> httpx2.Response:
request_bodies.append(json.loads(request.content))
if stream:
event = _response_completed_frame("resp-id", sequence_number=0)
return httpx2.Response(
200,
content=f"event: response.completed\ndata: {event}\n\n",
headers={"content-type": "text/event-stream"},
)
return httpx2.Response(
200,
content=get_response_obj([]).model_dump_json(),
headers={"content-type": "application/json"},
)
expected_input = {
"type": "function_call_output",
"name": "notifications",
"namespace": "slack",
"output": [
{"type": "input_text", "text": "Alice mentioned you in #deployments."},
{"type": "input_image", "image_url": "https://example.com/image.png"},
],
**call_id_fields,
}
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as http_client:
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=AsyncOpenAI(api_key="test-key", http_client=http_client),
)
request_kwargs: dict[str, Any] = {
"system_instructions": None,
"input": [dict(expected_input)],
"model_settings": ModelSettings(),
"tools": [],
"output_schema": None,
"handoffs": [],
"tracing": ModelTracing.DISABLED,
}
if stream:
async for _ in model.stream_response(**request_kwargs):
pass
else:
await model.get_response(**request_kwargs)
assert [body["input"] for body in request_bodies] == [[expected_input]]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"])
@pytest.mark.parametrize(
"tool_options, expected_include",
[
({}, []),
({"search_content_types": ["text"]}, []),
(
{
"search_content_types": ["image", "text"],
"image_settings": {"max_results": 3, "caption": False},
},
["web_search_call.results"],
),
],
ids=["default", "text_only", "image_and_text"],
)
async def test_web_search_image_options_reach_responses_request(
stream: bool, tool_options: dict[str, Any], expected_include: list[str]
) -> None:
"""Inspect the provider wire payload, including fields not yet typed by openai-python."""
request_bodies: list[dict[str, Any]] = []
async def handler(request: httpx2.Request) -> httpx2.Response:
request_bodies.append(json.loads(request.content))
if stream:
event = _response_completed_frame("resp-id", sequence_number=0)
return httpx2.Response(
200,
content=f"event: response.completed\ndata: {event}\n\n",
headers={"content-type": "text/event-stream"},
)
return httpx2.Response(
200,
content=get_response_obj([]).model_dump_json(),
headers={"content-type": "application/json"},
)
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as http_client:
model = OpenAIResponsesModel(
model="gpt-5.6",
openai_client=AsyncOpenAI(api_key="test-key", http_client=http_client),
)
request_kwargs: dict[str, Any] = {
"system_instructions": None,
"input": "Find images of the Golden Gate Bridge.",
"model_settings": ModelSettings(),
"tools": [WebSearchTool(**tool_options)],
"output_schema": None,
"handoffs": [],
"tracing": ModelTracing.DISABLED,
}
if stream:
async for _ in model.stream_response(**request_kwargs):
pass
else:
await model.get_response(**request_kwargs)
assert len(request_bodies) == 1
body = request_bodies[0]
assert body["tools"] == [
{
"type": "web_search",
"filters": None,
"user_location": None,
"search_context_size": "medium",
**tool_options,
}
]
assert body.get("include", []) == expected_include
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"])
@pytest.mark.parametrize(
"tool",
[
ImageGenerationTool(ImageGeneration(type="image_generation")),
ImageGenerationTool(
ImageGeneration(
type="image_generation",
model="gpt-image-1",
quality="high",
action="edit",
input_fidelity="high",
input_image_mask={"file_id": "file-mask"},
)
),
ImageGenerationTool(
tool_config={
"type": "image_generation",
"model": "gpt-image-2.5-sunburst",
"quality": "max",
"action": "generate",
"size": "1536x864",
"background": "transparent",
"output_format": "webp",
"output_compression": 0,
"partial_images": 0,
"moderation": "auto",
}
),
ImageGenerationTool(
tool_config={
"type": "image_generation",
"model": "gpt-image-2.5-flare",
"quality": "xhigh",
"action": "auto",
"size": "auto",
"output_format": "png",
"partial_images": 2,
}
),
],
ids=["defaults", "typed_edit", "sunburst_max", "flare_xhigh"],
)
async def test_image_generation_options_reach_responses_request(
stream: bool, tool: ImageGenerationTool
) -> None:
"""Use the real client's serializer to check options beyond its current annotations."""
expected_config = copy.deepcopy(tool.tool_config)
request_bodies: list[dict[str, Any]] = []
async def handler(request: httpx2.Request) -> httpx2.Response:
request_bodies.append(json.loads(request.content))
if stream:
event = _response_completed_frame("resp-id", sequence_number=0)
return httpx2.Response(
200,
content=f"event: response.completed\ndata: {event}\n\n",
headers={"content-type": "text/event-stream"},
)
return httpx2.Response(
200,
content=get_response_obj([]).model_dump_json(),
headers={"content-type": "application/json"},
)
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as http_client:
agent = Agent(
name="Image generator",
model=OpenAIResponsesModel(
model="gpt-5.5",
openai_client=AsyncOpenAI(api_key="test-key", http_client=http_client),
),
tools=[tool],
)
prompt = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "Generate an image, or edit this image."},
{"type": "input_image", "file_id": "file-image", "detail": "auto"},
],
}
]
if stream:
result = Runner.run_streamed(agent, prompt)
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, prompt)
assert len(request_bodies) == 1
assert request_bodies[0]["tools"] == [expected_config]
assert tool.tool_config == expected_config
assert request_bodies[0]["model"] == "gpt-5.5"
def test_image_generation_config_remains_mutable() -> None:
config: ImageGeneration = {"type": "image_generation", "quality": "low"}
tool = ImageGenerationTool(config)
assert tool.tool_config is config
assert [field.name for field in fields(tool)] == ["tool_config"]
assert asdict(tool) == {"tool_config": config}
assert replace(tool).tool_config is config
tool.tool_config["quality"] = "high"
assert config["quality"] == "high"
replacement: ImageGeneration = {"type": "image_generation", "quality": "auto"}
assert replace(tool, tool_config=replacement).tool_config is replacement
tool.tool_config = replacement
assert tool.tool_config is replacement
tool.tool_config = {
"type": "image_generation",
"model": "gpt-image-2.5-flare",
"quality": "xhigh",
}
tool.tool_config["quality"] = "max"
assert Converter.convert_tools([tool], []).tools == [
{
"type": "image_generation",
"model": "gpt-image-2.5-flare",
"quality": "max",
}
]
class DummyWSConnection:
def __init__(self, frames: list[str]):
self._frames = frames
self.sent_messages: list[dict[str, Any]] = []
self.close_calls = 0
self.close_code: int | None = None
async def send(self, payload: str) -> None:
self.sent_messages.append(json.loads(payload))
async def recv(self) -> str:
if not self._frames:
raise RuntimeError("No more websocket frames configured")
return self._frames.pop(0)
async def close(self) -> None:
self.close_calls += 1
if self.close_code is None:
self.close_code = 1000
class DummyWSClient:
def __init__(self):
self.base_url = httpx2.URL("https://api.openai.com/v1/")
self.websocket_base_url = None
self.default_query: dict[str, Any] = {}
self.auth_headers = {"Authorization": "Bearer test-key"}
self.default_headers = {"User-Agent": "AsyncOpenAI/Python test"}
self.timeout: Any = None
self.refresh_calls = 0
async def _refresh_api_key(self) -> None:
self.refresh_calls += 1
def _response_event_frame(event_type: str, response_id: str, sequence_number: int) -> str:
response = get_response_obj([]).model_dump()
response["id"] = response_id
return json.dumps(
{
"type": event_type,
"response": response,
"sequence_number": sequence_number,
}
)
def _response_completed_frame(response_id: str, sequence_number: int) -> str:
return _response_event_frame("response.completed", response_id, sequence_number)
def _response_error_frame(code: str, message: str, sequence_number: int) -> str:
return json.dumps(
{
"type": "response.error",
"error": {"code": code, "message": message, "param": None},
"sequence_number": sequence_number,
}
)
def _connection_closed_error(message: str) -> Exception:
class ConnectionClosedError(Exception):
pass
ConnectionClosedError.__module__ = "websockets.client"
return ConnectionClosedError(message)
def _invalid_message_error(message: str, *, cause: BaseException | None = None) -> Exception:
class InvalidMessage(Exception):
pass
InvalidMessage.__module__ = "websockets.exceptions"
error = InvalidMessage(message)
if cause is not None:
error.__cause__ = cause
return error
@pytest.mark.parametrize("parallel_tool_calls", [True, False, None])
@pytest.mark.parametrize("tool_source", ["none", "function", "handoff"])
def test_parallel_tool_calls_follow_converted_responses_tools(
parallel_tool_calls: bool | None,
tool_source: str,
) -> None:
tools: list[Tool] = (
[function_tool(lambda: "ok", name_override="test_tool")]
if tool_source == "function"
else []
)
handoffs = [handoff(Agent(name="handoff"))] if tool_source == "handoff" else []
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=cast(Any, object()),
)
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(parallel_tool_calls=parallel_tool_calls),
tools=tools,
output_schema=None,
handoffs=handoffs,
)
expected_parallel_tool_calls = (
parallel_tool_calls if tool_source != "none" and parallel_tool_calls is not None else omit
)
assert kwargs["parallel_tool_calls"] is expected_parallel_tool_calls
assert bool(kwargs["tools"]) is (tool_source != "none")
@pytest.mark.parametrize("parallel_tool_calls", [True, False, None])
def test_parallel_tool_calls_preserve_stored_prompt_overrides(
parallel_tool_calls: bool | None,
) -> None:
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=cast(Any, object()),
)
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(parallel_tool_calls=parallel_tool_calls),
tools=[],
output_schema=None,
handoffs=[],
prompt={"id": "pmpt_123"},
)
expected_parallel_tool_calls = parallel_tool_calls if parallel_tool_calls is not None else omit
assert kwargs["parallel_tool_calls"] is expected_parallel_tool_calls
assert kwargs["tools"] is omit
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
async def test_user_agent_header_responses(override_ua: str | None):
called_kwargs: dict[str, Any] = {}
expected_ua = override_ua or f"Agents/Python {__version__}"
class DummyStream:
def __aiter__(self):
async def gen():
yield ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([]),
sequence_number=0,
)
return gen()
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return DummyStream()
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore
if override_ua is not None:
token = RESP_HEADERS.set({"User-Agent": override_ua})
else:
token = None
try:
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
async for _ in stream:
pass
finally:
if token is not None:
RESP_HEADERS.reset(token)
assert "extra_headers" in called_kwargs
assert called_kwargs["extra_headers"]["User-Agent"] == expected_ua
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_get_response_exposes_request_id():
class DummyResponses:
async def create(self, **kwargs):
response = get_response_obj([], response_id="resp-request-id")
response._request_id = "req_nonstream_123"
return response
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore[arg-type]
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(preserve_raw_usage=True),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert response.response_id == "resp-request-id"
assert response.request_id == "req_nonstream_123"
assert response.raw_usage is not None
assert response.raw_usage["input_tokens_details"]["cached_tokens"] == 0
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_get_response_span_exports_usage():
class FalsyResponseUsage(ResponseUsage):
def __bool__(self) -> bool:
return False
class DummyResponses:
async def create(self, **kwargs):
response = get_response_obj(
[],
response_id="resp-usage",
usage=Usage(requests=1, input_tokens=10, output_tokens=4, total_tokens=14),
)
assert response.usage is not None
response.usage = FalsyResponseUsage.model_validate(response.usage.model_dump())
return response
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore[arg-type]
with trace("test"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.ENABLED,
)
response_spans = [
span.export() for span in fetch_ordered_spans() if span.span_data.type == "response"
]
assert len(response_spans) == 1
assert response_spans[0]
assert response_spans[0]["span_data"] == {
"type": "response",
"response_id": "resp-usage",
"usage": {
"requests": 1,
"input_tokens": 10,
"output_tokens": 4,
"total_tokens": 14,
"input_tokens_details": {
"cache_write_tokens_details": None,
"cached_tokens": 0,
"cache_write_tokens": 0,
},
"output_tokens_details": {"reasoning_tokens": 0},
},
}
def test_get_client_disables_provider_managed_retries_on_runner_retry() -> None:
class DummyResponsesClient:
def __init__(self) -> None:
self.responses = SimpleNamespace()
self.with_options_calls: list[dict[str, Any]] = []
def with_options(self, **kwargs):
self.with_options_calls.append(kwargs)
return self
client = DummyResponsesClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
assert cast(object, model._get_client()) is client
with provider_managed_retries_disabled(True):
assert cast(object, model._get_client()) is client
assert client.with_options_calls == [{"max_retries": 0}]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_fetch_response_stream_attaches_request_id_to_terminal_response():
class DummyHTTPStream:
def __init__(self):
self._yielded = False
def __aiter__(self):
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
return ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([], response_id="resp-stream-request-id"),
sequence_number=0,
)
inner_stream = DummyHTTPStream()
class DummyAPIResponse:
def __init__(self):
self.request_id = "req_stream_123"
self.close_calls = 0
self.parse_calls = 0
async def parse(self):
self.parse_calls += 1
return inner_stream
async def close(self) -> None:
self.close_calls += 1
api_response = DummyAPIResponse()
aexit_calls: list[tuple[Any, Any, Any]] = []
class DummyStreamingContextManager:
async def __aenter__(self):
return api_response
async def __aexit__(self, exc_type, exc, tb):
aexit_calls.append((exc_type, exc, tb))
await api_response.close()
return False
class DummyResponses:
def __init__(self):
self.with_streaming_response = SimpleNamespace(create=self.create_streaming)
def create_streaming(self, **kwargs):
return DummyStreamingContextManager()
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore[arg-type]
stream = await model._fetch_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=True,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert getattr(stream, "request_id", None) == "req_stream_123"
assert getattr(event.response, "_request_id", None) == "req_stream_123"
with pytest.raises(StopAsyncIteration):
await stream_agen.__anext__()
assert api_response.parse_calls == 1
assert api_response.close_calls == 1
assert aexit_calls == [(None, None, None)]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_fetch_response_stream_parse_failure_exits_streaming_context():
parse_error = RuntimeError("parse failed")
aexit_calls: list[tuple[Any, Any, Any]] = []
class DummyAPIResponse:
request_id = "req_stream_123"
async def parse(self):
raise parse_error
api_response = DummyAPIResponse()
class DummyStreamingContextManager:
async def __aenter__(self):
return api_response
async def __aexit__(self, exc_type, exc, tb):
aexit_calls.append((exc_type, exc, tb))
return False
class DummyResponses:
def __init__(self):
self.with_streaming_response = SimpleNamespace(create=self.create_streaming)
def create_streaming(self, **kwargs):
return DummyStreamingContextManager()
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore[arg-type]
with pytest.raises(RuntimeError, match="parse failed"):
await model._fetch_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=True,
)
assert len(aexit_calls) == 1
exc_type, exc, tb = aexit_calls[0]
assert exc_type is RuntimeError
assert exc is parse_error
assert tb is not None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_fetch_response_stream_without_request_id_still_returns_events():
class DummyHTTPStream:
def __init__(self):
self._yielded = False
def __aiter__(self):
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
return ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([], response_id="resp-stream-request-id"),
sequence_number=0,
)
inner_stream = DummyHTTPStream()
aexit_calls: list[tuple[Any, Any, Any]] = []
class DummyAPIResponse:
def __init__(self):
self.close_calls = 0
self.parse_calls = 0
async def parse(self):
self.parse_calls += 1
return inner_stream
async def close(self) -> None:
self.close_calls += 1
api_response = DummyAPIResponse()
class DummyStreamingContextManager:
async def __aenter__(self):
return api_response
async def __aexit__(self, exc_type, exc, tb):
aexit_calls.append((exc_type, exc, tb))
await api_response.close()
return False
class DummyResponses:
def __init__(self):
self.with_streaming_response = SimpleNamespace(create=self.create_streaming)
def create_streaming(self, **kwargs):
return DummyStreamingContextManager()
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore[arg-type]
stream = await model._fetch_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=True,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert getattr(stream, "request_id", None) is None
assert getattr(event.response, "_request_id", None) is None
with pytest.raises(StopAsyncIteration):
await stream_agen.__anext__()
assert api_response.parse_calls == 1
assert api_response.close_calls == 1
assert aexit_calls == [(None, None, None)]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("event_type", ["response.completed", "error"])
async def test_stream_response_ignores_streaming_context_exit_failure_after_terminal_event(
event_type,
):
class DummyHTTPStream:
def __init__(self):
self._yielded = False
def __aiter__(self):
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
if event_type == "error":
return ResponseErrorEvent(
type="error",
code="server_error",
message="synthetic provider failure",
param=None,
sequence_number=0,
)
return ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([], response_id="resp-stream-request-id"),
sequence_number=0,
)
inner_stream = DummyHTTPStream()
aexit_calls: list[tuple[Any, Any, Any]] = []
class DummyAPIResponse:
request_id = "req_stream_123"
async def parse(self):
return inner_stream
api_response = DummyAPIResponse()
class DummyStreamingContextManager:
async def __aenter__(self):
return api_response
async def __aexit__(self, exc_type, exc, tb):
aexit_calls.append((exc_type, exc, tb))
raise RuntimeError("stream context exit failed")
class DummyResponses:
def __init__(self):
self.with_streaming_response = SimpleNamespace(create=self.create_streaming)
def create_streaming(self, **kwargs):
return DummyStreamingContextManager()
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore[arg-type]
events = []
expected_error = (
pytest.raises(ModelBehaviorError, match="synthetic provider failure")
if event_type == "error"
else nullcontext()
)
with expected_error:
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
events.append(event)
assert [event.type for event in events] == [event_type]
assert aexit_calls == [(None, None, None)]
@pytest.mark.asyncio
async def test_response_stream_preserves_error_event_when_cleanup_fails() -> None:
event = ResponseErrorEvent(
type="error",
code="server_error",
message="synthetic provider failure",
param=None,
sequence_number=0,
)
cleanup_calls = 0
async def events():
yield event
async def cleanup():
nonlocal cleanup_calls
cleanup_calls += 1
raise RuntimeError("transport close failed")
stream = _ResponseStreamWithRequestId(events(), request_id=None, cleanup=cleanup)
assert [item async for item in stream] == [event]
assert cleanup_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("event_type", ["error", "response.failed", "response.incomplete", None])
async def test_stream_response_preserves_terminal_failure_when_http_close_fails(
event_type: str | None,
) -> None:
# Exercise real SSE parsing and transport teardown, which a model fake would bypass.
close_calls = 0
if event_type == "error":
payload = ResponseErrorEvent(
type="error",
code="server_error",
message="synthetic provider failure",
param=None,
sequence_number=0,
).model_dump()
expected_message = "code=server_error; message=synthetic provider failure"
elif event_type is not None:
status = event_type.removeprefix("response.")
payload = {
"type": event_type,
"sequence_number": 0,
"response": _response_with_terminal_status(status).model_dump(),
}
expected_message = f"status={status}"
else:
payload = None
expected_message = "transport close failed"
class FailingCloseStream(httpx2.AsyncByteStream):
async def __aiter__(self):
if payload is not None:
yield f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
yield b"data: [DONE]\n\n"
async def aclose(self):
nonlocal close_calls
close_calls += 1
raise RuntimeError("transport close failed")
async def handler(request: httpx2.Request) -> httpx2.Response:
return httpx2.Response(
200,
headers={"content-type": "text/event-stream"},
stream=FailingCloseStream(),
request=request,
)
events = []
async with httpx2.AsyncClient(transport=httpx2.MockTransport(handler)) as http_client:
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=AsyncOpenAI(api_key="test-key", http_client=http_client),
)
expected_exception = ModelBehaviorError if event_type is not None else RuntimeError
with pytest.raises(expected_exception, match=expected_message):
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
events.append(event)
assert [event.type for event in events] == ([event_type] if event_type is not None else [])
assert close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_close_closes_inner_http_stream_with_async_close(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class DummyHTTPStream:
def __init__(self):
self._yielded = False
self.close_calls = 0
def __aiter__(self):
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
return ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([]),
sequence_number=0,
)
async def close(self) -> None:
self.close_calls += 1
inner_stream = DummyHTTPStream()
async def fake_fetch_response(*args: Any, **kwargs: Any) -> DummyHTTPStream:
return inner_stream
monkeypatch.setattr(model, "_fetch_response", fake_fetch_response)
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert event.type == "response.completed"
await stream_agen.aclose()
assert inner_stream.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_span_records_io_when_consumer_stops_at_completed(monkeypatch):
"""A consumer that stops at `response.completed` closes the generator.
The SDK's own run loop does this (it wraps the model stream in `aclosing()` and
breaks once it sees the terminal event). Anything recorded only after the yield
loop therefore never runs for such a consumer, so the response and input must be
attached to the span before the terminal event is yielded, mirroring the existing
usage handling and the non-streamed `get_response` path.
"""
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class DummyHTTPStream:
def __init__(self, response):
self._response = response
self._yielded = False
def __aiter__(self):
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
return ResponseCompletedEvent(
type="response.completed",
response=self._response,
sequence_number=0,
)
async def aclose(self) -> None:
return None
response = get_response_obj(
[],
response_id="resp-stream-early-close",
usage=Usage(requests=1, input_tokens=10, output_tokens=4, total_tokens=14),
)
inner_stream = DummyHTTPStream(response)
async def fake_fetch_response(*args: Any, **kwargs: Any) -> DummyHTTPStream:
return inner_stream
monkeypatch.setattr(model, "_fetch_response", fake_fetch_response)
with trace(workflow_name="test"):
stream = model.stream_response(
system_instructions=None,
input="the user prompt",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.ENABLED,
)
stream_agen = cast(Any, stream)
async for event in stream_agen:
if event.type == "response.completed":
break # stop consuming, as a caller watching for the terminal event would
await stream_agen.aclose()
response_spans = [span for span in fetch_ordered_spans() if span.span_data.type == "response"]
assert len(response_spans) == 1
assert response_spans[0].span_data.response is not None
assert response_spans[0].span_data.input is not None
assert response_spans[0].span_data.usage is not None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_normal_exhaustion_closes_inner_http_stream(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class DummyHTTPStream:
def __init__(self):
self._yielded = False
self.close_calls = 0
def __aiter__(self):
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
return ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([]),
sequence_number=0,
)
async def close(self) -> None:
self.close_calls += 1
inner_stream = DummyHTTPStream()
async def fake_fetch_response(*args: Any, **kwargs: Any) -> DummyHTTPStream:
return inner_stream
monkeypatch.setattr(model, "_fetch_response", fake_fetch_response)
events: list[ResponseCompletedEvent] = []
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
assert isinstance(event, ResponseCompletedEvent)
events.append(event)
assert len(events) == 1
assert inner_stream.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_ignores_inner_close_failure_after_terminal_event(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class DummyHTTPStream:
def __init__(self):
self._yielded = False
self.close_calls = 0
def __aiter__(self):
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
return ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([]),
sequence_number=0,
)
async def close(self) -> None:
self.close_calls += 1
raise RuntimeError("stream close failed")
inner_stream = DummyHTTPStream()
async def fake_fetch_response(*args: Any, **kwargs: Any) -> DummyHTTPStream:
return inner_stream
monkeypatch.setattr(model, "_fetch_response", fake_fetch_response)
events: list[ResponseCompletedEvent] = []
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
assert isinstance(event, ResponseCompletedEvent)
events.append(event)
assert len(events) == 1
assert inner_stream.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_cancellation_does_not_block_on_inner_stream_close(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class BlockingHTTPStream:
def __init__(self):
self.next_started = asyncio.Event()
self.close_started = asyncio.Event()
self.close_release = asyncio.Event()
self.close_calls = 0
def __aiter__(self):
return self
async def __anext__(self):
self.next_started.set()
await asyncio.Event().wait()
raise StopAsyncIteration
async def aclose(self) -> None:
self.close_calls += 1
self.close_started.set()
await self.close_release.wait()
inner_stream = BlockingHTTPStream()
async def fake_fetch_response(*args: Any, **kwargs: Any) -> BlockingHTTPStream:
return inner_stream
monkeypatch.setattr(model, "_fetch_response", fake_fetch_response)
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
stream_agen = cast(Any, stream)
next_task = asyncio.create_task(stream_agen.__anext__())
await asyncio.wait_for(inner_stream.next_started.wait(), timeout=1.0)
next_task.cancel()
try:
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(next_task, timeout=0.5)
await asyncio.wait_for(inner_stream.close_started.wait(), timeout=1.0)
assert inner_stream.close_calls == 1
finally:
inner_stream.close_release.set()
await asyncio.sleep(0)
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_rejects_duplicate_extra_args_keys():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
with pytest.raises(TypeError, match="multiple values.*stream"):
model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"stream": False}),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=True,
prompt=None,
)
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_includes_extra_args_prompt_cache_key():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"prompt_cache_key": "cache-key"}),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["prompt_cache_key"] == "cache-key"
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_includes_context_management():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
context_management: list[ContextManagement] = [
{"type": "compaction", "compact_threshold": 200000}
]
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(context_management=context_management),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["context_management"] == context_management
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_includes_gpt_5_6_request_controls():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-5.6-sol", openai_client=client) # type: ignore[arg-type]
reasoning = Reasoning(mode="pro", effort="max", context="all_turns")
prompt_cache_options: PromptCacheOptions = {"mode": "explicit", "ttl": "30m"}
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(
reasoning=reasoning,
prompt_cache_retention="24h",
prompt_cache_options=prompt_cache_options,
),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id="resp-previous",
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["reasoning"] is reasoning
assert kwargs["prompt_cache_retention"] == "24h"
assert kwargs["prompt_cache_options"] == prompt_cache_options
assert kwargs["previous_response_id"] == "resp-previous"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("use_dictionary", [False, True], ids=["model-settings", "dictionary"])
async def test_responses_requests_normalize_dictionary_agent_settings(use_dictionary: bool) -> None:
settings: dict[str, Any] = {
"reasoning": {"effort": "low", "context": "all_turns"},
"context_management": [{"type": "compaction", "compact_threshold": 200000}],
"prompt_cache_options": {"mode": "explicit", "ttl": "30m"},
"prompt_cache_retention": "24h",
"store": False,
"metadata": {"request": "example"},
"temperature": 0.0,
"top_p": 1.0,
"frequency_penalty": 0.0,
"presence_penalty": 0.0,
"max_tokens": 64,
"parallel_tool_calls": False,
"extra_headers": {"x-model-settings-parity": "preserved"},
"extra_query": {"model_settings_parity": "verified"},
"extra_body": {"prompt_cache_key": "extra-body-cache-key"},
"retry": {
"max_retries": 0,
"backoff": {"initial_delay": 0.0, "jitter": False},
},
}
kwargs = await _run_responses_model_with_custom_base_url(
model_settings=settings if use_dictionary else ModelSettings(**settings)
)
assert isinstance(kwargs["reasoning"], Reasoning)
assert kwargs["reasoning"].effort == "low"
assert kwargs["reasoning"].context == "all_turns"
assert kwargs["context_management"] == settings["context_management"]
assert kwargs["prompt_cache_options"] == settings["prompt_cache_options"]
assert kwargs["prompt_cache_retention"] == "24h"
assert kwargs["store"] is False
assert kwargs["metadata"] == {"request": "example"}
assert kwargs["temperature"] == 0.0
assert kwargs["top_p"] == 1.0
assert kwargs["max_output_tokens"] == 64
assert "max_tokens" not in kwargs
assert kwargs["parallel_tool_calls"] is omit
assert kwargs["extra_headers"]["x-model-settings-parity"] == "preserved"
assert kwargs["extra_query"] == {"model_settings_parity": "verified"}
assert kwargs["extra_body"] == {"prompt_cache_key": "extra-body-cache-key"}
assert "retry" not in kwargs
assert "frequency_penalty" not in kwargs
assert "presence_penalty" not in kwargs
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_rejects_duplicate_prompt_cache_options_extra_args():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-5.6-sol", openai_client=client) # type: ignore[arg-type]
with pytest.raises(TypeError, match="multiple values.*prompt_cache_options"):
model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(
prompt_cache_options={"mode": "explicit", "ttl": "30m"},
extra_args={"prompt_cache_options": {"mode": "implicit"}},
),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_allows_prompt_cache_options_extra_args_when_direct_omitted():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-5.6-sol", openai_client=client) # type: ignore[arg-type]
prompt_cache_options = {"mode": "explicit", "ttl": "30m"}
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"prompt_cache_options": prompt_cache_options}),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["prompt_cache_options"] == prompt_cache_options
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_allows_extra_arg_when_explicit_arg_is_omitted():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
context_management: list[ContextManagement] = [
{"type": "compaction", "compact_threshold": 200000}
]
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"context_management": context_management}),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["context_management"] == context_management
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_rejects_duplicate_context_management_extra_args():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
with pytest.raises(TypeError, match="multiple values.*context_management"):
model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(
context_management=[{"type": "compaction", "compact_threshold": 200000}],
extra_args={"context_management": [{"type": "compaction"}]},
),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_keeps_unset_transport_extra_kwargs_as_none():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["extra_query"] is None
assert kwargs["extra_body"] is None
@pytest.fixture
async def recording_responses_provider() -> AsyncIterator[
tuple[OpenAIProvider, list[dict[str, Any]]]
]:
# Capture the serialized fields after the runner and the Responses adapter apply settings.
bodies: list[dict[str, Any]] = []
def handler(request: httpx2.Request) -> httpx2.Response:
bodies.append(json.loads(request.content))
return httpx2.Response(
200, content=get_response_obj([get_text_message("done")]).model_dump_json()
)
async with AsyncOpenAI(
api_key="test-key",
base_url="https://example.invalid/v1",
http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)),
) as client:
yield OpenAIProvider(openai_client=client), bodies
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_runner_preserves_explicit_responses_fields_over_model_defaults(
recording_responses_provider: tuple[OpenAIProvider, list[dict[str, Any]]],
) -> None:
provider, bodies = recording_responses_provider
text = {"format": {"type": "json_object"}}
reasoning = {"effort": "high"}
for model, extra_args in (
("gpt-6-sol", {"text": text}),
("gpt-6-astra", {"text": text, "reasoning": reasoning}),
):
await Runner.run(
Agent(name="test", model=model),
"Return JSON.",
run_config=RunConfig(
model_provider=provider,
model_settings=ModelSettings(extra_args=extra_args),
tracing_disabled=True,
),
)
with pytest.raises(TypeError, match="multiple values for keyword argument 'text'"):
await Runner.run(
Agent(name="test", model="gpt-6-sol"),
"Return JSON.",
run_config=RunConfig(
model_provider=provider,
model_settings=ModelSettings(verbosity="low", extra_args={"text": text}),
tracing_disabled=True,
),
)
assert [body["text"] for body in bodies] == [text, text]
assert "reasoning" not in bodies[0]
assert bodies[1]["reasoning"] == reasoning
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("field", ["reasoning", "text"])
async def test_runner_preserves_in_place_responses_fields(
recording_responses_provider: tuple[OpenAIProvider, list[dict[str, Any]]], field: str
) -> None:
provider, bodies = recording_responses_provider
config = RunConfig(model_provider=provider, tracing_disabled=True)
agent = Agent(name="test", model="gpt-6-astra")
text = {"format": {"type": "json_object"}}
reasoning = {"effort": "high"}
agent.model_settings.extra_args = {field: reasoning if field == "reasoning" else text}
if field == "text":
agent = agent.clone(name="cloned")
await Runner.run(agent, "Return JSON.", run_config=config)
(body,) = bodies
assert body["reasoning"] == (reasoning if field == "reasoning" else {"effort": "low"})
assert body["text"] == (text if field == "text" else {"verbosity": "low"})
# A typed assignment makes the field explicit even when its value equals the SDK default.
if field == "reasoning":
agent.model_settings.reasoning = Reasoning(effort="low")
else:
agent.model_settings.verbosity = "low"
with pytest.raises(TypeError, match=f"multiple values for keyword argument '{field}'"):
await Runner.run(agent, "Return JSON.", run_config=config)
assert len(bodies) == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_get_response_with_official_client_accepts_unset_transport_extra_kwargs() -> None:
requests = await _run_responses_model_with_official_client()
assert len(requests) == 1
assert requests[0].url == "https://example.test/v1/responses"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_get_response_with_official_client_applies_transport_extra_kwargs() -> None:
requests = await _run_responses_model_with_official_client(
ModelSettings(
extra_query={"api-version": "2026-01-01-preview"},
extra_body={"extra_transport_field": "enabled"},
)
)
assert len(requests) == 1
assert requests[0].url == ("https://example.test/v1/responses?api-version=2026-01-01-preview")
assert json.loads(requests[0].content)["extra_transport_field"] == "enabled"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_custom_base_url_prompt_cache_key_uses_model_settings_only() -> None:
default_kwargs = await _run_responses_model_with_custom_base_url()
explicit_kwargs = await _run_responses_model_with_custom_base_url(
model_settings=ModelSettings(extra_args={"prompt_cache_key": "cache-key"})
)
assert "prompt_cache_key" not in default_kwargs
assert explicit_kwargs["prompt_cache_key"] == "cache-key"
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_preserves_unknown_response_include_values():
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(response_include=["response.future_flag"]),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["include"] == ["response.future_flag"]
@pytest.mark.allow_call_model_methods
def test_build_response_create_kwargs_preserves_unknown_tool_types(monkeypatch) -> None:
client = DummyWSClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
future_tool = cast(Any, {"type": "future_beta_tool", "label": "preview"})
monkeypatch.setattr(
Converter,
"convert_tools",
classmethod(
lambda cls, tools, handoffs, **kwargs: ConvertedTools(tools=[future_tool], includes=[])
),
)
kwargs = model._build_response_create_kwargs(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert kwargs["tools"] == [future_tool]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_prompt_id_omits_model_parameter():
called_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs["prompt"] == {"id": "pmpt_123"}
assert called_kwargs["model"] is omit
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_prompt_id_omits_tools_parameter_when_no_tools_configured():
called_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs["tools"] is omit
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_prompt_id_omits_tool_choice_when_no_tools_configured():
called_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(tool_choice="web_search_preview"),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs["tools"] is omit
assert called_kwargs["tool_choice"] is omit
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_choice", ["none", "required"])
async def test_prompt_id_keeps_literal_tool_choice_without_local_tools(tool_choice: str):
called_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(tool_choice=tool_choice),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs["tools"] is omit
assert called_kwargs["tool_choice"] == tool_choice
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_prompt_id_keeps_explicit_tool_search_without_local_surface() -> None:
called_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[ToolSearchTool()],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs["prompt"] == {"id": "pmpt_123"}
assert called_kwargs["tools"] == [{"type": "tool_search"}]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_ga_computer_tool_does_not_require_preview_metadata() -> None:
called_kwargs: dict[str, Any] = {}
class DummyComputer(AsyncComputer):
async def screenshot(self) -> str:
return "screenshot"
async def click(self, x: int, y: int, button: str) -> None:
pass
async def double_click(self, x: int, y: int) -> None:
pass
async def drag(self, path: list[tuple[int, int]]) -> None:
pass
async def keypress(self, keys: list[str]) -> None:
pass
async def move(self, x: int, y: int) -> None:
pass
async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
pass
async def type(self, text: str) -> None:
pass
async def wait(self) -> None:
pass
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-5.4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=True,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[ComputerTool(computer=DummyComputer())],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt=None,
)
assert called_kwargs["tools"] == [{"type": "computer"}]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_prompt_id_uses_preview_computer_payload_when_prompt_owns_model() -> None:
called_kwargs: dict[str, Any] = {}
class DummyComputer(Computer):
@property
def environment(self) -> str: # type: ignore[override]
return "mac"
@property
def dimensions(self) -> tuple[int, int]:
return (800, 600)
def screenshot(self) -> str:
return "screenshot"
def click(self, x: int, y: int, button: str) -> None:
pass
def double_click(self, x: int, y: int) -> None:
pass
def drag(self, path: list[tuple[int, int]]) -> None:
pass
def keypress(self, keys: list[str]) -> None:
pass
def move(self, x: int, y: int) -> None:
pass
def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
pass
def type(self, text: str) -> None:
pass
def wait(self) -> None:
pass
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-5.4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[ComputerTool(computer=DummyComputer())],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs["model"] is omit
assert called_kwargs["tool_choice"] is omit
assert called_kwargs["tools"] == [
{
"type": "computer_use_preview",
"environment": "mac",
"display_width": 800,
"display_height": 600,
}
]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_prompt_id_computer_without_preview_metadata_raises_clear_error() -> None:
called_kwargs: dict[str, Any] = {}
class DummyComputer(Computer):
def screenshot(self) -> str:
return "screenshot"
def click(self, x: int, y: int, button: str) -> None:
pass
def double_click(self, x: int, y: int) -> None:
pass
def drag(self, path: list[tuple[int, int]]) -> None:
pass
def keypress(self, keys: list[str]) -> None:
pass
def move(self, x: int, y: int) -> None:
pass
def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
pass
def type(self, text: str) -> None:
pass
def wait(self) -> None:
pass
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-5.4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
with pytest.raises(
UserError,
match="Preview computer tool payloads require `environment` and `dimensions`",
):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[ComputerTool(computer=DummyComputer())],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs == {}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_prompt_id_unresolved_computer_uses_preview_payload_shape() -> None:
called_kwargs: dict[str, Any] = {}
class DummyComputer(Computer):
@property
def environment(self) -> str: # type: ignore[override]
return "mac"
@property
def dimensions(self) -> tuple[int, int]:
return (800, 600)
def screenshot(self) -> str:
return "screenshot"
def click(self, x: int, y: int, button: str) -> None:
pass
def double_click(self, x: int, y: int) -> None:
pass
def drag(self, path: list[tuple[int, int]]) -> None:
pass
def keypress(self, keys: list[str]) -> None:
pass
def move(self, x: int, y: int) -> None:
pass
def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
pass
def type(self, text: str) -> None:
pass
def wait(self) -> None:
pass
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-5.4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
with pytest.raises(UserError, match="Computer tool is not initialized for serialization"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[ComputerTool(computer=lambda **_: DummyComputer())],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs == {}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_choice", ["computer", "computer_use"])
async def test_prompt_id_explicit_ga_computer_tool_choice_uses_ga_selector_and_tool(
tool_choice: str,
) -> None:
called_kwargs: dict[str, Any] = {}
class DummyComputer(Computer):
@property
def environment(self) -> str: # type: ignore[override]
return "mac"
@property
def dimensions(self) -> tuple[int, int]:
return (800, 600)
def screenshot(self) -> str:
return "screenshot"
def click(self, x: int, y: int, button: str) -> None:
pass
def double_click(self, x: int, y: int) -> None:
pass
def drag(self, path: list[tuple[int, int]]) -> None:
pass
def keypress(self, keys: list[str]) -> None:
pass
def move(self, x: int, y: int) -> None:
pass
def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
pass
def type(self, text: str) -> None:
pass
def wait(self) -> None:
pass
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="gpt-5.4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
model_is_explicit=False,
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(tool_choice=tool_choice),
tools=[ComputerTool(computer=DummyComputer())],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
prompt={"id": "pmpt_123"},
)
assert called_kwargs["model"] is omit
assert called_kwargs["tool_choice"] == {"type": "computer"}
assert called_kwargs["tools"] == [{"type": "computer"}]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_choice", ["computer", "computer_use"])
async def test_preview_model_forced_computer_tool_choice_uses_preview_selector(
tool_choice: str,
) -> None:
called_kwargs: dict[str, Any] = {}
class DummyComputer(Computer):
@property
def environment(self) -> str: # type: ignore[override]
return "mac"
@property
def dimensions(self) -> tuple[int, int]:
return (800, 600)
def screenshot(self) -> str:
return "screenshot"
def click(self, x: int, y: int, button: str) -> None:
pass
def double_click(self, x: int, y: int) -> None:
pass
def drag(self, path: list[tuple[int, int]]) -> None:
pass
def keypress(self, keys: list[str]) -> None:
pass
def move(self, x: int, y: int) -> None:
pass
def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
pass
def type(self, text: str) -> None:
pass
def wait(self) -> None:
pass
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(
model="computer-use-preview",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(tool_choice=tool_choice),
tools=[ComputerTool(computer=DummyComputer())],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert called_kwargs["model"] == "computer-use-preview"
assert called_kwargs["tool_choice"] == {"type": "computer_use_preview"}
assert called_kwargs["tools"] == [
{
"type": "computer_use_preview",
"environment": "mac",
"display_width": 800,
"display_height": 600,
}
]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
("name", "with_hosted", "as_handoff", "expected_choice"),
[
("web_search", False, False, {"type": "function", "name": "web_search"}),
("web_search", True, False, {"type": "web_search"}),
("web_search_preview", True, False, {"type": "web_search_preview"}),
("mcp", False, True, {"type": "function", "name": "mcp"}),
(
"programmatic_tool_calling",
False,
False,
{"type": "function", "name": "programmatic_tool_calling"},
),
],
)
async def test_responses_tool_choice_with_hosted_name(
name: str, with_hosted: bool, as_handoff: bool, expected_choice: dict[str, str]
) -> None:
tools: list[Tool] = [function_tool(lambda: "ok", name_override="lookup")]
handoffs = [handoff(Agent(name="Target"), tool_name_override=name)] if as_handoff else []
if not as_handoff:
tools.append(function_tool(lambda: "ok", name_override=name))
if with_hosted:
tools.append(WebSearchTool())
# Inspect the serialized request because the bug is in the provider wire format.
def handle(request: httpx2.Request) -> httpx2.Response:
body = json.loads(request.content)
assert {t["name"] for t in body["tools"] if t["type"] == "function"} == {"lookup", name}
assert any(t["type"] == "web_search" for t in body["tools"]) == with_hosted
assert body["tool_choice"] == expected_choice
return httpx2.Response(200, json=get_response_obj([]).model_dump())
async with AsyncOpenAI(
api_key="test-key",
base_url="https://provider.example/v1",
http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handle)),
max_retries=0,
) as client:
await OpenAIResponsesModel("gpt-4.1", client).get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(tool_choice=name),
tools=tools,
output_schema=None,
handoffs=handoffs,
tracing=ModelTracing.DISABLED,
)
@pytest.mark.parametrize(
"tools",
[
[
function_tool(lambda: "ok", name_override="web_search", defer_loading=True),
ToolSearchTool(),
],
tool_namespace(
name="internal",
description="Internal tools",
tools=[function_tool(lambda: "ok", name_override="web_search")],
),
[
function_tool(
lambda: "ok", name_override="web_search", allowed_callers=["programmatic"]
),
ProgrammaticToolCallingTool(),
],
],
ids=["deferred", "namespaced", "programmatic-only"],
)
def test_hosted_name_ignores_functions_unavailable_for_direct_selection(tools: list[Tool]) -> None:
converted = Converter.convert_tools(tools, [], tool_choice="web_search")
assert converted.tool_choice == {"type": "web_search"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_choice", ["shell", "apply_patch"])
async def test_builtin_tool_choice_selects_builtin_among_multiple_tools(tool_choice: str) -> None:
called_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return get_response_obj([])
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
builtin_tool: Tool = (
ShellTool(executor=lambda request: "ok")
if tool_choice == "shell"
else ApplyPatchTool(editor=cast(Any, object()))
)
model = OpenAIResponsesModel(
model="gpt-5.4",
openai_client=DummyResponsesClient(), # type: ignore[arg-type]
)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(tool_choice=tool_choice),
tools=[
function_tool(lambda: "ok", name_override="lookup_account"),
builtin_tool,
function_tool(lambda: "ok", name_override="summarize"),
],
output_schema=None,
handoffs=[handoff(Agent(name="Target"))],
tracing=ModelTracing.DISABLED,
)
assert called_kwargs["tool_choice"] == {"type": tool_choice}
assert tool_choice in [dict(tool).get("type") for tool in called_kwargs["tools"]]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_reuses_connection_and_sends_response_create_frames(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection(
[
_response_completed_frame("resp-1", 1),
_response_completed_frame("resp-2", 2),
]
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
opened: list[tuple[str, dict[str, str]]] = []
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
opened.append((ws_url, headers))
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
configured_agent = Agent(
name="configured",
model=model,
model_settings={
"reasoning": {"mode": "pro", "effort": "max", "context": "all_turns"},
"context_management": [{"type": "compaction", "compact_threshold": 200000}],
"prompt_cache_options": {"mode": "explicit", "ttl": "30m"},
"prompt_cache_retention": "24h",
"store": False,
"metadata": {"request": "example"},
},
)
first = await model.get_response(
system_instructions=None,
input="hi",
model_settings=configured_agent.model_settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
second = await model.get_response(
system_instructions=None,
input="next",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp-1",
)
assert first.response_id == "resp-1"
assert second.response_id == "resp-2"
assert client.refresh_calls == 2
assert len(opened) == 1
assert ws.sent_messages[0]["type"] == "response.create"
assert ws.sent_messages[0]["stream"] is True
assert ws.sent_messages[0]["reasoning"] == {
"context": "all_turns",
"effort": "max",
"mode": "pro",
}
assert ws.sent_messages[0]["prompt_cache_options"] == {
"mode": "explicit",
"ttl": "30m",
}
assert ws.sent_messages[0]["context_management"] == [
{"type": "compaction", "compact_threshold": 200000}
]
assert ws.sent_messages[0]["prompt_cache_retention"] == "24h"
assert ws.sent_messages[0]["store"] is False
assert ws.sent_messages[0]["metadata"] == {"request": "example"}
assert ws.sent_messages[1]["type"] == "response.create"
assert ws.sent_messages[1]["stream"] is True
assert ws.sent_messages[1]["previous_response_id"] == "resp-1"
@pytest.mark.asyncio
async def test_websocket_model_passes_keepalive_options_to_connect(monkeypatch):
import websockets.asyncio.client as websockets_client
client = DummyWSClient()
model = OpenAIResponsesWSModel(
model="gpt-4",
openai_client=client, # type: ignore[arg-type]
websocket_options={"ping_interval": 45.0, "ping_timeout": None},
)
ws = DummyWSConnection([])
captured_kwargs: dict[str, Any] = {}
async def fake_connect(ws_url: str, **kwargs: Any) -> DummyWSConnection:
captured_kwargs["ws_url"] = ws_url
captured_kwargs.update(kwargs)
return ws
monkeypatch.setattr(websockets_client, "connect", fake_connect)
opened = await model._open_websocket_connection(
"wss://example.test/v1/responses",
{"Authorization": "Bearer test-key"},
connect_timeout=10.0,
)
assert opened is ws
assert captured_kwargs["ws_url"] == "wss://example.test/v1/responses"
assert captured_kwargs["additional_headers"] == {"Authorization": "Bearer test-key"}
assert captured_kwargs["logger"].isEnabledFor(logging.DEBUG) is False
assert captured_kwargs["open_timeout"] == 10.0
assert captured_kwargs["ping_interval"] == 45.0
assert captured_kwargs["ping_timeout"] is None
@pytest.mark.asyncio
async def test_websocket_model_passes_max_size_to_connect(monkeypatch):
import websockets.asyncio.client as websockets_client
client = DummyWSClient()
model = OpenAIResponsesWSModel(
model="gpt-4",
openai_client=client, # type: ignore[arg-type]
websocket_options={"max_size": 8 * 1024 * 1024},
)
ws = DummyWSConnection([])
captured_kwargs: dict[str, Any] = {}
async def fake_connect(ws_url: str, **kwargs: Any) -> DummyWSConnection:
captured_kwargs["ws_url"] = ws_url
captured_kwargs.update(kwargs)
return ws
monkeypatch.setattr(websockets_client, "connect", fake_connect)
opened = await model._open_websocket_connection(
"wss://example.test/v1/responses",
{"Authorization": "Bearer test-key"},
connect_timeout=10.0,
)
assert opened is ws
assert captured_kwargs["max_size"] == 8 * 1024 * 1024
@pytest.mark.allow_call_model_methods
def test_websocket_model_reconnects_when_reused_from_different_event_loop(monkeypatch):
client = DummyWSClient()
ws1 = DummyWSConnection([_response_completed_frame("resp-1", 1)])
ws2 = DummyWSConnection([_response_completed_frame("resp-2", 2)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
opened: list[tuple[str, dict[str, str]]] = []
ws_connections = [ws1, ws2]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
opened.append((ws_url, headers))
return ws_connections.pop(0)
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
async def get_response(input_text: str, previous_response_id: str | None = None):
return await model.get_response(
system_instructions=None,
input=input_text,
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=previous_response_id,
)
loop1 = asyncio.new_event_loop()
loop2 = asyncio.new_event_loop()
try:
first = loop1.run_until_complete(get_response("hi"))
second = loop2.run_until_complete(get_response("next", previous_response_id="resp-1"))
finally:
loop1.close()
loop2.close()
asyncio.set_event_loop(None)
assert first.response_id == "resp-1"
assert second.response_id == "resp-2"
assert len(opened) == 2
assert ws1.close_calls == 1
assert ws2.close_calls == 0
@pytest.mark.allow_call_model_methods
def test_websocket_model_init_lazily_creates_request_lock(monkeypatch):
client = DummyWSClient()
def fail_lock(*args, **kwargs):
raise RuntimeError("asyncio.Lock() should not be called in __init__")
monkeypatch.setattr("agents.models.openai_responses.asyncio.Lock", fail_lock)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
assert model._ws_request_lock is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_stream_response_yields_typed_events(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection([_response_completed_frame("resp-stream", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
events = []
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
events.append(event)
assert len(events) == 1
assert isinstance(events[0], ResponseCompletedEvent)
assert events[0].response.id == "resp-stream"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_event_type", ["response.incomplete", "response.failed"])
async def test_websocket_model_get_response_rejects_failed_terminal_response_payload_events(
monkeypatch, terminal_event_type: str
):
client = DummyWSClient()
ws = DummyWSConnection([_response_event_frame(terminal_event_type, "resp-terminal", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ["incomplete", "failed"])
async def test_get_response_rejects_failed_terminal_response_status(status: str) -> None:
class Responses:
async def create(self, **kwargs: Any) -> Response:
return _response_with_terminal_status(status)
class Client:
responses = Responses()
base_url = httpx2.URL("https://custom.example.test/v1/")
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, Client()))
with pytest.raises(ModelBehaviorError, match=f"response.{status}"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
("transport", "stream", "status", "has_usage"),
[
pytest.param("http", False, "incomplete", True, id="http-incomplete"),
pytest.param("http", True, "incomplete", True, id="http-stream-incomplete"),
pytest.param("websocket", False, "failed", True, id="websocket-failed"),
pytest.param("websocket", True, "incomplete", True, id="websocket-stream-incomplete"),
pytest.param("http", False, "failed", False, id="http-without-usage"),
],
)
async def test_runner_accounts_for_terminal_responses_usage(
monkeypatch: pytest.MonkeyPatch, transport: str, stream: bool, status: str, has_usage: bool
) -> None:
# Exercise the Responses transport parsers before the runner sees the terminal failure.
payload = _response_with_terminal_status(status).model_dump()
if has_usage:
payload["usage"] = {
"input_tokens": 12,
"input_tokens_details": {"cached_tokens": 0},
"output_tokens": 16,
"output_tokens_details": {"reasoning_tokens": 16},
"total_tokens": 28,
}
terminal = f"response.{status}"
frame = json.dumps({"type": terminal, "response": payload, "sequence_number": 0})
async def handler(request: httpx2.Request) -> httpx2.Response:
if stream:
return httpx2.Response(
200,
content=f"event: {terminal}\ndata: {frame}\n\n",
headers={"content-type": "text/event-stream"},
)
return httpx2.Response(200, json=payload)
async with AsyncOpenAI(
api_key="test-key",
base_url="https://provider.example/v1",
http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)),
) as client:
if transport == "websocket":
model = OpenAIResponsesWSModel(model="fake", openai_client=client)
monkeypatch.setattr(
model,
"_open_websocket_connection",
AsyncMock(return_value=DummyWSConnection([frame])),
)
else:
model = OpenAIResponsesModel(model="fake", openai_client=client)
agent = Agent(name="test", model=model)
run_config = RunConfig(trace_include_sensitive_data=False)
if stream:
result = Runner.run_streamed(agent, "hi", run_config=run_config)
raw_event_types = []
with pytest.raises(ModelBehaviorError, match=terminal) as exc:
async for event in result.stream_events():
if event.type == "raw_response_event":
raw_event_types.append(event.data.type)
assert raw_event_types == [terminal]
assert result.final_output is None
assert result.raw_responses == []
assert exc.value.run_data is not None
assert result.context_wrapper.usage == exc.value.run_data.context_wrapper.usage
else:
with pytest.raises(ModelBehaviorError, match=terminal) as exc:
await Runner.run(agent, "hi", run_config=run_config)
assert exc.value.run_data is not None
assert exc.value.run_data.raw_responses == []
usage = exc.value.run_data.context_wrapper.usage
(span,) = [span for span in fetch_ordered_spans() if span.span_data.type == "response"]
assert span.error is not None
assert span.span_data.response is None
assert span.span_data.input is None
if has_usage:
assert usage.requests == 1
assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (12, 16, 28)
assert usage.output_tokens_details.reasoning_tokens == 16
(entry,) = usage.request_usage_entries
assert (entry.input_tokens, entry.output_tokens, entry.total_tokens) == (12, 16, 28)
assert entry.output_tokens_details.reasoning_tokens == 16
assert span.span_data.usage["requests"] == 1
assert span.span_data.usage["total_tokens"] == 28
assert span.span_data.usage["output_tokens_details"]["reasoning_tokens"] == 16
else:
assert usage == Usage()
assert span.span_data.usage is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("terminal_event_type", ["response.incomplete", "response.failed"])
async def test_websocket_model_stream_response_rejects_failed_terminal_response_payload_events(
monkeypatch, terminal_event_type: str
):
client = DummyWSClient()
ws = DummyWSConnection([_response_event_frame(terminal_event_type, "resp-terminal", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
events = []
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
events.append(event)
assert len(events) == 1
assert events[0].type == terminal_event_type
assert cast(Any, events[0]).response.id == "resp-terminal"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_rejects_response_error_terminal_event(monkeypatch):
model = OpenAIResponsesModel(model="gpt-4", openai_client=object()) # type: ignore[arg-type]
async def dummy_fetch_response(
system_instructions,
input,
model_settings,
tools,
output_schema,
handoffs,
previous_response_id,
conversation_id,
stream,
prompt,
):
class DummyStream:
async def __aiter__(self):
yield ResponseErrorEvent(
type="error",
code="invalid_request_error",
message="bad request",
param=None,
sequence_number=0,
)
return DummyStream()
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
events = []
with pytest.raises(ModelBehaviorError, match="invalid_request_error"):
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
events.append(event)
assert len(events) == 1
assert events[0].type == "error"
assert events[0].code == "invalid_request_error"
assert events[0].message == "bad request"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_surfaces_response_error_event(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection([_response_error_frame("invalid_request_error", "bad request", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(ResponsesWebSocketError, match="response\\.error") as exc_info:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert "invalid_request_error" in str(exc_info.value)
assert "bad request" in str(exc_info.value)
assert exc_info.value.event_type == "response.error"
assert exc_info.value.code == "invalid_request_error"
assert exc_info.value.error_message == "bad request"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_stream_response_raises_on_response_error_event(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection([_response_error_frame("invalid_request_error", "bad request", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(ResponsesWebSocketError, match="response\\.error") as exc_info:
async for _event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
):
pass
assert "invalid_request_error" in str(exc_info.value)
assert "bad request" in str(exc_info.value)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_stream_break_drops_persistent_connection(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection(
[
_response_event_frame("response.created", "resp-created", 1),
_response_completed_frame("resp-complete", 2),
]
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
stream = await model._fetch_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=True,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert event.type == "response.created"
await stream_agen.aclose()
assert ws.close_calls == 0
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_stream_close_after_terminal_event_preserves_persistent_connection(
monkeypatch,
):
client = DummyWSClient()
ws = DummyWSConnection(
[
_response_completed_frame("resp-complete-1", 1),
_response_completed_frame("resp-complete-2", 2),
]
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
opened: list[DummyWSConnection] = []
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
opened.append(ws)
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
stream = await model._fetch_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=True,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert event.type == "response.completed"
await stream_agen.aclose()
assert ws.close_calls == 0
assert model._ws_connection is ws
assert model._ws_request_lock is not None
assert model._ws_request_lock.locked() is False
second = await model.get_response(
system_instructions=None,
input="next",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert second.response_id == "resp-complete-2"
assert len(opened) == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_stream_response_terminal_close_keeps_connection(
monkeypatch,
):
client = DummyWSClient()
ws = DummyWSConnection(
[
_response_completed_frame("resp-complete-1", 1),
_response_completed_frame("resp-complete-2", 2),
]
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
opened: list[DummyWSConnection] = []
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
opened.append(ws)
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert event.type == "response.completed"
await stream_agen.aclose()
assert ws.close_calls == 0
assert model._ws_connection is ws
second = await model.get_response(
system_instructions=None,
input="next",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert second.response_id == "resp-complete-2"
assert len(opened) == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_stream_response_close_releases_inner_iterator(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection(
[
_response_event_frame("response.created", "resp-created", 1),
_response_completed_frame("resp-complete", 2),
]
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert event.type == "response.created"
await stream_agen.aclose()
assert ws.close_calls == 0
assert model._ws_connection is None
assert model._ws_request_lock is not None
assert model._ws_request_lock.locked() is False
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_stream_response_non_terminal_close_does_not_await_close_handshake(
monkeypatch,
):
class BlockingCloseWSConnection(DummyWSConnection):
def __init__(self):
super().__init__(
[
_response_event_frame("response.created", "resp-created", 1),
_response_completed_frame("resp-complete", 2),
]
)
self.close_started = asyncio.Event()
self.close_release = asyncio.Event()
class DummyTransport:
def __init__(inner_self, outer: BlockingCloseWSConnection):
inner_self.outer = outer
inner_self.abort_calls = 0
def abort(inner_self) -> None:
inner_self.abort_calls += 1
self.transport = DummyTransport(self)
async def close(self) -> None:
self.close_calls += 1
self.close_started.set()
await self.close_release.wait()
client = DummyWSClient()
ws = BlockingCloseWSConnection()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
stream_agen = cast(Any, stream)
event = await stream_agen.__anext__()
assert event.type == "response.created"
try:
await asyncio.wait_for(stream_agen.aclose(), timeout=0.5)
assert ws.transport.abort_calls == 1
assert ws.close_calls == 0
assert model._ws_connection is None
assert model._ws_request_lock is not None
assert model._ws_request_lock.locked() is False
finally:
ws.close_release.set()
await asyncio.sleep(0)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_cancellation_drops_persistent_connection(monkeypatch):
class CancelOnRecvWSConnection(DummyWSConnection):
async def recv(self) -> str:
raise asyncio.CancelledError()
client = DummyWSClient()
ws = CancelOnRecvWSConnection([])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(asyncio.CancelledError):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert ws.close_calls == 0
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_cancellation_does_not_await_close_handshake(monkeypatch):
class BlockingCloseCancelOnRecvWSConnection(DummyWSConnection):
def __init__(self):
super().__init__([])
self.recv_started = asyncio.Event()
self.close_started = asyncio.Event()
self.close_release = asyncio.Event()
class DummyTransport:
def __init__(inner_self, outer: BlockingCloseCancelOnRecvWSConnection):
inner_self.outer = outer
inner_self.abort_calls = 0
def abort(inner_self) -> None:
inner_self.abort_calls += 1
self.transport = DummyTransport(self)
async def recv(self) -> str:
self.recv_started.set()
await asyncio.Event().wait()
raise RuntimeError("unreachable")
async def close(self) -> None:
self.close_calls += 1
self.close_started.set()
await self.close_release.wait()
client = DummyWSClient()
ws = BlockingCloseCancelOnRecvWSConnection()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
request_task = asyncio.create_task(
model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
)
await asyncio.wait_for(ws.recv_started.wait(), timeout=1.0)
request_task.cancel()
try:
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(request_task, timeout=0.5)
assert ws.transport.abort_calls == 1
assert ws.close_calls == 0
assert model._ws_connection is None
finally:
ws.close_release.set()
await asyncio.sleep(0)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_preserves_pre_event_usererror(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
raise UserError("websockets dependency missing")
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(UserError, match="websockets dependency missing"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_preserves_pre_event_server_error_frame_message(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection(
[
json.dumps(
{
"type": "error",
"error": {"message": "bad auth", "type": "invalid_request_error"},
}
)
]
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(ResponsesWebSocketError, match="Responses websocket error:") as exc_info:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert "feature may not be enabled" not in str(exc_info.value)
assert "invalid_request_error" in str(exc_info.value)
assert exc_info.value.event_type == "error"
assert exc_info.value.error_type == "invalid_request_error"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_reconnects_if_cached_connection_is_closed(monkeypatch):
client = DummyWSClient()
ws1 = DummyWSConnection([_response_completed_frame("resp-1", 1)])
ws2 = DummyWSConnection([_response_completed_frame("resp-2", 2)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
opened: list[DummyWSConnection] = []
queue = [ws1, ws2]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
next_ws = queue.pop(0)
opened.append(next_ws)
return next_ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
first = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert first.response_id == "resp-1"
assert len(opened) == 1
# Simulate an idle timeout/server-side close on the cached websocket connection.
ws1.close_code = 1001
second = await model.get_response(
system_instructions=None,
input="next",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert second.response_id == "resp-2"
assert len(opened) == 2
assert ws1.close_calls == 1
assert model._ws_connection is ws2
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_retries_if_handshake_fails_before_request(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection([_response_completed_frame("resp-retried", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
nonlocal open_calls
open_calls += 1
if open_calls == 1:
raise _invalid_message_error(
"did not receive a valid HTTP response",
cause=EOFError("connection closed while reading HTTP status line"),
)
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert response.response_id == "resp-retried"
assert open_calls == 2
assert len(ws.sent_messages) == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_does_not_retry_malformed_handshake(monkeypatch):
client = DummyWSClient()
error = _invalid_message_error("malformed HTTP status line")
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
nonlocal open_calls
open_calls += 1
raise error
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(type(error)) as exc_info:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert exc_info.value is error
assert open_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_exhausts_one_retry_after_repeated_eof_handshake_failures(
monkeypatch,
):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
nonlocal open_calls
open_calls += 1
raise _invalid_message_error(
"did not receive a valid HTTP response",
cause=EOFError("connection closed while reading HTTP status line"),
)
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(RuntimeError, match="before any response events were received"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert open_calls == 2
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_close_during_failing_handshake_prevents_retry(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
handshake_started = asyncio.Event()
release_handshake = asyncio.Event()
error = _invalid_message_error(
"did not receive a valid HTTP response",
cause=EOFError("connection closed while reading HTTP status line"),
)
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
nonlocal open_calls
open_calls += 1
handshake_started.set()
await release_handshake.wait()
raise error
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
request_task = asyncio.create_task(
model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
)
await handshake_started.wait()
await model.close()
release_handshake.set()
with pytest.raises(type(error)) as exc_info:
await request_task
assert exc_info.value is error
advice = model.get_retry_advice(
ModelRetryAdviceRequest(error=exc_info.value, attempt=1, stream=False)
)
assert advice is not None
assert advice.normalized is not None
assert advice.normalized.is_abort is True
assert open_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_close_during_retry_handshake_discards_acquired_connection(
monkeypatch,
):
client = DummyWSClient()
ws = DummyWSConnection([_response_completed_frame("resp-after-close", 1)])
first_handshake = True
second_handshake_started = asyncio.Event()
release_second_handshake = asyncio.Event()
error = _invalid_message_error(
"did not receive a valid HTTP response",
cause=EOFError("connection closed while reading HTTP status line"),
)
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
nonlocal first_handshake, open_calls
open_calls += 1
if first_handshake:
first_handshake = False
raise error
second_handshake_started.set()
await release_second_handshake.wait()
return ws
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
request_task = asyncio.create_task(
model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
)
await second_handshake_started.wait()
await model.close()
release_second_handshake.set()
with pytest.raises(RuntimeError) as exc_info:
await request_task
advice = model.get_retry_advice(
ModelRetryAdviceRequest(error=exc_info.value, attempt=1, stream=False)
)
assert advice is not None
assert advice.normalized is not None
assert advice.normalized.is_abort is True
assert open_calls == 2
assert ws.sent_messages == []
assert ws.close_calls == 1
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_close_during_cached_connection_drop_prevents_replacement(
monkeypatch,
):
client = DummyWSClient()
class BlockingCloseWSConnection(DummyWSConnection):
def __init__(self, frames: list[str]):
super().__init__(frames)
self.close_started = asyncio.Event()
self.release_close = asyncio.Event()
async def close(self) -> None:
self.close_started.set()
await self.release_close.wait()
await super().close()
ws1 = BlockingCloseWSConnection([_response_completed_frame("resp-1", 1)])
ws2 = DummyWSConnection([_response_completed_frame("resp-2", 1)])
opened: list[DummyWSConnection] = []
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
connection = ws1 if not opened else ws2
opened.append(connection)
return connection
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
first = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert first.response_id == "resp-1"
ws1.close_code = 1001
request_task = asyncio.create_task(
model.get_response(
system_instructions=None,
input="next",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
)
await ws1.close_started.wait()
await model.close()
ws1.release_close.set()
with pytest.raises(RuntimeError):
await request_task
assert opened == [ws1]
assert ws2.sent_messages == []
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_does_not_retry_if_send_raises_after_writing_on_reused_connection(
monkeypatch,
):
client = DummyWSClient()
class ConnectionClosedError(Exception):
pass
ConnectionClosedError.__module__ = "websockets.client"
class DropAfterSendWriteOnReuseWSConnection(DummyWSConnection):
def __init__(self, frames: list[str]):
super().__init__(frames)
self.send_calls = 0
async def send(self, payload: str) -> None:
self.send_calls += 1
if self.send_calls > 1:
await super().send(payload)
raise ConnectionClosedError("peer closed during send after request write")
await super().send(payload)
ws1 = DropAfterSendWriteOnReuseWSConnection([_response_completed_frame("resp-1", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
nonlocal open_calls
open_calls += 1
if open_calls > 1:
raise AssertionError("Unexpected websocket retry after send started")
return ws1
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
first = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
with pytest.raises(RuntimeError, match="before any response events were received"):
await model.get_response(
system_instructions=None,
input="next",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert first.response_id == "resp-1"
assert open_calls == 1
assert ws1.send_calls == 2
assert len(ws1.sent_messages) == 2
assert ws1.close_calls == 1
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_does_not_retry_after_pre_event_disconnect_once_request_sent(
monkeypatch,
):
client = DummyWSClient()
class ConnectionClosedError(Exception):
pass
ConnectionClosedError.__module__ = "websockets.client"
class DisconnectAfterSendWSConnection(DummyWSConnection):
def __init__(self):
super().__init__([])
self.send_calls = 0
self.recv_calls = 0
async def send(self, payload: str) -> None:
self.send_calls += 1
await super().send(payload)
async def recv(self) -> str:
self.recv_calls += 1
raise ConnectionClosedError("peer closed after request send")
ws = DisconnectAfterSendWSConnection()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DisconnectAfterSendWSConnection:
nonlocal open_calls
open_calls += 1
if open_calls > 1:
raise AssertionError("Unexpected websocket retry after request frame was sent")
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(RuntimeError, match="before any response events were received"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert open_calls == 1
assert ws.send_calls == 1
assert ws.recv_calls == 1
assert ws.close_calls == 1
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_does_not_retry_after_client_initiated_close(monkeypatch):
client = DummyWSClient()
class ConnectionClosedError(Exception):
pass
ConnectionClosedError.__module__ = "websockets.client"
class AbortableRecvWSConnection(DummyWSConnection):
def __init__(self):
super().__init__([])
self.send_calls = 0
self.recv_started = asyncio.Event()
self.abort_event = asyncio.Event()
class DummyTransport:
def __init__(inner_self, outer: AbortableRecvWSConnection):
inner_self.outer = outer
inner_self.abort_calls = 0
def abort(inner_self) -> None:
inner_self.abort_calls += 1
inner_self.outer.abort_event.set()
self.transport = DummyTransport(self)
async def send(self, payload: str) -> None:
self.send_calls += 1
await super().send(payload)
async def recv(self) -> str:
self.recv_started.set()
await self.abort_event.wait()
raise ConnectionClosedError("client closed websocket")
ws = AbortableRecvWSConnection()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
open_calls = 0
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> AbortableRecvWSConnection:
nonlocal open_calls
open_calls += 1
if open_calls > 1:
raise AssertionError("Unexpected websocket reconnect after client close")
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
request_task = asyncio.create_task(
model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
)
await asyncio.wait_for(ws.recv_started.wait(), timeout=1.0)
await asyncio.wait_for(model.close(), timeout=1.0)
with pytest.raises(ConnectionClosedError, match="client closed websocket"):
await asyncio.wait_for(request_task, timeout=1.0)
assert open_calls == 1
assert ws.send_calls == 1
assert ws.transport.abort_calls == 1
assert model._ws_connection is None
@pytest.mark.allow_call_model_methods
def test_websocket_model_prepare_websocket_url_preserves_non_tls_scheme_mapping():
client = DummyWSClient()
client.base_url = httpx2.URL("http://127.0.0.1:8080/v1/")
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(extra_query=None)
assert ws_url == "ws://127.0.0.1:8080/v1/responses"
@pytest.mark.allow_call_model_methods
def test_websocket_model_prepare_websocket_url_appends_path_with_existing_query():
client = DummyWSClient()
client.websocket_base_url = "wss://proxy.example.test/v1?token=abc"
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(extra_query={"route": "team-a"})
parsed = httpx2.URL(ws_url)
assert parsed.path == "/v1/responses"
assert dict(parsed.params) == {"token": "abc", "route": "team-a"}
@pytest.mark.allow_call_model_methods
def test_websocket_model_prepare_websocket_url_accepts_legacy_httpx_url():
import httpx
client = DummyWSClient()
client.websocket_base_url = httpx.URL("https://proxy.example.test/v1?token=abc")
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(extra_query={"route": "team-a"})
parsed = httpx2.URL(ws_url)
assert parsed.scheme == "wss"
assert parsed.path == "/v1/responses"
assert dict(parsed.params) == {"token": "abc", "route": "team-a"}
@pytest.mark.allow_call_model_methods
@pytest.mark.parametrize(
("configured_ws_base_url", "expected_scheme"),
[
("http://proxy.example.test/v1?token=abc", "ws"),
("https://proxy.example.test/v1?token=abc", "wss"),
],
)
def test_websocket_model_prepare_websocket_url_normalizes_explicit_http_schemes(
configured_ws_base_url: str, expected_scheme: str
):
client = DummyWSClient()
client.websocket_base_url = configured_ws_base_url
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(extra_query={"route": "team-a"})
parsed = httpx2.URL(ws_url)
assert parsed.scheme == expected_scheme
assert parsed.path == "/v1/responses"
assert dict(parsed.params) == {"token": "abc", "route": "team-a"}
@pytest.mark.allow_call_model_methods
@pytest.mark.parametrize("extra_query", [omit, NOT_GIVEN])
def test_websocket_model_prepare_websocket_url_treats_top_level_omit_sentinels_as_absent(
extra_query,
):
client = DummyWSClient()
client.websocket_base_url = "wss://proxy.example.test/v1?token=abc"
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(extra_query=extra_query)
parsed = httpx2.URL(ws_url)
assert parsed.path == "/v1/responses"
assert dict(parsed.params) == {"token": "abc"}
@pytest.mark.allow_call_model_methods
def test_websocket_model_prepare_websocket_url_skips_not_given_query_values():
client = DummyWSClient()
client.websocket_base_url = "wss://proxy.example.test/v1?token=abc"
client.default_query = {"api-version": NOT_GIVEN, "route": "team-a"}
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(extra_query={"tenant": NOT_GIVEN, "region": "us"})
parsed = httpx2.URL(ws_url)
assert parsed.path == "/v1/responses"
assert dict(parsed.params) == {"token": "abc", "route": "team-a", "region": "us"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_filters_omit_from_extra_body():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
frame, _ws_url, _headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"extra_body": {"keep": "value", "drop": omit},
}
)
assert frame["type"] == "response.create"
assert frame["keep"] == "value"
assert "drop" not in frame
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("extra_body", [omit, NOT_GIVEN])
async def test_websocket_model_prepare_websocket_request_ignores_top_level_extra_body_sentinels(
extra_body,
):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
frame, _ws_url, _headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"extra_body": extra_body,
}
)
assert frame["type"] == "response.create"
assert frame["stream"] is True
assert frame["model"] == "gpt-4"
assert frame["input"] == "hi"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_preserves_envelope_fields():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
frame, _ws_url, _headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"extra_body": {
"type": "not-response-create",
"stream": False,
"custom": "value",
},
}
)
assert frame["type"] == "response.create"
assert frame["stream"] is True
assert frame["custom"] == "value"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_strips_client_timeout_kwarg():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
frame, _ws_url, _headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"timeout": 30.0,
"metadata": {"request_id": "123"},
}
)
assert frame["type"] == "response.create"
assert frame["metadata"] == {"request_id": "123"}
assert "timeout" not in frame
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_skips_not_given_values():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
frame, _ws_url, _headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"user": NOT_GIVEN,
"stream_options": NOT_GIVEN,
"extra_body": {
"metadata": {"request_id": "123"},
"optional_field": NOT_GIVEN,
},
}
)
assert frame["type"] == "response.create"
assert frame["stream"] is True
assert frame["metadata"] == {"request_id": "123"}
assert "user" not in frame
assert "stream_options" not in frame
assert "optional_field" not in frame
json.dumps(frame)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_applies_timeout_to_recv(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class SlowRecvWSConnection(DummyWSConnection):
async def recv(self) -> str:
await asyncio.sleep(0.2)
return await super().recv()
ws = SlowRecvWSConnection([_response_completed_frame("resp-timeout", 1)])
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(TimeoutError, match="Responses websocket receive timed out"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"timeout": 0.01}),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert ws.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_marks_partial_receive_timeout_unsafe_to_replay(
monkeypatch,
):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class PartialThenSlowRecvWSConnection(DummyWSConnection):
def __init__(self) -> None:
super().__init__([_response_event_frame("response.created", "resp-partial", 1)])
self.recv_calls = 0
async def recv(self) -> str:
self.recv_calls += 1
if self.recv_calls == 1:
return await super().recv()
await asyncio.sleep(0.2)
return await super().recv()
ws = PartialThenSlowRecvWSConnection()
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(TimeoutError, match="Responses websocket receive timed out") as exc_info:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"timeout": 0.01}),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
error = exc_info.value
assert getattr(error, "_openai_agents_ws_replay_safety", None) == "unsafe"
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
assert ws.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_applies_timeout_while_waiting_for_request_lock(
monkeypatch,
):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
recv_started = asyncio.Event()
release_first_request = asyncio.Event()
class BlockingRecvWSConnection(DummyWSConnection):
async def recv(self) -> str:
recv_started.set()
await release_first_request.wait()
return await super().recv()
ws = BlockingRecvWSConnection(
[
_response_completed_frame("resp-lock-1", 1),
_response_completed_frame("resp-lock-2", 2),
]
)
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
first_task = asyncio.create_task(
model.get_response(
system_instructions=None,
input="first",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
)
await asyncio.wait_for(recv_started.wait(), timeout=1.0)
with pytest.raises(TimeoutError, match="request lock wait timed out"):
await model.get_response(
system_instructions=None,
input="second",
model_settings=ModelSettings(extra_args={"timeout": 0.01}),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
release_first_request.set()
first_response = await first_task
assert first_response.response_id == "resp-lock-1"
assert len(ws.sent_messages) == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_allows_zero_pool_timeout_when_lock_uncontended(
monkeypatch,
):
client = DummyWSClient()
client.timeout = httpx2.Timeout(connect=1.0, read=1.0, write=1.0, pool=0.0)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws = DummyWSConnection([_response_completed_frame("resp-zero-pool", 1)])
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert response.response_id == "resp-zero-pool"
assert len(ws.sent_messages) == 1
@pytest.mark.allow_call_model_methods
def test_websocket_model_request_timeouts_accept_legacy_httpx_timeout():
import httpx
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
timeouts = model._get_websocket_request_timeouts(
httpx.Timeout(connect=1.0, read=2.0, write=3.0, pool=4.0)
)
assert timeouts.lock == 4.0
assert timeouts.connect == 1.0
assert timeouts.send == 3.0
assert timeouts.recv == 2.0
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_allows_zero_timeout_when_ws_ops_are_immediate(
monkeypatch,
):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws = DummyWSConnection([_response_completed_frame("resp-zero-timeout", 1)])
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"timeout": 0}),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert response.response_id == "resp-zero-timeout"
assert len(ws.sent_messages) == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_uses_client_default_timeout_when_no_override(
monkeypatch,
):
client = DummyWSClient()
client.timeout = httpx2.Timeout(connect=1.0, read=0.01, write=1.0, pool=1.0)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class SlowRecvWSConnection(DummyWSConnection):
async def recv(self) -> str:
await asyncio.sleep(0.2)
return await super().recv()
ws = SlowRecvWSConnection([_response_completed_frame("resp-timeout-default", 1)])
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(TimeoutError, match="Responses websocket receive timed out"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert ws.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_get_response_uses_client_default_timeout_when_override_is_not_given(
monkeypatch,
):
client = DummyWSClient()
client.timeout = httpx2.Timeout(connect=1.0, read=0.01, write=1.0, pool=1.0)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class SlowRecvWSConnection(DummyWSConnection):
async def recv(self) -> str:
await asyncio.sleep(0.2)
return await super().recv()
ws = SlowRecvWSConnection([_response_completed_frame("resp-timeout-not-given", 1)])
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
with pytest.raises(TimeoutError, match="Responses websocket receive timed out"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(extra_args={"timeout": NOT_GIVEN}),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert ws.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_includes_client_auth_headers():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
_frame, _ws_url, headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
}
)
assert headers["Authorization"] == "Bearer test-key"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_default_headers_override_auth_case_insensitively():
client = DummyWSClient()
client.default_headers["authorization"] = "Bearer override-key"
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
_frame, _ws_url, headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
}
)
assert headers["authorization"] == "Bearer override-key"
assert "Authorization" not in headers
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_omit_removes_inherited_header():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
_frame, _ws_url, headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"extra_headers": {"User-Agent": omit},
}
)
assert "Authorization" in headers
assert "User-Agent" not in headers
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_replaces_header_case_insensitively():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
_frame, _ws_url, headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"extra_headers": {
"authorization": "Bearer override-key",
"user-agent": "Custom UA",
},
}
)
assert headers["authorization"] == "Bearer override-key"
assert headers["user-agent"] == "Custom UA"
assert "Authorization" not in headers
assert "User-Agent" not in headers
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_prepare_websocket_request_skips_not_given_header_values():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
_frame, _ws_url, headers = await model._prepare_websocket_request(
{
"model": "gpt-4",
"input": "hi",
"stream": True,
"extra_headers": {
"Authorization": NOT_GIVEN,
"X-Optional": NOT_GIVEN,
},
}
)
assert headers["Authorization"] == "Bearer test-key"
assert "X-Optional" not in headers
assert "NOT_GIVEN" not in headers.values()
@pytest.mark.allow_call_model_methods
def test_websocket_model_prepare_websocket_url_includes_client_default_query():
client = DummyWSClient()
client.websocket_base_url = "wss://proxy.example.test/v1?token=abc"
client.default_query = {"api-version": "2025-01-01-preview", "omit_me": omit}
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(
extra_query={"route": "team-a", "api-version": "2026-01-01-preview"}
)
parsed = httpx2.URL(ws_url)
assert parsed.path == "/v1/responses"
assert dict(parsed.params) == {
"token": "abc",
"api-version": "2026-01-01-preview",
"route": "team-a",
}
@pytest.mark.allow_call_model_methods
def test_websocket_model_prepare_websocket_url_omit_removes_inherited_query_params():
client = DummyWSClient()
client.websocket_base_url = "wss://proxy.example.test/v1?token=abc"
client.default_query = {"route": "team-a", "region": "us"}
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
ws_url = model._prepare_websocket_url(extra_query={"token": omit, "route": omit, "keep": "1"})
parsed = httpx2.URL(ws_url)
assert parsed.path == "/v1/responses"
assert dict(parsed.params) == {"region": "us", "keep": "1"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_close_closes_persistent_connection(monkeypatch):
client = DummyWSClient()
ws = DummyWSConnection([_response_completed_frame("resp-close", 1)])
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return ws
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert ws.close_calls == 0
await model.close()
assert ws.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_close_falls_back_to_transport_abort_on_close_error():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
class DummyTransport:
def __init__(self):
self.abort_calls = 0
def abort(self):
self.abort_calls += 1
class FailingWSConnection:
def __init__(self):
self.transport = DummyTransport()
async def close(self):
raise RuntimeError("attached to a different loop")
ws = FailingWSConnection()
model._ws_connection = ws
model._ws_connection_identity = ("wss://example.test", (("authorization", "x"),))
await model.close()
assert ws.transport.abort_calls == 1
assert model._ws_connection is None
assert model._ws_connection_identity is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_close_does_not_wait_for_held_request_lock():
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
request_lock = model._get_ws_request_lock()
await request_lock.acquire()
class DummyTransport:
def __init__(self):
self.abort_calls = 0
def abort(self):
self.abort_calls += 1
class HangingCloseWSConnection:
def __init__(self):
self.transport = DummyTransport()
self.close_calls = 0
async def close(self) -> None:
self.close_calls += 1
await asyncio.sleep(3600)
ws = HangingCloseWSConnection()
model._ws_connection = ws
model._ws_connection_identity = ("wss://example.test", (("authorization", "x"),))
try:
await asyncio.wait_for(model.close(), timeout=0.1)
finally:
if request_lock.locked():
request_lock.release()
assert ws.transport.abort_calls == 1
assert ws.close_calls == 0
assert model._ws_connection is None
assert model._ws_connection_identity is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_open_websocket_connection_disables_message_size_limit(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
captured: dict[str, Any] = {}
sentinel = object()
async def fake_connect(*args: Any, **kwargs: Any) -> object:
captured["args"] = args
captured["kwargs"] = kwargs
return sentinel
monkeypatch.setattr("websockets.asyncio.client.connect", fake_connect)
result = await model._open_websocket_connection(
"wss://proxy.example.test/v1/responses",
{"Authorization": "Bearer test-key"},
connect_timeout=None,
)
assert result is sentinel
assert captured["args"] == ("wss://proxy.example.test/v1/responses",)
assert captured["kwargs"]["user_agent_header"] is None
assert captured["kwargs"]["additional_headers"] == {"Authorization": "Bearer test-key"}
assert captured["kwargs"]["max_size"] is None
assert captured["kwargs"]["open_timeout"] is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_model_open_websocket_connection_honors_connect_timeout(monkeypatch):
client = DummyWSClient()
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=client) # type: ignore[arg-type]
captured: dict[str, Any] = {}
sentinel = object()
async def fake_connect(*args: Any, **kwargs: Any) -> object:
captured["args"] = args
captured["kwargs"] = kwargs
return sentinel
monkeypatch.setattr("websockets.asyncio.client.connect", fake_connect)
result = await model._open_websocket_connection(
"wss://proxy.example.test/v1/responses",
{"Authorization": "Bearer test-key"},
connect_timeout=42.0,
)
assert result is sentinel
assert captured["kwargs"]["open_timeout"] == 42.0
@pytest.mark.allow_call_model_methods
def test_get_retry_advice_uses_openai_headers() -> None:
request = httpx2.Request("POST", "https://api.openai.com/v1/responses")
response = httpx2.Response(
429,
request=request,
headers={
"x-should-retry": "true",
"retry-after-ms": "250",
"x-request-id": "req_456",
},
json={"error": {"code": "rate_limit"}},
)
error = RateLimitError(
"rate limited", response=response, body={"error": {"code": "rate_limit"}}
)
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, object()))
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.retry_after == 0.25
assert advice.replay_safety == "safe"
assert advice.normalized is not None
assert advice.normalized.error_code == "rate_limit"
assert advice.normalized.status_code == 429
assert advice.normalized.request_id == "req_456"
@pytest.mark.allow_call_model_methods
def test_get_retry_advice_keeps_stateful_transport_failures_ambiguous() -> None:
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, object()))
error = APIConnectionError(
message="connection error",
request=httpx2.Request("POST", "https://api.openai.com/v1/responses"),
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
assert advice.normalized is not None
assert advice.normalized.is_network_error is True
@pytest.mark.allow_call_model_methods
def test_get_retry_advice_marks_stateful_http_failures_replay_safe() -> None:
request = httpx2.Request("POST", "https://api.openai.com/v1/responses")
response = httpx2.Response(
429,
request=request,
json={"error": {"code": "rate_limit"}},
)
error = RateLimitError(
"rate limited", response=response, body={"error": {"code": "rate_limit"}}
)
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, object()))
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety == "safe"
assert advice.normalized is not None
assert advice.normalized.status_code == 429
@pytest.mark.allow_call_model_methods
def test_get_retry_advice_keeps_stateless_transport_failures_retryable() -> None:
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, object()))
error = APIConnectionError(
message="connection error",
request=httpx2.Request("POST", "https://api.openai.com/v1/responses"),
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
assert advice.normalized is not None
assert advice.normalized.is_network_error is True
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_ambiguous_replay_unsafe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = RuntimeError("Responses websocket connection closed before a terminal response event.")
error.__cause__ = _connection_closed_error("peer closed after request send")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_allows_stateless_ambiguous_disconnect_retry() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = RuntimeError("Responses websocket connection closed before a terminal response event.")
error.__cause__ = _connection_closed_error("peer closed after request send")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_keeps_wrapped_pre_send_disconnect_safe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = RuntimeError(
"Responses websocket connection closed before any response events were received."
)
setattr(error, "_openai_agents_ws_replay_safety", "safe") # noqa: B010
error.__cause__ = _connection_closed_error("peer closed before request send")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety == "safe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_allows_stateless_wrapped_post_send_disconnect_retry() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = RuntimeError(
"Responses websocket connection closed before any response events were received."
)
setattr(error, "_openai_agents_ws_replay_safety", "unsafe") # noqa: B010
error.__cause__ = _connection_closed_error("peer closed after request send")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_allows_stateless_nonstream_post_send_retry() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = RuntimeError(
"Responses websocket connection closed before any response events were received."
)
setattr(error, "_openai_agents_ws_replay_safety", "unsafe") # noqa: B010
error.__cause__ = _connection_closed_error("peer closed after request send")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_wrapped_post_send_disconnect_unsafe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = RuntimeError(
"Responses websocket connection closed before any response events were received."
)
setattr(error, "_openai_agents_ws_replay_safety", "unsafe") # noqa: B010
error.__cause__ = _connection_closed_error("peer closed after request send")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_partial_nonstream_failure_unsafe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = TimeoutError("Responses websocket receive timed out after 5.0 seconds.")
setattr(error, "_openai_agents_ws_replay_safety", "unsafe") # noqa: B010
setattr(error, "_openai_agents_ws_response_started", True) # noqa: B010
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_connect_timeout_replay_safe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = TimeoutError("Responses websocket connect timed out after 5.0 seconds.")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety == "safe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_handshake_failure_replay_safe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = _invalid_message_error(
"did not receive a valid HTTP response",
cause=EOFError("connection closed while reading HTTP status line"),
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety == "safe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_ignores_malformed_handshake() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = _invalid_message_error("malformed HTTP status line")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id="resp_prev",
)
)
assert advice is None
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_close_invalidation_non_retryable() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = RuntimeError("Responses websocket connection closed while establishing a connection.")
setattr(error, "_openai_agents_ws_close_invalidated", True) # noqa: B010
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is False
assert advice.normalized is not None
assert advice.normalized.is_abort is True
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_request_lock_timeout_replay_safe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = TimeoutError("Responses websocket request lock wait timed out after 5.0 seconds.")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety == "safe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_marks_stateful_receive_timeout_unsafe() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = TimeoutError("Responses websocket receive timed out after 5.0 seconds.")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_allows_stateless_receive_timeout_retry() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = TimeoutError("Responses websocket receive timed out after 5.0 seconds.")
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@pytest.mark.allow_call_model_methods
@pytest.mark.parametrize("previous_response_id", [None, "resp_prev"])
def test_websocket_get_retry_advice_marks_pre_response_overload_retryable(
previous_response_id: str | None,
) -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = ResponsesWebSocketError(
{
"type": "error",
"error": {
"type": "service_unavailable_error",
"code": "server_is_overloaded",
"message": "Our servers are currently overloaded. Please try again later.",
},
}
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
previous_response_id=previous_response_id,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@pytest.mark.allow_call_model_methods
@pytest.mark.parametrize("previous_response_id", [None, "resp_prev"])
def test_websocket_get_retry_advice_marks_pre_response_server_error_retryable(
previous_response_id: str | None,
) -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = ResponsesWebSocketError(
{
"type": "error",
"error": {
"type": "server_error",
"code": None,
"message": "Sorry, something went wrong.",
},
}
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
previous_response_id=previous_response_id,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_does_not_override_non_transient_error_code() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = ResponsesWebSocketError(
{
"type": "error",
"error": {
"type": "server_error",
"code": "invalid_request_error",
"message": "Invalid request.",
},
}
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=True,
)
)
assert advice is None
@pytest.mark.allow_call_model_methods
@pytest.mark.parametrize(
("error_type", "code"),
[
("service_unavailable_error", "server_is_overloaded"),
("server_error", None),
],
)
def test_websocket_get_retry_advice_keeps_partial_transient_error_unsafe(
error_type: str,
code: str | None,
) -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = ResponsesWebSocketError(
{
"type": "error",
"error": {
"type": error_type,
"code": code,
"message": "Transient provider error.",
},
}
)
setattr(error, "_openai_agents_ws_replay_safety", "unsafe") # noqa: B010
setattr(error, "_openai_agents_ws_response_started", True) # noqa: B010
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_ignores_other_pre_response_error_codes() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = ResponsesWebSocketError(
{
"type": "error",
"error": {
"type": "invalid_request_error",
"code": "invalid_request",
"message": "Invalid request.",
},
}
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is None
def test_get_client_disables_provider_managed_retries_when_requested() -> None:
class DummyClient:
def __init__(self):
self.calls: list[dict[str, int]] = []
def with_options(self, **kwargs):
self.calls.append(kwargs)
return "retry-client"
client = DummyClient()
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, client))
assert cast(object, model._get_client()) is client
with provider_managed_retries_disabled(True):
assert cast(object, model._get_client()) == "retry-client"
assert client.calls == [{"max_retries": 0}]
def test_websocket_pre_event_disconnect_retry_respects_websocket_retry_disable() -> None:
assert _should_retry_pre_event_websocket_disconnect() is True
with websocket_pre_event_retries_disabled(True):
assert _should_retry_pre_event_websocket_disconnect() is False
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_lets_in_flight_close_finish_after_cancellation() -> None:
"""Cancelling during the cleanup `aclose` continues that close instead of abandoning it."""
class CloseSignalingStream:
"""Exhausts normally, then signals from `aclose` and blocks until released."""
def __init__(self, close_started: asyncio.Event, release: asyncio.Event) -> None:
self._yielded = False
self._close_started = close_started
self._release = release
self.aclose_calls = 0
self.aclose_completed = 0
def __aiter__(self) -> CloseSignalingStream:
return self
async def __anext__(self):
if self._yielded:
raise StopAsyncIteration
self._yielded = True
return ResponseCompletedEvent(
type="response.completed",
response=get_response_obj([]),
sequence_number=0,
)
async def aclose(self) -> None:
self.aclose_calls += 1
self._close_started.set()
await self._release.wait()
self.aclose_completed += 1
close_started = asyncio.Event()
release = asyncio.Event()
provider_stream = CloseSignalingStream(close_started, release)
class DummyResponses:
async def create(self, **kwargs):
return provider_stream
class DummyResponsesClient:
def __init__(self):
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4", openai_client=DummyResponsesClient()) # type: ignore[arg-type]
stream_agen = cast(
Any,
model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
),
)
async def consume() -> None:
async for _event in stream_agen:
pass
task = asyncio.create_task(consume())
try:
# The stream exhausts on its own, so the consumer reaches the cleanup `finally`
# and suspends inside the provider close.
await asyncio.wait_for(close_started.wait(), timeout=5)
assert provider_stream.aclose_calls == 1
assert provider_stream.aclose_completed == 0
task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=5)
# The cancelled consumer must not have started a second close.
assert provider_stream.aclose_calls == 1
assert provider_stream.aclose_completed == 0
release.set()
for _ in range(200):
if provider_stream.aclose_completed == 1:
break
await asyncio.sleep(0.01)
assert provider_stream.aclose_calls == 1
assert provider_stream.aclose_completed == 1
finally:
release.set()
task.cancel()
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_reports_response_started() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = _connection_closed_error("no close frame received or sent")
setattr(error, "_openai_agents_ws_replay_safety", "unsafe") # noqa: B010
setattr(error, "_openai_agents_ws_response_started", True) # noqa: B010
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.replay_safety == "unsafe"
# A retry policy needs this to tell a response-started disconnect apart from
# other replay-unsafe failures before approving a replay.
assert advice.response_started is True
@pytest.mark.allow_call_model_methods
def test_websocket_get_retry_advice_reports_no_response_started_for_stateful_request() -> None:
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
error = _connection_closed_error("no close frame received or sent")
setattr(error, "_openai_agents_ws_replay_safety", "unsafe") # noqa: B010
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
previous_response_id="resp_1",
)
)
assert advice is not None
assert advice.replay_safety == "unsafe"
assert advice.response_started is False
def _response_without_usage() -> Response:
return Response(
id="resp-no-usage",
created_at=0,
model="fake",
object="response",
output=[],
tool_choice="none",
tools=[],
top_p=None,
parallel_tool_calls=False,
usage=None,
)
def _response_with_terminal_status(status: str) -> Response:
return Response(
id="resp-terminal",
created_at=0,
model="fake",
object="response",
output=[],
tool_choice="none",
tools=[],
top_p=None,
parallel_tool_calls=False,
usage=None,
status=status,
incomplete_details=(
IncompleteDetails(reason="max_output_tokens") if status == "incomplete" else None
),
)
def _completed_event_without_usage() -> ResponseCompletedEvent:
return ResponseCompletedEvent(
response=_response_without_usage(),
type="response.completed",
sequence_number=0,
)
def _streaming_client_for(events: list[Any]) -> Any:
class IteratorStream:
def __init__(self) -> None:
self._remaining = list(events)
def __aiter__(self) -> IteratorStream:
return self
async def __anext__(self) -> Any:
if not self._remaining:
raise StopAsyncIteration
return self._remaining.pop(0)
async def close(self) -> None:
return None
stream = IteratorStream()
class APIResponse:
request_id = "req-1"
async def parse(self) -> Any:
return stream
class StreamingContextManager:
async def __aenter__(self) -> APIResponse:
return APIResponse()
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
return False
class Responses:
with_streaming_response = SimpleNamespace(create=lambda **kwargs: StreamingContextManager())
class Client:
responses = Responses()
base_url = httpx2.URL("https://custom.example.test/v1/")
return Client()
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_request_is_counted_when_responses_provider_omits_usage() -> None:
class Responses:
async def create(self, **kwargs: Any) -> Response:
return _response_without_usage()
class Client:
responses = Responses()
base_url = httpx2.URL("https://custom.example.test/v1/")
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, Client()))
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(preserve_raw_usage=True),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert response.usage.requests == 1
assert response.usage.total_tokens == 0
assert response.raw_usage is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_streamed_request_is_counted_when_responses_provider_omits_usage() -> None:
completed = _completed_event_without_usage()
client = _streaming_client_for([completed])
agent = Agent(
name="test",
model=OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, client)),
)
result = Runner.run_streamed(agent, "hi")
async for _ in result.stream_events():
pass
assert result.context_wrapper.usage.requests == 1
assert result.context_wrapper.usage.total_tokens == 0
assert completed.response.usage is None
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_fallback_response_stream_counts_request_without_usage() -> None:
completed = _completed_event_without_usage()
class IteratorStream:
def __init__(self) -> None:
self._remaining = [completed]
self.close_calls = 0
def __aiter__(self) -> IteratorStream:
return self
async def __anext__(self) -> ResponseCompletedEvent:
if not self._remaining:
raise StopAsyncIteration
return self._remaining.pop()
async def aclose(self) -> None:
self.close_calls += 1
stream = IteratorStream()
class Responses:
async def create(self, **kwargs: Any) -> IteratorStream:
return stream
class Client:
responses = Responses()
base_url = httpx2.URL("https://custom.example.test/v1/")
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, Client()))
result = Runner.run_streamed(Agent(name="test", model=model), "hi")
async for _ in result.stream_events():
pass
assert result.context_wrapper.usage.requests == 1
assert result.context_wrapper.usage.total_tokens == 0
assert stream.close_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_fallback_response_stream_closes_resolved_iterable_iterator() -> None:
class IterableStream:
def __init__(self) -> None:
self.source_closed = False
self.iterator_closed = False
async def __aiter__(self) -> Any:
try:
yield _completed_event_without_usage()
yield _completed_event_without_usage()
finally:
self.iterator_closed = True
async def aclose(self) -> None:
self.source_closed = True
source = IterableStream()
class Responses:
async def create(self, **kwargs: Any) -> IterableStream:
return source
class Client:
responses = Responses()
base_url = httpx2.URL("https://custom.example.test/v1/")
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, Client()))
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
async for _ in stream:
break
await stream.aclose()
assert source.iterator_closed
assert source.source_closed
@pytest.mark.asyncio
async def test_response_stream_closes_source_when_resolved_iterator_close_fails() -> None:
class FailingIterator:
def __init__(self) -> None:
self._remaining = [_completed_event_without_usage()]
def __aiter__(self) -> FailingIterator:
return self
async def __anext__(self) -> ResponseCompletedEvent:
if not self._remaining:
raise StopAsyncIteration
return self._remaining.pop()
async def aclose(self) -> None:
raise RuntimeError("iterator close failed")
iterator = FailingIterator()
class IterableSource:
def __init__(self) -> None:
self.closed = False
def __aiter__(self) -> FailingIterator:
return iterator
async def aclose(self) -> None:
self.closed = True
raise RuntimeError("source close failed")
source = IterableSource()
async def cleanup() -> None:
return None
stream = _ResponseStreamWithRequestId(source, request_id=None, cleanup=cleanup)
await stream.__anext__()
with pytest.raises(RuntimeError, match="iterator close failed"):
await stream.aclose()
assert source.closed
@pytest.mark.asyncio
async def test_response_stream_closes_distinct_source_after_normal_exhaustion() -> None:
class IterableSource:
def __init__(self) -> None:
self.source_close_calls = 0
self.iterator_finalized = False
async def __aiter__(self) -> Any:
try:
yield _completed_event_without_usage()
finally:
self.iterator_finalized = True
async def aclose(self) -> None:
self.source_close_calls += 1
source = IterableSource()
cleanup_calls = 0
async def cleanup() -> None:
nonlocal cleanup_calls
cleanup_calls += 1
stream = _ResponseStreamWithRequestId(source, request_id=None, cleanup=cleanup)
async for _ in stream:
pass
assert source.iterator_finalized
assert source.source_close_calls == 1
assert cleanup_calls == 1
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_request_is_counted_when_responses_provider_omits_usage(
monkeypatch: pytest.MonkeyPatch,
) -> None:
frame = json.dumps(
{
"type": "response.completed",
"response": _response_without_usage().model_dump(),
"sequence_number": 1,
}
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return DummyWSConnection([frame])
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert response.usage.requests == 1
assert response.usage.total_tokens == 0
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_websocket_stream_counts_request_without_usage(
monkeypatch: pytest.MonkeyPatch,
) -> None:
frame = json.dumps(
{
"type": "response.completed",
"response": _response_without_usage().model_dump(),
"sequence_number": 1,
}
)
model = OpenAIResponsesWSModel(model="gpt-4", openai_client=cast(Any, DummyWSClient()))
async def fake_open(
ws_url: str, headers: dict[str, str], *, connect_timeout: float | None = None
) -> DummyWSConnection:
return DummyWSConnection([frame])
monkeypatch.setattr(model, "_open_websocket_connection", fake_open)
result = Runner.run_streamed(Agent(name="test", model=model), "hi")
async for _ in result.stream_events():
pass
assert result.context_wrapper.usage.requests == 1
assert result.context_wrapper.usage.total_tokens == 0
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_response_span_counts_request_without_usage() -> None:
class Responses:
async def create(self, **kwargs: Any) -> Response:
return _response_without_usage()
class Client:
responses = Responses()
base_url = httpx2.URL("https://custom.example.test/v1/")
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, Client()))
with trace("test"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.ENABLED,
)
spans = [span.export() for span in fetch_ordered_spans() if span.span_data.type == "response"]
assert len(spans) == 1
assert spans[0]["span_data"]["usage"]["requests"] == 1 # type: ignore[index]
assert spans[0]["span_data"]["usage"]["total_tokens"] == 0 # type: ignore[index]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_streamed_span_counts_request_before_terminal_event_close() -> None:
client = _streaming_client_for([_completed_event_without_usage()])
model = OpenAIResponsesModel(model="gpt-4", openai_client=cast(Any, client))
with trace("test"):
stream = model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.ENABLED,
)
async for event in stream:
if event.type == "response.completed":
break
await stream.aclose()
spans = [span.export() for span in fetch_ordered_spans() if span.span_data.type == "response"]
assert len(spans) == 1
assert spans[0]["span_data"]["usage"]["requests"] == 1 # type: ignore[index]
assert spans[0]["span_data"]["usage"]["total_tokens"] == 0 # type: ignore[index]