openai-agents-python/tests/test_tool_approval_call_id_reuse.py
Justin Beckwith dda19f9b91
perf: avoid repeatedly fingerprinting completed tool calls (#5421)
Co-authored-by: Afonso Ribeiro <afonso.diamantino2@gmail.com>
2026-10-26 21:37:07 +00:00

3356 lines
108 KiB
Python

from __future__ import annotations
import asyncio
import json
import warnings
from types import SimpleNamespace
from typing import Any, Literal, cast
import pytest
from openai.types.responses import ResponseCustomToolCall, ResponseFunctionToolCall
from openai.types.responses.response_computer_tool_call import (
ActionScreenshot,
PendingSafetyCheck,
ResponseComputerToolCall,
)
from openai.types.responses.response_function_web_search import (
ActionSearch,
ActionSearchSource,
ResponseFunctionWebSearch,
)
from openai.types.responses.response_output_item import McpApprovalRequest
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
from agents import (
Agent,
ApplyPatchTool,
ComputerTool,
CustomTool,
HostedMCPTool,
MCPToolApprovalFunctionResult,
MCPToolApprovalRequest,
RunConfig,
Runner,
ShellTool,
ToolGuardrailFunctionOutput,
ToolOutputGuardrailData,
ToolOutputGuardrailTripwireTriggered,
function_tool,
handoff,
tool_output_guardrail,
)
from agents._tool_invocation import (
tool_invocation_call_id,
tool_invocation_identity,
tool_invocation_identity_and_scope,
)
from agents.decorators import tool
from agents.editor import ApplyPatchOperation, ApplyPatchResult
from agents.exceptions import ModelBehaviorError, UserError
from agents.items import (
ModelResponse,
ToolApprovalItem,
ToolCallItem,
ToolCallOutputItem,
TResponseOutputItem,
)
from agents.lifecycle import RunHooks
from agents.models.interface import Model, ModelProvider
from agents.run_context import RunContextWrapper
from agents.run_internal.run_loop import ToolRunFunction
from agents.run_internal.run_steps import ProcessedResponse
from agents.run_internal.tool_execution import (
collect_manual_mcp_approvals,
process_hosted_mcp_approvals,
resolve_approval_rejection_message,
)
from agents.run_internal.tool_planning import (
_collect_runs_by_approval,
_dedupe_processed_response_invocations,
)
from agents.run_state import RunState
from agents.stream_events import RunItemStreamEvent
from agents.testing import ScriptedModel
from agents.tool import Tool
from agents.tool_context import ToolContext
from tests.model_test_helpers import get_exact_output_stream_step
from tests.test_computer_tool_lifecycle import FakeComputer
from tests.test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message
from tests.utils.hitl import make_apply_patch_dict, make_shell_call, make_state_with_interruptions
class _ScriptedProvider(ModelProvider):
def __init__(self, model: Model) -> None:
self.model = model
def get_model(self, model_name: str | None) -> Model:
assert model_name == "scripted-provider-model"
return self.model
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_unique_tool_turns_do_not_refingerprint_completed_history(
monkeypatch: pytest.MonkeyPatch, mode: Literal["non_streamed", "streamed"]
) -> None:
turns = 24
executed: list[str] = []
identity_calls = 0
@tool
async def lookup(value: str) -> str:
executed.append(value)
return value
def counting_identity(raw_item: Any, **kwargs: Any) -> tuple[str, str, str] | None:
nonlocal identity_calls
identity_calls += 1
return tool_invocation_identity(raw_item, **kwargs)
monkeypatch.setattr(
"agents.run_internal.tool_planning.tool_invocation_identity", counting_identity
)
steps: list[list[TResponseOutputItem]] = [
[
get_text_message("thinking"),
get_function_tool_call("lookup", json.dumps({"value": str(i)}), call_id=f"call_{i}"),
]
for i in range(turns)
]
steps.append([get_text_message("done")])
model = ScriptedModel(steps)
agent = Agent(name="agent", model=model, tools=[lookup])
if mode == "non_streamed":
result = await Runner.run(agent, "hi", max_turns=turns + 1)
else:
result = Runner.run_streamed(agent, "hi", max_turns=turns + 1)
async for _ in result.stream_events():
pass
assert result.final_output == "done"
assert executed == [str(i) for i in range(turns)]
assert len(result.new_items) == 3 * turns + 1
outputs = [
item
for item in cast(list[dict[str, Any]], result.to_input_list())
if item.get("type") == "function_call_output"
]
assert [(item["call_id"], item["output"]) for item in outputs] == [
(f"call_{i}", str(i)) for i in range(turns)
]
model.assert_complete()
# Unique IDs require only the current response's identity, not historical identities.
assert identity_calls <= 2 * turns
def test_canonical_shell_identity_ignores_stripped_provider_metadata() -> None:
provider_call = {
"type": "shell_call",
"call_id": "shell_0",
"action": {"commands": ["echo safe"]},
"created_by": "server",
}
persisted_call = dict(provider_call)
persisted_call.pop("created_by")
assert tool_invocation_identity(provider_call) == tool_invocation_identity(persisted_call)
def test_web_search_source_schema_drift_does_not_warn_during_invocation_lookup() -> None:
source = ActionSearchSource.model_construct(type="api", name="oai-calculator")
output_item = ResponseFunctionWebSearch(
id="ws_123",
action=ActionSearch(type="search", query="current market data", sources=[source]),
status="completed",
type="web_search_call",
)
with warnings.catch_warnings(record=True) as caught_warnings:
warnings.simplefilter("always", UserWarning)
call_id = tool_invocation_call_id(output_item)
serializer_warnings = [
warning
for warning in caught_warnings
if "Pydantic serializer warnings" in str(warning.message)
]
assert call_id is None
assert not serializer_warnings
def test_invocation_lookup_preserves_legacy_model_dump_signature() -> None:
class LegacyModel:
def model_dump(self, *, exclude_none: bool, exclude_unset: bool) -> dict[str, str]:
assert exclude_none is True
assert exclude_unset is True
return {"type": "function_call", "call_id": "call_legacy"}
assert tool_invocation_call_id(LegacyModel()) == ("function_call", "call_legacy")
def test_invocation_lookup_propagates_internal_model_dump_type_error() -> None:
class FailingModel:
def __init__(self) -> None:
self.calls = 0
def model_dump(
self,
*,
exclude_none: bool,
exclude_unset: bool,
warnings: bool = True,
) -> dict[str, str]:
self.calls += 1
raise TypeError("internal serialization failure")
model = FailingModel()
with pytest.raises(TypeError, match="internal serialization failure"):
tool_invocation_call_id(model)
assert model.calls == 1
def test_canonical_shell_identity_treats_optional_nulls_as_omitted() -> None:
provider_call = {
"type": "shell_call",
"call_id": "shell_0",
"action": {
"commands": ["echo safe"],
"max_output_length": None,
"timeout_ms": None,
},
"environment": None,
}
normalized_call = dict(provider_call)
normalized_call["action"] = {"commands": ["echo safe"]}
normalized_call.pop("environment")
assert tool_invocation_identity(provider_call) == tool_invocation_identity(normalized_call)
def test_canonical_function_identity_preserves_null_argument_values() -> None:
call_with_null = {
"type": "function_call",
"call_id": "call_0",
"name": "lookup",
"arguments": '{"value": null}',
}
call_without_value = {
"type": "function_call",
"call_id": "call_0",
"name": "lookup",
"arguments": "{}",
}
assert tool_invocation_identity(call_with_null) != tool_invocation_identity(call_without_value)
@pytest.mark.asyncio
@pytest.mark.parametrize("fallback_type", ["custom", "function"])
async def test_completed_apply_patch_fallback_run_state_round_trip(fallback_type: str) -> None:
class RecordingEditor:
def __init__(self) -> None:
self.paths: list[str] = []
def update_file(self, operation: ApplyPatchOperation) -> ApplyPatchResult:
self.paths.append(operation.path)
return ApplyPatchResult(status="completed", output="updated")
editor = RecordingEditor()
tool = ApplyPatchTool(editor=cast(Any, editor))
operation = {"type": "update_file", "path": "safe.txt", "diff": "-old\n+new\n"}
if fallback_type == "custom":
call: Any = ResponseCustomToolCall(
type="custom_tool_call",
name="apply_patch",
call_id="patch_0",
input=json.dumps(operation),
)
else:
call = ResponseFunctionToolCall(
type="function_call",
name="apply_patch",
call_id="patch_0",
arguments=json.dumps(operation),
)
model = ScriptedModel(steps=[[call]])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[tool])
result = await Runner.run(agent, "update the file")
restored = await RunState.from_json(agent, result.to_state().to_json())
assert result.final_output == "done"
assert editor.paths == ["safe.txt"]
assert restored._context is not None
assert restored._context._tool_invocations["patch_0"].completed is True
@pytest.mark.asyncio
async def test_streamed_function_apply_patch_replay_emits_one_tool_called_event() -> None:
class RecordingEditor:
def __init__(self) -> None:
self.paths: list[str] = []
def update_file(self, operation: ApplyPatchOperation) -> ApplyPatchResult:
self.paths.append(operation.path)
return ApplyPatchResult(status="completed", output="updated")
editor = RecordingEditor()
operation = {"type": "update_file", "path": "safe.txt", "diff": "-old\n+new\n"}
call = ResponseFunctionToolCall(
type="function_call",
name="apply_patch",
call_id="patch_0",
arguments=json.dumps(operation),
)
model = ScriptedModel()
model.extend([[call], [call.model_copy(deep=True)], [get_text_message("done")]])
agent = Agent(
name="agent",
model=model,
tools=[ApplyPatchTool(editor=cast(Any, editor))],
)
streamed = Runner.run_streamed(agent, "update the file")
events = [event async for event in streamed.stream_events()]
tool_called_events = [
event
for event in events
if isinstance(event, RunItemStreamEvent) and event.name == "tool_called"
]
assert streamed.final_output == "done"
assert editor.paths == ["safe.txt"]
assert len(tool_called_events) == 1
assert tool_called_events[0].item.call_id == "patch_0"
@pytest.mark.parametrize(
("raw_item", "first_tool_name", "replacement_tool_name"),
[
(make_shell_call("call_0", commands=["echo safe"]), "safe_shell", "other_shell"),
(make_apply_patch_dict("call_0"), "safe_patch", "other_patch"),
],
ids=["shell", "apply_patch"],
)
def test_native_tool_approval_scope_rejects_resolved_tool_replacement(
raw_item: Any,
first_tool_name: str,
replacement_tool_name: str,
) -> None:
context: RunContextWrapper[Any] = RunContextWrapper(context=None)
approval_item = ToolApprovalItem(
agent=Agent(name="agent"),
raw_item=cast(Any, raw_item),
tool_name=first_tool_name,
)
context.approve_tool(approval_item)
assert (
context._approved_tool_invocation_status( # noqa: SLF001
raw_item,
tool_name=first_tool_name,
)
is not None
)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
context._approved_tool_invocation_status( # noqa: SLF001
raw_item,
tool_name=replacement_tool_name,
)
@pytest.mark.asyncio
async def test_custom_named_shell_sticky_approval_applies_to_fresh_call_ids() -> None:
executed: list[str] = []
def execute(request: Any) -> str:
executed.extend(request.data.action.commands)
return "ok"
first_call = make_shell_call("call_0", commands=["echo first"])
second_call = make_shell_call("call_1", commands=["echo second"])
model = ScriptedModel()
model.extend([[first_call], [second_call], [get_text_message("done")]])
tool = ShellTool(
executor=execute,
name="safe_shell",
needs_approval=True,
)
agent = Agent(name="agent", model=model, tools=[tool])
first = await Runner.run(agent, "run commands")
state = first.to_state()
state.approve(first.interruptions[0], always_approve=True)
resumed = await Runner.run(agent, state)
assert resumed.final_output == "done"
assert executed == ["echo first", "echo second"]
@pytest.mark.asyncio
async def test_custom_named_shell_replacement_rejects_completed_call_id_replay() -> None:
first_executions: list[str] = []
replacement_executions: list[str] = []
def run_first(_request: Any) -> str:
first_executions.append("ran")
return "ok"
def run_replacement(_request: Any) -> str:
replacement_executions.append("ran")
return "ok"
call = make_shell_call("call_0", commands=["echo safe"])
model = ScriptedModel(steps=[[call]])
original_tool = ShellTool(
executor=run_first,
name="safe_shell",
needs_approval=True,
)
agent = Agent(name="agent", model=model, tools=[original_tool])
first = await Runner.run(agent, "run command")
state = first.to_state()
state.approve(first.interruptions[0])
model.enqueue([get_text_message("done")])
completed = await Runner.run(agent, state)
agent.tools = [
ShellTool(
executor=run_replacement,
name="other_shell",
needs_approval=False,
)
]
model.enqueue([cast(Any, dict(cast(dict[str, Any], call)))])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, completed.to_state())
assert first_executions == ["ran"]
assert replacement_executions == []
@pytest.mark.asyncio
async def test_schema_1_13_custom_named_shell_skips_exact_completed_replay() -> None:
executions: list[str] = []
def execute(request: Any) -> str:
executions.append(request.data.call_id)
return "ok"
call = make_shell_call("legacy-shell", commands=["echo safe"])
model = ScriptedModel(steps=[[call]])
model.enqueue([get_text_message("done")])
agent = Agent(
name="agent",
model=model,
tools=[ShellTool(executor=execute, name="safe_shell")],
)
completed = await Runner.run(agent, "run")
serialized = completed.to_state().to_json()
serialized["$schemaVersion"] = "1.13"
serialized["context"].pop("tool_invocations", None)
for key in ("generated_items", "session_items"):
for item in serialized.get(key, []):
item.pop("tool_name", None)
for item in (serialized.get("last_processed_response") or {}).get("new_items", []):
item.pop("tool_name", None)
restored = await RunState.from_json(agent, serialized)
assert restored._context is not None
expected_identity = tool_invocation_identity_and_scope(call, tool_name="safe_shell")
assert expected_identity is not None
restored_record = restored._context._tool_invocations["legacy-shell"]
assert restored_record.approval_scope == expected_identity[2]
assert restored_record.fingerprint == expected_identity[3]
model.extend(
[
[call],
[get_text_message("done again")],
]
)
resumed = await Runner.run(agent, restored)
assert resumed.final_output == "done again"
assert executions == ["legacy-shell"]
@pytest.mark.parametrize(
"raw_item",
[
{"type": "custom_tool_call", "name": "raw_editor", "call_id": "call_0"},
{"type": "computer_call", "call_id": "call_0"},
{"type": "local_shell_call", "call_id": "call_0"},
{"type": "shell_call", "call_id": "call_0"},
{"type": "apply_patch_call", "call_id": "call_0"},
],
)
def test_canonical_identity_requires_each_tool_payload(raw_item: dict[str, Any]) -> None:
assert tool_invocation_identity(raw_item) is None
@pytest.mark.asyncio
async def test_cancelled_rejection_formatter_leaves_invocation_executed() -> None:
tool_call = {
"type": "function_call",
"name": "approval_tool",
"call_id": "call_rejected",
"arguments": "{}",
}
context: RunContextWrapper[Any] = RunContextWrapper(context=None)
assert context._tool_invocation_status(tool_call) == ( # noqa: SLF001
("function_call", "call_rejected"),
False,
False,
)
formatter_entered = asyncio.Event()
async def blocking_formatter(_args: Any) -> str:
formatter_entered.set()
await asyncio.Event().wait()
return "rejected"
task = asyncio.create_task(
resolve_approval_rejection_message(
context_wrapper=context,
run_config=RunConfig(tool_error_formatter=blocking_formatter),
tool_call=tool_call,
tool_type="function",
tool_name="approval_tool",
call_id="call_rejected",
)
)
await formatter_entered.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert context._tool_invocation_status(tool_call) == ( # noqa: SLF001
("function_call", "call_rejected"),
False,
True,
)
async def _run(
agent: Agent[Any],
input_value: Any,
*,
run_config: RunConfig,
mode: Literal["non_streamed", "streamed"],
hooks: RunHooks[Any] | None = None,
events: list[Any] | None = None,
) -> Any:
if mode == "non_streamed":
return await Runner.run(agent, input_value, run_config=run_config, hooks=hooks)
result = Runner.run_streamed(agent, input_value, run_config=run_config, hooks=hooks)
async for event in result.stream_events():
if events is not None:
events.append(event)
return result
def _build_scenario(
second_call_id: str,
second_value: str,
) -> tuple[Agent[Any], RunConfig, list[str]]:
executed: list[str] = []
@function_tool(name_override="record_value", needs_approval=True)
def record_value(value: str) -> str:
executed.append(value)
return value
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
"record_value",
json.dumps({"value": "safe"}),
call_id="call_0",
)
],
[
get_function_tool_call(
"record_value",
json.dumps({"value": second_value}),
call_id=second_call_id,
)
],
[get_text_message("done")],
]
)
provider = _ScriptedProvider(model)
agent = Agent(
name="custom-provider-agent",
model="scripted-provider-model",
tools=[record_value],
)
return agent, RunConfig(model_provider=provider), executed
@pytest.mark.asyncio
async def test_empty_custom_tool_call_id_fails_before_approval_or_execution() -> None:
callbacks: list[str] = []
executed: list[str] = []
async def approve(_context: RunContextWrapper[Any], _item: ToolApprovalItem) -> Any:
callbacks.append("approval")
return {"approve": True}
async def invoke(_context: Any, raw_input: str) -> str:
executed.append(raw_input)
return raw_input
tool = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke,
format={"type": "text"},
needs_approval=True,
on_approval=approve,
)
model = ScriptedModel(
steps=[
[
ResponseCustomToolCall(
type="custom_tool_call",
name=tool.name,
call_id="",
input="changed",
)
]
]
)
agent = Agent(name="agent", model=model, tools=[tool])
with pytest.raises(ModelBehaviorError, match="non-empty string call ID"):
await Runner.run(agent, "run it")
assert callbacks == []
assert executed == []
@pytest.mark.asyncio
async def test_empty_unresolved_function_call_id_fails_before_error_formatter() -> None:
formatter_calls: list[str] = []
def format_tool_error(args: Any) -> str:
formatter_calls.append(args.tool_name)
return "error"
model = ScriptedModel(
steps=[
[
ResponseFunctionToolCall(
id="item_0",
type="function_call",
name="missing",
arguments="{}",
call_id="",
)
]
]
)
agent = Agent(name="agent", model=model)
with pytest.raises(ModelBehaviorError, match="non-empty string call ID"):
await Runner.run(
agent,
"run it",
run_config=RunConfig(
tool_error_formatter=format_tool_error,
tool_not_found_behavior="return_error_to_model",
),
)
assert formatter_calls == []
@pytest.mark.asyncio
async def test_bound_call_id_with_missing_arguments_fails_before_error_formatter() -> None:
executed: list[str] = []
formatter_calls: list[str] = []
@function_tool
def record_value(context: ToolContext[Any], value: str) -> str:
executed.append(value)
context._restored_unbound_approval_call_ids.add("shared")
return value
def format_tool_error(args: Any) -> str:
formatter_calls.append(args.tool_name)
return "error"
valid_call = get_function_tool_call(
"record_value",
json.dumps({"value": "safe"}),
call_id="shared",
)
malformed_replacement = ResponseFunctionToolCall.model_construct(
type="function_call",
name="missing",
call_id="shared",
)
model = ScriptedModel()
model.extend([[valid_call], [malformed_replacement]])
agent = Agent(name="agent", model=model, tools=[record_value])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(
agent,
"run it",
run_config=RunConfig(
tool_error_formatter=format_tool_error,
tool_not_found_behavior="return_error_to_model",
),
)
assert executed == ["safe"]
assert formatter_calls == []
@pytest.mark.asyncio
async def test_empty_handoff_call_id_fails_before_handoff_callback() -> None:
handoff_calls: list[str] = []
target = Agent(name="target")
route = handoff(
target,
tool_name_override="route",
on_handoff=lambda _context: handoff_calls.append("route"),
)
model = ScriptedModel(
steps=[
[
ResponseFunctionToolCall(
id="item_0",
type="function_call",
name="route",
arguments="{}",
call_id="",
)
]
]
)
agent = Agent(name="agent", model=model, handoffs=[route])
with pytest.raises(ModelBehaviorError, match="non-empty string call ID"):
await Runner.run(agent, "route it")
assert handoff_calls == []
@pytest.mark.asyncio
async def test_failed_handoff_hook_does_not_commit_output_or_repeat_callback() -> None:
hook_calls: list[str] = []
target = Agent(name="target")
call = get_handoff_tool_call(target, call_id="handoff_0")
class FailingHooks(RunHooks[Any]):
async def on_handoff(
self,
context: RunContextWrapper[Any],
from_agent: Agent[Any],
to_agent: Agent[Any],
) -> None:
hook_calls.append(to_agent.name)
raise RuntimeError("handoff hook failed")
model = ScriptedModel(steps=[[call]])
model.enqueue([call.model_copy(deep=True)])
agent = Agent(name="source", model=model, handoffs=[target])
context = RunContextWrapper(context=None)
hooks = FailingHooks()
with pytest.raises(RuntimeError, match="handoff hook failed"):
await Runner.run(agent, "handoff", context=context, hooks=hooks)
assert context._tool_invocation_status(call, invocation_role="handoff") == ( # noqa: SLF001
("function_call", "handoff_0"),
False,
True,
)
with pytest.raises(ModelBehaviorError, match="already executed"):
await Runner.run(agent, "handoff", context=context, hooks=hooks)
assert hook_calls == ["target"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_changed_arguments_under_reused_call_id_fail_before_second_side_effect(
mode: Literal["non_streamed", "streamed"],
) -> None:
agent, run_config, executed = _build_scenario("call_0", "changed")
first = await _run(agent, "record a value", run_config=run_config, mode=mode)
assert len(first.interruptions) == 1
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await _run(agent, state, run_config=run_config, mode=mode)
assert executed == ["safe"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_exact_replay_reuses_committed_output_without_executing_again(
mode: Literal["non_streamed", "streamed"],
) -> None:
agent, run_config, executed = _build_scenario("call_0", "safe")
first = await _run(agent, "record a value", run_config=run_config, mode=mode)
state = first.to_state()
state.approve(first.interruptions[0])
resumed = await _run(agent, state, run_config=run_config, mode=mode)
assert resumed.final_output == "done"
assert resumed.interruptions == []
assert executed == ["safe"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_non_approval_identical_siblings_execute_once(
mode: Literal["non_streamed", "streamed"],
) -> None:
executed: list[str] = []
events: list[Any] = []
@function_tool
async def record_value(value: str) -> str:
executed.append(value)
return value
duplicate = get_function_tool_call(
"record_value",
'{"value":"safe"}',
call_id="call_0",
)
model = ScriptedModel(steps=[[duplicate, duplicate.model_copy(deep=True)]])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[record_value])
result = await _run(
agent,
"record a value",
run_config=RunConfig(),
mode=mode,
events=events,
)
assert result.final_output == "done"
assert executed == ["safe"]
if mode == "streamed":
assert [
event.name for event in events if getattr(event, "name", None) == "tool_called"
] == ["tool_called"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_non_approval_completed_replay_does_not_execute_again(
mode: Literal["non_streamed", "streamed"],
) -> None:
executed: list[str] = []
@function_tool
async def record_value(value: str) -> str:
executed.append(value)
return value
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[get_function_tool_call("record_value", '{ "value" : "safe" }', call_id="call_0")],
[get_text_message("done")],
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
result = await _run(agent, "record a value", run_config=RunConfig(), mode=mode)
assert result.final_output == "done"
assert executed == ["safe"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_non_approval_changed_completed_call_id_fails_before_execution(
mode: Literal["non_streamed", "streamed"],
) -> None:
executed: list[str] = []
events: list[Any] = []
class RecordingHooks(RunHooks[Any]):
def __init__(self) -> None:
self.llm_end_calls = 0
async def on_llm_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
response: ModelResponse,
) -> None:
self.llm_end_calls += 1
hooks = RecordingHooks()
@function_tool
async def record_value(value: str) -> str:
executed.append(value)
return value
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[get_function_tool_call("record_value", '{"value":"changed"}', call_id="call_0")],
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await _run(
agent,
"record a value",
run_config=RunConfig(),
mode=mode,
hooks=hooks,
events=events,
)
assert executed == ["safe"]
assert hooks.llm_end_calls == 1
if mode == "streamed":
assert [
event.name for event in events if getattr(event, "name", None) == "tool_called"
] == ["tool_called"]
@pytest.mark.asyncio
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "serialized"])
async def test_nested_agent_tool_approval_allows_outer_call_id_collision(
round_trip: bool,
) -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
async def inner_sensitive_tool(value: str) -> str:
executed.append(value)
return value
inner_model = ScriptedModel(
steps=[
[
get_function_tool_call(
"inner_sensitive_tool",
'{"value":"safe"}',
call_id="shared",
)
]
]
)
inner_model.enqueue([get_text_message("inner done")])
inner_agent = Agent(name="inner", model=inner_model, tools=[inner_sensitive_tool])
outer_model = ScriptedModel(
steps=[
[
get_function_tool_call(
"nested_agent",
'{"input":"hello"}',
call_id="shared",
)
]
]
)
outer_model.enqueue([get_text_message("outer done")])
outer_agent = Agent(
name="outer",
model=outer_model,
tools=[
inner_agent.as_tool(
tool_name="nested_agent",
tool_description="Run the nested agent.",
)
],
)
interrupted = await Runner.run(outer_agent, "start")
assert len(interrupted.interruptions) == 1
state = interrupted.to_state()
if round_trip:
state = await RunState.from_json(outer_agent, state.to_json())
state.approve(state.get_interruptions()[0])
resumed = await Runner.run(outer_agent, state)
assert resumed.final_output == "outer done"
assert resumed.interruptions == []
assert executed == ["safe"]
@pytest.mark.asyncio
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "serialized"])
async def test_parent_approval_does_not_authorize_independent_nested_run(
round_trip: bool,
) -> None:
executed: list[str] = []
@function_tool(name_override="sensitive", needs_approval=True)
async def sensitive(value: str) -> str:
executed.append(value)
return value
inner_model = ScriptedModel()
inner_model.extend(
[
[get_function_tool_call("sensitive", '{"value":"same"}', call_id="shared")],
[get_text_message("inner done")],
]
)
inner_agent = Agent(name="inner", model=inner_model, tools=[sensitive])
outer_model = ScriptedModel()
outer_model.extend(
[
[get_function_tool_call("sensitive", '{"value":"same"}', call_id="shared")],
[
get_function_tool_call(
"nested_agent",
'{"input":"hello"}',
call_id="outer-nested",
)
],
[get_text_message("outer done")],
]
)
outer_agent = Agent(
name="outer",
model=outer_model,
tools=[
sensitive,
inner_agent.as_tool(
tool_name="nested_agent",
tool_description="Run the nested agent.",
),
],
)
parent_interruption = await Runner.run(outer_agent, "start")
parent_state = parent_interruption.to_state()
parent_state.approve(parent_state.get_interruptions()[0])
nested_interruption = await Runner.run(outer_agent, parent_state)
assert len(nested_interruption.interruptions) == 1
assert executed == ["same"]
nested_state = nested_interruption.to_state()
if round_trip:
nested_state = await RunState.from_json(outer_agent, nested_state.to_json())
still_pending = await Runner.run(outer_agent, nested_state)
assert len(still_pending.interruptions) == 1
assert executed == ["same"]
approved_state = still_pending.to_state()
approved_state.approve(approved_state.get_interruptions()[0])
completed = await Runner.run(outer_agent, approved_state)
assert completed.final_output == "outer done"
assert completed.interruptions == []
assert executed == ["same", "same"]
@pytest.mark.asyncio
async def test_streamed_validated_items_precede_llm_end_hook_failure() -> None:
executed: list[str] = []
events: list[Any] = []
class FailingHooks(RunHooks[Any]):
async def on_llm_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
response: ModelResponse,
) -> None:
_ = (context, agent, response)
raise RuntimeError("llm end failed")
@function_tool
async def record_value(value: str) -> str:
executed.append(value)
return value
model = ScriptedModel(
steps=[
[
get_function_tool_call(
"record_value",
'{"value":"safe"}',
call_id="call_0",
)
]
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
result = Runner.run_streamed(agent, "record a value", hooks=FailingHooks())
with pytest.raises(RuntimeError, match="llm end failed"):
async for event in result.stream_events():
events.append(event)
assert [event.name for event in events if isinstance(event, RunItemStreamEvent)] == [
"tool_called"
]
assert executed == []
@pytest.mark.asyncio
async def test_non_approval_failed_tool_body_does_not_reexecute() -> None:
attempts: list[str] = []
@function_tool(failure_error_function=None)
async def perform_side_effect() -> str:
attempts.append("ran")
raise RuntimeError("failed after side effect")
call = get_function_tool_call("perform_side_effect", "{}", call_id="call_0")
model = ScriptedModel()
model.extend([[call], [call.model_copy(deep=True)]])
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
context = RunContextWrapper(context=None)
with pytest.raises(UserError, match="failed after side effect"):
await Runner.run(agent, "run it", context=context)
with pytest.raises(ModelBehaviorError, match="already executed"):
await Runner.run(agent, "run it", context=context)
assert attempts == ["ran"]
@pytest.mark.asyncio
async def test_non_approval_custom_tool_identical_siblings_execute_once() -> None:
executed: list[str] = []
async def invoke(_context: Any, value: str) -> str:
executed.append(value)
return value
tool = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke,
format={"type": "text"},
)
duplicate = ResponseCustomToolCall(
type="custom_tool_call",
name=tool.name,
call_id="call_0",
input="safe",
)
model = ScriptedModel(steps=[[duplicate, duplicate.model_copy(deep=True)]])
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[tool])
result = await Runner.run(agent, "edit text")
assert result.final_output == "done"
assert executed == ["safe"]
@pytest.mark.asyncio
async def test_approved_custom_tool_resume_rebinds_to_current_tool() -> None:
executed: list[str] = []
async def invoke_original(_context: Any, value: str) -> str:
executed.append(f"original:{value}")
return value
async def invoke_replacement(_context: Any, value: str) -> str:
executed.append(f"replacement:{value}")
return value
original = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke_original,
format={"type": "text"},
needs_approval=True,
)
replacement = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke_replacement,
format={"type": "text"},
needs_approval=True,
)
call = ResponseCustomToolCall(
type="custom_tool_call",
name="raw_editor",
call_id="call_0",
input="safe",
)
model = ScriptedModel(steps=[[call], [get_text_message("done")]])
agent = Agent(name="agent", model=model, tools=[original])
first = await Runner.run(agent, "edit text")
state = first.to_state()
state.approve(state.get_interruptions()[0])
agent.tools = [replacement]
resumed = await Runner.run(agent, state)
assert resumed.final_output == "done"
assert executed == ["replacement:safe"]
@pytest.mark.asyncio
async def test_approved_custom_tool_resume_fails_when_current_tool_is_missing() -> None:
executed: list[str] = []
async def invoke(_context: Any, value: str) -> str:
executed.append(value)
return value
tool = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke,
format={"type": "text"},
needs_approval=True,
)
call = ResponseCustomToolCall(
type="custom_tool_call",
name="raw_editor",
call_id="call_0",
input="safe",
)
model = ScriptedModel(steps=[[call]])
agent = Agent(name="agent", model=model, tools=[tool])
first = await Runner.run(agent, "edit text")
state = first.to_state()
state.approve(state.get_interruptions()[0])
agent.tools = []
with pytest.raises(ModelBehaviorError, match="Tool raw_editor not found"):
await Runner.run(agent, state)
assert executed == []
@pytest.mark.asyncio
async def test_approved_apply_patch_custom_resume_uses_public_tool() -> None:
class RecordingEditor:
def __init__(self) -> None:
self.paths: list[str] = []
def update_file(self, operation: ApplyPatchOperation) -> ApplyPatchResult:
self.paths.append(operation.path)
return ApplyPatchResult(status="completed", output="updated")
editor = RecordingEditor()
tool = ApplyPatchTool(editor=cast(Any, editor), needs_approval=True)
call = ResponseCustomToolCall(
type="custom_tool_call",
name="apply_patch",
call_id="patch_0",
input=json.dumps(
{
"type": "update_file",
"path": "safe.txt",
"diff": "-old\n+new\n",
}
),
)
model = ScriptedModel(steps=[[call], [get_text_message("done")]])
agent = Agent(name="agent", model=model, tools=[tool])
first = await Runner.run(agent, "update the file")
state = first.to_state()
state.approve(state.get_interruptions()[0])
resumed = await Runner.run(agent, state)
assert resumed.final_output == "done"
assert editor.paths == ["safe.txt"]
@pytest.mark.asyncio
async def test_pending_custom_tool_resume_survives_missing_current_tool() -> None:
async def invoke(_context: Any, value: str) -> str:
raise AssertionError(f"pending custom tool must not execute: {value}")
tool = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke,
format={"type": "text"},
needs_approval=True,
)
call = ResponseCustomToolCall(
type="custom_tool_call",
name="raw_editor",
call_id="call_0",
input="safe",
)
agent = Agent(
name="agent",
model=ScriptedModel(steps=[[call]]),
tools=[tool],
)
first = await Runner.run(agent, "edit")
state = first.to_state()
agent.tools = []
resumed = await Runner.run(agent, state)
assert len(resumed.interruptions) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("serialize_state", [False, True], ids=["in-memory", "json"])
async def test_rejected_custom_tool_resume_emits_output_when_current_tool_is_missing(
serialize_state: bool,
) -> None:
executed: list[str] = []
async def invoke(_context: Any, value: str) -> str:
executed.append(value)
return value
tool = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke,
format={"type": "text"},
needs_approval=True,
)
call = ResponseCustomToolCall(
type="custom_tool_call",
name="raw_editor",
call_id="call_0",
input="safe",
)
model = ScriptedModel(steps=[[call], [get_text_message("done")]])
agent = Agent(name="agent", model=model, tools=[tool])
first = await Runner.run(agent, "edit text")
state = first.to_state()
state.reject(state.get_interruptions()[0])
agent.tools = []
if serialize_state:
state = await RunState.from_json(agent, state.to_json())
resumed = await Runner.run(agent, state)
assert resumed.final_output == "done"
assert executed == []
model_input = model.calls[-1].input
assert isinstance(model_input, list)
rejection_outputs = [
item
for item in model_input
if isinstance(item, dict)
and item.get("type") == "custom_tool_call_output"
and item.get("call_id") == "call_0"
]
assert len(rejection_outputs) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_changed_same_id_siblings_fail_before_approval_callback(
mode: Literal["non_streamed", "streamed"],
) -> None:
approval_calls: list[str] = []
async def needs_approval(_context: Any, arguments: dict[str, Any], _call_id: str) -> bool:
approval_calls.append(arguments["value"])
return True
@function_tool(needs_approval=needs_approval)
async def record_value(value: str) -> str:
return value
model = ScriptedModel(
steps=[
[
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
]
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
with pytest.raises(ModelBehaviorError, match="one response"):
await _run(agent, "record values", run_config=RunConfig(), mode=mode)
assert approval_calls == []
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_response_processing_error_still_invokes_llm_end(
mode: Literal["non_streamed", "streamed"],
) -> None:
class CountingHooks(RunHooks[Any]):
def __init__(self) -> None:
self.llm_end_calls = 0
async def on_llm_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
response: ModelResponse,
) -> None:
_ = (context, agent, response)
self.llm_end_calls += 1
hooks = CountingHooks()
output: list[TResponseOutputItem] = [make_shell_call("call_0", commands=["echo safe"])]
step = get_exact_output_stream_step(output) if mode == "streamed" else output
model = ScriptedModel(steps=[step])
agent = Agent(name="agent", model=model)
with pytest.raises(ModelBehaviorError, match="without a shell tool"):
await _run(agent, "run command", run_config=RunConfig(), mode=mode, hooks=hooks)
assert hooks.llm_end_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_processing_error_with_changed_call_id_still_suppresses_llm_end(
mode: Literal["non_streamed", "streamed"],
) -> None:
class CountingHooks(RunHooks[Any]):
def __init__(self) -> None:
self.llm_end_calls = 0
async def on_llm_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
response: ModelResponse,
) -> None:
_ = (context, agent, response)
self.llm_end_calls += 1
@function_tool
async def record_value(value: str) -> str:
return value
outputs: list[list[TResponseOutputItem]] = [
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="shared")],
[
get_function_tool_call(
"record_value",
'{"value":"changed"}',
call_id="shared",
),
make_shell_call("shell_0", commands=["echo safe"]),
],
]
model = ScriptedModel(
[get_exact_output_stream_step(output) for output in outputs]
if mode == "streamed"
else outputs
)
hooks = CountingHooks()
agent = Agent(name="agent", model=model, tools=[record_value])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await _run(agent, "record value", run_config=RunConfig(), mode=mode, hooks=hooks)
assert hooks.llm_end_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_processing_error_with_repeated_uncanonical_id_suppresses_llm_end(
mode: Literal["non_streamed", "streamed"],
) -> None:
class CountingHooks(RunHooks[Any]):
def __init__(self) -> None:
self.llm_end_calls = 0
async def on_llm_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
response: ModelResponse,
) -> None:
_ = (context, agent, response)
self.llm_end_calls += 1
first_call = ResponseCustomToolCall.model_construct(
type="custom_tool_call",
name="first_missing_tool",
call_id="shared",
)
second_call = ResponseCustomToolCall.model_construct(
type="custom_tool_call",
name="second_missing_tool",
call_id="shared",
)
hooks = CountingHooks()
output: list[TResponseOutputItem] = [first_call, second_call]
step = get_exact_output_stream_step(output) if mode == "streamed" else output
model = ScriptedModel(steps=[step])
agent = Agent(name="agent", model=model)
with pytest.raises(ModelBehaviorError, match="one response"):
await _run(agent, "run tool", run_config=RunConfig(), mode=mode, hooks=hooks)
assert hooks.llm_end_calls == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_changed_same_id_siblings_fail_before_non_approval_execution(
mode: Literal["non_streamed", "streamed"],
) -> None:
executed: list[str] = []
@function_tool
async def record_value(value: str) -> str:
executed.append(value)
return value
model = ScriptedModel(
steps=[
[
get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"),
get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"),
]
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
with pytest.raises(ModelBehaviorError, match="one response"):
await _run(agent, "record values", run_config=RunConfig(), mode=mode)
assert executed == []
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_changed_computer_safety_checks_fail_before_same_response_effects(
mode: Literal["non_streamed", "streamed"],
) -> None:
safety_checks: list[str] = []
screenshots: list[str] = []
class RecordingComputer(FakeComputer):
def screenshot(self) -> str:
screenshots.append("screenshot")
return "img"
def acknowledge_safety_check(data: Any) -> bool:
safety_checks.append(data.safety_check.id)
return True
tool = ComputerTool(
computer=RecordingComputer(),
on_safety_check=acknowledge_safety_check,
)
first_call = ResponseComputerToolCall(
id="computer-item-1",
type="computer_call",
action=ActionScreenshot(type="screenshot"),
call_id="computer-call",
pending_safety_checks=[PendingSafetyCheck(id="safety-1", code="code-1", message="first")],
status="completed",
)
changed_call = first_call.model_copy(
update={
"id": "computer-item-2",
"pending_safety_checks": [
PendingSafetyCheck(id="safety-2", code="code-2", message="changed")
],
}
)
output: list[TResponseOutputItem] = [first_call, changed_call]
step = get_exact_output_stream_step(output) if mode == "streamed" else output
agent = Agent(
name="computer-agent",
model=ScriptedModel(steps=[step]),
tools=[tool],
)
with pytest.raises(ModelBehaviorError, match="one response"):
await _run(agent, "use computer", run_config=RunConfig(), mode=mode)
assert safety_checks == []
assert screenshots == []
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_changed_computer_safety_checks_fail_before_completed_replay_effects(
mode: Literal["non_streamed", "streamed"],
) -> None:
safety_checks: list[str] = []
screenshots: list[str] = []
class RecordingComputer(FakeComputer):
def screenshot(self) -> str:
screenshots.append("screenshot")
return "img"
def acknowledge_safety_check(data: Any) -> bool:
safety_checks.append(data.safety_check.id)
return True
tool = ComputerTool(
computer=RecordingComputer(),
on_safety_check=acknowledge_safety_check,
)
first_call = ResponseComputerToolCall(
id="computer-item-1",
type="computer_call",
action=ActionScreenshot(type="screenshot"),
call_id="computer-call",
pending_safety_checks=[PendingSafetyCheck(id="safety-1", code="code-1", message="first")],
status="completed",
)
changed_call = first_call.model_copy(
update={
"id": "computer-item-2",
"pending_safety_checks": [
PendingSafetyCheck(id="safety-2", code="code-2", message="changed")
],
}
)
outputs: list[list[TResponseOutputItem]] = [[first_call], [changed_call]]
model = ScriptedModel(
[get_exact_output_stream_step(output) for output in outputs]
if mode == "streamed"
else outputs
)
agent = Agent(name="computer-agent", model=model, tools=[tool])
with pytest.raises(ModelBehaviorError, match="completed tool call ID"):
await _run(agent, "use computer", run_config=RunConfig(), mode=mode)
assert safety_checks == ["safety-1"]
assert screenshots == ["screenshot"]
@pytest.mark.asyncio
async def test_computer_hook_failure_does_not_repeat_side_effect() -> None:
screenshots: list[str] = []
class RecordingComputer(FakeComputer):
def screenshot(self) -> str:
screenshots.append("screenshot")
return "img"
class FailOnceHooks(RunHooks[Any]):
def __init__(self) -> None:
self.failed = False
async def on_tool_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
tool: Tool,
result: object,
) -> None:
if not self.failed:
self.failed = True
raise RuntimeError("end hook failed")
tool = ComputerTool(computer=RecordingComputer())
call = ResponseComputerToolCall(
id="computer-item",
type="computer_call",
action=ActionScreenshot(type="screenshot"),
call_id="computer-call",
pending_safety_checks=[],
status="completed",
)
model = ScriptedModel()
model.extend([[call], [call.model_copy(deep=True)]])
agent = Agent(name="computer-agent", model=model, tools=[tool])
context = RunContextWrapper(context=None)
hooks = FailOnceHooks()
with pytest.raises(RuntimeError, match="end hook failed"):
await Runner.run(agent, "use computer", context=context, hooks=hooks)
with pytest.raises(ModelBehaviorError, match="already executed"):
await Runner.run(agent, "use computer", context=context, hooks=hooks)
assert screenshots == ["screenshot"]
@pytest.mark.asyncio
async def test_exact_replay_drops_tied_reasoning_item() -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
async def record_value(value: str) -> str:
executed.append(value)
return value
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[
ResponseReasoningItem(id="rs_replay", summary=[], type="reasoning"),
get_function_tool_call(
"record_value",
'{ "value" : "safe" }',
call_id="call_0",
),
],
[get_text_message("done")],
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
first = await Runner.run(agent, "record a value")
state = first.to_state()
state.approve(first.interruptions[0])
resumed = await Runner.run(agent, state)
assert resumed.final_output == "done"
assert executed == ["safe"]
model_input = model.calls[-1].input
assert isinstance(model_input, list)
assert not any(item.get("id") == "rs_replay" for item in model_input)
assert (
sum(
item.get("type") == "function_call" and item.get("call_id") == "call_0"
for item in model_input
)
== 1
)
@pytest.mark.asyncio
async def test_streamed_exact_replay_does_not_emit_tied_reasoning_item() -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
async def record_value(value: str) -> str:
executed.append(value)
return value
model = ScriptedModel()
model.extend(
[
[get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")],
[
ResponseReasoningItem(id="rs_replay", summary=[], type="reasoning"),
get_function_tool_call(
"record_value",
'{ "value" : "safe" }',
call_id="call_0",
),
],
[get_text_message("done")],
]
)
agent = Agent(name="agent", model=model, tools=[record_value])
first = await Runner.run(agent, "record a value")
state = first.to_state()
state.approve(first.interruptions[0])
resumed = Runner.run_streamed(agent, state)
item_events = [
event async for event in resumed.stream_events() if isinstance(event, RunItemStreamEvent)
]
assert resumed.final_output == "done"
assert executed == ["safe"]
assert not any(
event.name == "reasoning_item_created"
and getattr(event.item.raw_item, "id", None) == "rs_replay"
for event in item_events
)
assert not any(event.name == "tool_called" for event in item_events)
@pytest.mark.asyncio
async def test_failed_tool_end_hook_does_not_reexecute_approved_call() -> None:
agent, run_config, executed = _build_scenario("call_0", "safe")
class FailOnceHooks(RunHooks[Any]):
def __init__(self) -> None:
self.failed = False
async def on_tool_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
tool: Tool,
result: object,
) -> None:
if not self.failed:
self.failed = True
raise RuntimeError("end hook failed")
hooks = FailOnceHooks()
first = await Runner.run(agent, "record a value", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(UserError, match="end hook failed"):
await Runner.run(agent, state, run_config=run_config, hooks=hooks)
resumed = await Runner.run(agent, state, run_config=run_config, hooks=hooks)
assert resumed.final_output == "done"
assert executed == ["safe"]
@pytest.mark.asyncio
async def test_failed_output_guardrail_does_not_reexecute_approved_call() -> None:
executed: list[str] = []
@tool_output_guardrail
async def reject_output(_data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.raise_exception(output_info="blocked")
@function_tool(needs_approval=True, tool_output_guardrails=[reject_output])
async def record_value() -> str:
executed.append("ran")
return "sensitive"
model = ScriptedModel(steps=[[get_function_tool_call("record_value", "{}", call_id="call_0")]])
agent = Agent(name="agent", model=model, tools=[record_value])
first = await Runner.run(agent, "record a value")
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(ToolOutputGuardrailTripwireTriggered):
await Runner.run(agent, state)
restored = await RunState.from_json(agent, state.to_json())
with pytest.raises(ModelBehaviorError, match="already executed"):
await Runner.run(agent, restored)
assert executed == ["ran"]
@pytest.mark.asyncio
async def test_failed_approved_tool_body_does_not_reexecute() -> None:
attempts: list[str] = []
@function_tool(needs_approval=True, failure_error_function=None)
async def perform_side_effect() -> str:
attempts.append("ran")
raise RuntimeError("failed after side effect")
model = ScriptedModel(
steps=[[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]]
)
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
first = await Runner.run(agent, "run it")
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(UserError, match="failed after side effect"):
await Runner.run(agent, state)
with pytest.raises(ModelBehaviorError, match="already executed"):
await Runner.run(agent, state)
assert attempts == ["ran"]
@pytest.mark.asyncio
async def test_cancelled_approved_tool_body_does_not_reexecute() -> None:
attempts: list[str] = []
started = asyncio.Event()
keep_running = asyncio.Event()
@function_tool(needs_approval=True, failure_error_function=None)
async def perform_side_effect() -> str:
attempts.append("ran")
started.set()
await keep_running.wait()
return "done"
model = ScriptedModel(
steps=[[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")]]
)
agent = Agent(name="agent", model=model, tools=[perform_side_effect])
first = await Runner.run(agent, "run it")
state = first.to_state()
state.approve(first.interruptions[0])
resume_task = asyncio.create_task(Runner.run(agent, state))
await started.wait()
resume_task.cancel()
with pytest.raises(asyncio.CancelledError):
await resume_task
with pytest.raises(ModelBehaviorError, match="already executed"):
await Runner.run(agent, state)
assert attempts == ["ran"]
@pytest.mark.asyncio
async def test_failed_approved_agent_tool_start_does_not_reexecute() -> None:
hook_calls: list[str] = []
inner_agent = Agent(
name="inner",
model=ScriptedModel(steps=[[get_text_message("inner done")]]),
)
agent_tool = inner_agent.as_tool(
tool_name="delegate",
tool_description="Delegate work.",
needs_approval=True,
)
class FailingHooks(RunHooks[Any]):
async def on_tool_start(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
tool: Tool,
) -> None:
if tool is agent_tool:
hook_calls.append("ran")
raise RuntimeError("failed after side effect")
outer_model = ScriptedModel(
steps=[[get_function_tool_call("delegate", '{"input":"hi"}', call_id="call_0")]]
)
outer_agent = Agent(name="outer", model=outer_model, tools=[agent_tool])
first = await Runner.run(outer_agent, "delegate")
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(UserError, match="failed after side effect"):
await Runner.run(outer_agent, state, hooks=FailingHooks())
with pytest.raises(ModelBehaviorError, match="already executed"):
await Runner.run(outer_agent, state, hooks=FailingHooks())
assert hook_calls == ["ran"]
@pytest.mark.asyncio
async def test_failed_parallel_tool_end_hook_checkpoints_outputs_in_model_order() -> None:
executed: list[str] = []
first_finished = asyncio.Event()
@function_tool(needs_approval=True)
async def first_tool() -> str:
await asyncio.sleep(0.01)
executed.append("first")
first_finished.set()
return "first"
@function_tool(needs_approval=True)
async def second_tool() -> str:
executed.append("second")
return "second"
class FailSecondHookOnce(RunHooks[Any]):
def __init__(self) -> None:
self.failed = False
async def on_tool_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
tool: Tool,
result: object,
) -> None:
if tool.name == "second_tool" and not self.failed:
await first_finished.wait()
self.failed = True
raise RuntimeError("second end hook failed")
model = ScriptedModel(
steps=[
[
get_function_tool_call("first_tool", "{}", call_id="call_first"),
get_function_tool_call("second_tool", "{}", call_id="call_second"),
]
]
)
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[first_tool, second_tool])
hooks = FailSecondHookOnce()
first = await Runner.run(agent, "run tools")
state = first.to_state()
for interruption in first.interruptions:
state.approve(interruption)
with pytest.raises(UserError, match="second end hook failed"):
await Runner.run(agent, state, hooks=hooks)
resumed = await Runner.run(agent, state, hooks=hooks)
assert resumed.final_output == "done"
assert sorted(executed) == ["first", "second"]
model_input = model.calls[-1].input
assert isinstance(model_input, list)
output_call_ids = [
item["call_id"]
for item in model_input
if isinstance(item, dict) and item.get("type") == "function_call_output"
]
assert output_call_ids == ["call_first", "call_second"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_identical_approval_bound_siblings_execute_once(
mode: Literal["non_streamed", "streamed"],
) -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
def record_value(value: str) -> str:
executed.append(value)
return value
duplicated_call = get_function_tool_call(
"record_value",
json.dumps({"value": "safe"}),
call_id="call-duplicate",
)
model = ScriptedModel()
model.extend(
[[duplicated_call, duplicated_call.model_copy(deep=True)], [get_text_message("done")]]
)
run_config = RunConfig(model_provider=_ScriptedProvider(model))
agent = Agent(
name="custom-provider-agent",
model="scripted-provider-model",
tools=[record_value],
)
first = await _run(agent, "record a value", run_config=run_config, mode=mode)
assert len(first.interruptions) == 1
state = first.to_state()
state.approve(first.interruptions[0])
resumed = await _run(agent, state, run_config=run_config, mode=mode)
assert resumed.final_output == "done"
assert executed == ["safe"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_new_call_id_requires_a_new_per_call_approval(
mode: Literal["non_streamed", "streamed"],
) -> None:
agent, run_config, executed = _build_scenario("call_1", "changed")
first = await _run(agent, "record a value", run_config=run_config, mode=mode)
state = first.to_state()
state.approve(first.interruptions[0])
resumed = await _run(agent, state, run_config=run_config, mode=mode)
assert len(resumed.interruptions) == 1
assert resumed.interruptions[0].arguments == json.dumps({"value": "changed"})
assert executed == ["safe"]
@pytest.mark.asyncio
@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"])
async def test_serialized_sticky_decision_rejects_changed_reused_call_id(
approve: bool,
) -> None:
agent, run_config, executed = _build_scenario("call_0", "changed")
first = await Runner.run(agent, "record a value", run_config=run_config)
state = first.to_state()
if approve:
state.approve(first.interruptions[0], always_approve=True)
else:
state.reject(first.interruptions[0], always_reject=True)
restored_state = await RunState.from_json(agent, state.to_json())
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, restored_state, run_config=run_config)
assert executed == (["safe"] if approve else [])
@pytest.mark.asyncio
async def test_sticky_upgrade_rejects_changed_reuse_of_prior_call_id() -> None:
agent, run_config, executed = _build_scenario("call_0", "changed")
first = await Runner.run(agent, "record a value", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
state.approve(first.interruptions[0], always_approve=True)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, state, run_config=run_config)
assert executed == ["safe"]
@pytest.mark.asyncio
async def test_serialized_sticky_identical_siblings_execute_once() -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
async def record_value(value: int) -> str:
executed.append(str(value))
return str(value)
duplicate = get_function_tool_call(
"record_value",
'{"value":1}',
call_id="call_0",
)
model = ScriptedModel(
steps=[
[
duplicate,
duplicate.model_copy(deep=True),
]
]
)
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[record_value])
first = await Runner.run(agent, "record a value")
state = first.to_state()
state.approve(first.interruptions[0], always_approve=True)
restored = await RunState.from_json(agent, state.to_json())
resumed = await Runner.run(agent, restored)
assert resumed.final_output == "done"
assert executed == ["1"]
@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"])
def test_sticky_decision_preserves_prior_per_call_binding(approve: bool) -> None:
agent = Agent(name="agent")
context = RunContextWrapper(context=None)
first = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_a", '{"value":1}', call_id="call_0"),
),
tool_name="tool_a",
)
sticky = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_a", '{"value":2}', call_id="call_1"),
),
tool_name="tool_a",
)
changed = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_a", '{"value":3}', call_id="call_0"),
),
tool_name="tool_a",
)
context.approve_tool(first)
if approve:
context.approve_tool(sticky, always_approve=True)
else:
context.reject_tool(sticky, always_reject=True)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
context.get_approval_status(
"tool_a",
"call_0",
current_invocation=changed,
)
@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"])
def test_sticky_decision_does_not_disable_other_tool_binding(approve: bool) -> None:
agent = Agent(name="agent")
context = RunContextWrapper(context=None)
other = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_b", '{"value":1}', call_id="call_0"),
),
tool_name="tool_b",
)
sticky = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_a", '{"value":2}', call_id="call_1"),
),
tool_name="tool_a",
)
changed_other = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_b", '{"value":3}', call_id="call_0"),
),
tool_name="tool_b",
)
context.approve_tool(other)
if approve:
context.approve_tool(sticky, always_approve=True)
else:
context.reject_tool(sticky, always_reject=True)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
context.get_approval_status(
"tool_b",
"call_0",
current_invocation=changed_other,
)
@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"])
def test_matching_sticky_decision_does_not_mask_other_tool_binding(approve: bool) -> None:
agent = Agent(name="agent")
context = RunContextWrapper(context=None)
other = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_b", '{"value":1}', call_id="call_0"),
),
tool_name="tool_b",
)
sticky = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_a", '{"value":2}', call_id="call_1"),
),
tool_name="tool_a",
)
current = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("tool_a", '{"value":3}', call_id="call_0"),
),
tool_name="tool_a",
)
context.approve_tool(other)
if approve:
context.approve_tool(sticky, always_approve=True)
else:
context.reject_tool(sticky, always_reject=True)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
context.get_approval_status(
"tool_a",
"call_0",
current_invocation=current,
)
@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"])
def test_deferred_sticky_decision_preserves_mirrored_per_call_binding(approve: bool) -> None:
agent = Agent(name="agent")
context = RunContextWrapper(context=None)
def approval_item(call_id: str, value: int) -> ToolApprovalItem:
raw_item = cast(
ResponseFunctionToolCall,
get_function_tool_call(
"lookup",
json.dumps({"value": value}),
call_id=call_id,
),
)
return ToolApprovalItem(
agent=agent,
raw_item=raw_item,
tool_name="lookup",
tool_namespace="lookup",
tool_lookup_key=("deferred_top_level", "lookup"),
_allow_bare_name_alias=True,
)
first = approval_item("call_0", 1)
sticky = approval_item("call_1", 2)
changed = approval_item("call_0", 3)
context.approve_tool(first)
if approve:
context.approve_tool(sticky, always_approve=True)
else:
context.reject_tool(sticky, always_reject=True)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
context.get_approval_status(
"lookup",
"call_0",
tool_namespace="lookup",
tool_lookup_key=("deferred_top_level", "lookup"),
current_invocation=changed,
)
def test_sticky_function_approval_rejects_same_id_shell_call() -> None:
agent = Agent(name="agent")
context = RunContextWrapper(context=None)
approval_item = ToolApprovalItem(
agent=agent,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call("shell", "{}", call_id="call_0"),
),
tool_name="shell",
)
context.approve_tool(approval_item, always_approve=True)
shell_item = ToolApprovalItem(
agent=agent,
raw_item={
"type": "shell_call",
"call_id": "call_0",
"action": {"commands": ["echo safe"]},
},
tool_name="shell",
)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
context.get_approval_status(
"shell",
"call_0",
current_invocation=shell_item,
)
@pytest.mark.asyncio
async def test_changed_tool_under_approved_call_id_fails_before_second_tool_starts() -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
def first_tool(value: str) -> str:
executed.append(f"first:{value}")
return value
@function_tool(needs_approval=True)
def second_tool(value: str) -> str:
executed.append(f"second:{value}")
return value
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
"first_tool",
json.dumps({"value": "same"}),
call_id="call_0",
)
],
[
get_function_tool_call(
"second_tool",
json.dumps({"value": "same"}),
call_id="call_0",
)
],
]
)
run_config = RunConfig(model_provider=_ScriptedProvider(model))
agent = Agent(
name="custom-provider-agent",
model="scripted-provider-model",
tools=[first_tool, second_tool],
)
first = await Runner.run(agent, "run tools", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, state, run_config=run_config)
assert executed == ["first:same"]
@pytest.mark.asyncio
async def test_approved_call_id_reused_for_another_invocation_type_fails_before_execution() -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
def approved_tool(value: str) -> str:
executed.append(f"function:{value}")
return value
async def invoke_custom(_ctx: Any, raw_input: str) -> str:
executed.append(f"custom:{raw_input}")
return raw_input
custom_tool = CustomTool(
name="raw_editor",
description="Edit raw text.",
on_invoke_tool=invoke_custom,
format={"type": "text"},
)
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
"approved_tool",
json.dumps({"value": "safe"}),
call_id="call_0",
)
],
[
ResponseCustomToolCall(
type="custom_tool_call",
name="raw_editor",
call_id="call_0",
input="changed-kind",
)
],
]
)
run_config = RunConfig(model_provider=_ScriptedProvider(model))
agent = Agent(
name="custom-provider-agent",
model="scripted-provider-model",
tools=[approved_tool, custom_tool],
)
first = await Runner.run(agent, "run tools", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, state, run_config=run_config)
assert executed == ["function:safe"]
@pytest.mark.asyncio
async def test_changed_missing_tool_under_approved_call_id_fails_before_sibling_tool() -> None:
executed: list[str] = []
approval_checks: list[str] = []
@function_tool(needs_approval=True)
def approved_tool(value: str) -> str:
executed.append(f"approved:{value}")
return value
async def sibling_needs_approval(_ctx: Any, _args: dict[str, Any], call_id: str) -> bool:
approval_checks.append(call_id)
return False
@function_tool(needs_approval=sibling_needs_approval)
def sibling_tool() -> str:
executed.append("sibling")
return "sibling"
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
"approved_tool",
json.dumps({"value": "safe"}),
call_id="call_0",
)
],
[
get_function_tool_call("sibling_tool", "{}", call_id="call_1"),
get_function_tool_call("missing_tool", "{}", call_id="call_0"),
],
]
)
run_config = RunConfig(
model_provider=_ScriptedProvider(model),
tool_not_found_behavior="return_error_to_model",
)
agent = Agent(
name="custom-provider-agent",
model="scripted-provider-model",
tools=[approved_tool, sibling_tool],
)
first = await Runner.run(agent, "run tools", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, state, run_config=run_config)
assert executed == ["approved:safe"]
assert approval_checks == []
def _build_serialized_replay_scenario(
replay_value: str,
) -> tuple[Agent[Any], RunConfig, list[str]]:
executed: list[str] = []
@function_tool(needs_approval=True)
def record_value(value: str) -> str:
executed.append(f"record:{value}")
return value
@function_tool(needs_approval=True)
def approval_gate(value: str) -> str:
executed.append(f"gate:{value}")
return value
model = ScriptedModel()
model.extend(
[
[
get_function_tool_call(
"record_value",
json.dumps({"value": "safe"}),
call_id="call_0",
)
],
[
get_function_tool_call(
"approval_gate",
json.dumps({"value": "pause"}),
call_id="call_1",
)
],
[
get_function_tool_call(
"record_value",
json.dumps({"value": replay_value}, separators=(",", ":")),
call_id="call_0",
)
],
[get_text_message("done")],
]
)
run_config = RunConfig(model_provider=_ScriptedProvider(model))
agent = Agent(
name="custom-provider-agent",
model="scripted-provider-model",
tools=[record_value, approval_gate],
)
return agent, run_config, executed
@pytest.mark.asyncio
async def test_serialized_pending_approval_keeps_its_invocation_binding() -> None:
agent, run_config, executed = _build_scenario("call_0", "changed")
first = await Runner.run(agent, "record a value", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
restored = await RunState.from_string(agent, state.to_string())
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, restored, run_config=run_config)
assert executed == ["safe"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_serialized_completed_approval_rejects_changed_replay(
mode: Literal["non_streamed", "streamed"],
) -> None:
agent, run_config, executed = _build_serialized_replay_scenario("changed")
first = await _run(agent, "record a value", run_config=run_config, mode=mode)
state = first.to_state()
state.approve(first.interruptions[0])
second = await _run(agent, state, run_config=run_config, mode=mode)
restored = await RunState.from_string(agent, second.to_state().to_string())
restored.approve(restored.get_interruptions()[0])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await _run(agent, restored, run_config=run_config, mode=mode)
assert executed == ["record:safe", "gate:pause"]
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
async def test_serialized_completed_approval_skips_exact_replay(
mode: Literal["non_streamed", "streamed"],
) -> None:
agent, run_config, executed = _build_serialized_replay_scenario("safe")
first = await _run(agent, "record a value", run_config=run_config, mode=mode)
state = first.to_state()
state.approve(first.interruptions[0])
second = await _run(agent, state, run_config=run_config, mode=mode)
restored = await RunState.from_string(agent, second.to_state().to_string())
restored.approve(restored.get_interruptions()[0])
result = await _run(agent, restored, run_config=run_config, mode=mode)
assert result.final_output == "done"
assert executed == ["record:safe", "gate:pause"]
provider = run_config.model_provider
assert isinstance(provider, _ScriptedProvider)
model = provider.model
assert isinstance(model, ScriptedModel)
model_input = model.calls[-1].input
assert isinstance(model_input, list)
for call_id in ("call_0", "call_1"):
calls = [
item
for item in model_input
if item.get("type") == "function_call" and item.get("call_id") == call_id
]
outputs = [
item
for item in model_input
if item.get("type") == "function_call_output" and item.get("call_id") == call_id
]
assert len(calls) == 1
assert len(outputs) == 1
async def _restore_as_schema_1_13(
agent: Agent[Any],
state: RunState[Any, Agent[Any]],
) -> RunState[Any, Agent[Any]]:
json_data = state.to_json()
for entry in json_data["context"].pop("function_tool_approvals", []):
json_data["context"]["approvals"][entry["tool_key"]] = entry["decision"]
json_data["$schemaVersion"] = "1.13"
json_data["context"].pop("tool_invocations", None)
return await RunState.from_json(agent, json_data)
@pytest.mark.asyncio
@pytest.mark.parametrize("schema_version", ["1.13", "1.14"])
@pytest.mark.parametrize("mode", ["non_streamed", "streamed"])
@pytest.mark.parametrize("replay_value", ["safe", "changed"])
async def test_legacy_schema_historical_sticky_call_id_is_not_reexecuted(
schema_version: str,
mode: Literal["non_streamed", "streamed"],
replay_value: str,
) -> None:
executed: list[str] = []
@function_tool(needs_approval=True)
async def record_value(value: str) -> str:
executed.append(value)
return value
historical_call = cast(
ResponseFunctionToolCall,
get_function_tool_call(
"record_value",
'{"value":"safe"}',
call_id="call_0",
),
)
model = ScriptedModel(
steps=[
[
get_function_tool_call(
"record_value",
json.dumps({"value": replay_value}),
call_id="call_0",
)
]
]
)
model.enqueue([get_text_message("done")])
agent = Agent(name="agent", model=model, tools=[record_value])
context: RunContextWrapper[Any] = RunContextWrapper(context=None)
context.approve_tool(
ToolApprovalItem(agent=agent, raw_item=historical_call),
always_approve=True,
)
state = RunState(
context=context,
original_input=[
historical_call.model_dump(exclude_none=True),
{
"type": "function_call_output",
"call_id": "call_0",
"output": "safe",
},
],
starting_agent=agent,
)
serialized = state.to_json()
for entry in serialized["context"].pop("function_tool_approvals"):
serialized["context"]["approvals"][entry["tool_key"]] = entry["decision"]
serialized["$schemaVersion"] = schema_version
serialized["context"].pop("tool_invocations", None)
restored = await RunState.from_json(agent, serialized)
if replay_value == "changed":
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await _run(agent, restored, run_config=RunConfig(), mode=mode)
else:
result = await _run(agent, restored, run_config=RunConfig(), mode=mode)
assert result.final_output == "done"
assert executed == []
@pytest.mark.asyncio
@pytest.mark.parametrize("schema_version", ["1.13", "1.14"])
async def test_legacy_changed_pending_call_fails_before_approval_callback(
schema_version: str,
) -> None:
approval_calls: list[str] = []
async def needs_approval(_context: Any, arguments: dict[str, Any], _call_id: str) -> bool:
approval_calls.append(arguments["value"])
return True
@function_tool(needs_approval=needs_approval)
async def record_value(value: str) -> str:
return value
historical_call = cast(
ResponseFunctionToolCall,
get_function_tool_call(
"record_value",
'{"value":"safe"}',
call_id="call_0",
),
)
changed_call = cast(
ResponseFunctionToolCall,
get_function_tool_call(
"record_value",
'{"value":"changed"}',
call_id="call_0",
),
)
agent = Agent(name="agent", tools=[record_value])
context: RunContextWrapper[Any] = RunContextWrapper(context=None)
context.approve_tool(ToolApprovalItem(agent=agent, raw_item=historical_call))
pending_item = ToolApprovalItem(agent=agent, raw_item=changed_call)
state = make_state_with_interruptions(
agent,
[pending_item],
original_input=cast(
Any,
[
historical_call.model_dump(exclude_none=True),
{
"type": "function_call_output",
"call_id": "call_0",
"output": "safe",
},
],
),
)
state._context = context
serialized = state.to_json()
for entry in serialized["context"].pop("function_tool_approvals", []):
serialized["context"]["approvals"][entry["tool_key"]] = entry["decision"]
serialized["$schemaVersion"] = schema_version
serialized["context"].pop("tool_invocations", None)
restored = await RunState.from_json(agent, serialized)
restored_pending = restored.get_interruptions()[0]
run = ToolRunFunction(tool_call=changed_call, function_tool=record_value)
async def build_rejection(_run: ToolRunFunction, _call_id: str) -> ToolApprovalItem:
return restored_pending
async def check_approval(_run: ToolRunFunction) -> bool:
return await needs_approval(None, {"value": "changed"}, "call_0")
assert restored._context is not None
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await _collect_runs_by_approval(
[run],
call_id_extractor=lambda item: item.tool_call.call_id,
tool_name_resolver=lambda item: item.function_tool.name,
rejection_builder=build_rejection,
context_wrapper=restored._context,
approval_items_by_call_id={"call_0": restored_pending},
agent=agent,
pending_interruption_adder=lambda _item: None,
needs_approval_checker=check_approval,
)
assert approval_calls == []
@pytest.mark.asyncio
async def test_schema_1_13_completed_approval_rejects_changed_replay() -> None:
agent, run_config, executed = _build_serialized_replay_scenario("changed")
first = await Runner.run(agent, "record a value", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
second = await Runner.run(agent, state, run_config=run_config)
restored = await _restore_as_schema_1_13(agent, second.to_state())
restored.approve(restored.get_interruptions()[0])
with pytest.raises(ModelBehaviorError, match="unique call ID"):
await Runner.run(agent, restored, run_config=run_config)
assert executed == ["record:safe", "gate:pause"]
@pytest.mark.asyncio
async def test_schema_1_13_completed_approval_skips_exact_replay() -> None:
agent, run_config, executed = _build_serialized_replay_scenario("safe")
first = await Runner.run(agent, "record a value", run_config=run_config)
state = first.to_state()
state.approve(first.interruptions[0])
second = await Runner.run(agent, state, run_config=run_config)
restored = await _restore_as_schema_1_13(agent, second.to_state())
restored.approve(restored.get_interruptions()[0])
result = await Runner.run(agent, restored, run_config=run_config)
assert result.final_output == "done"
assert executed == ["record:safe", "gate:pause"]
def _build_mcp_approval_request(
agent: Agent[Any],
*,
name: str = "lookup",
server_label: str = "test_server",
call_id: str = "mcp_call_0",
arguments: str | None = None,
) -> tuple[Any, ToolApprovalItem]:
request_item = McpApprovalRequest(
id=call_id,
type="mcp_approval_request",
name=name,
server_label=server_label,
arguments=arguments or json.dumps({"query": "safe"}),
)
request = SimpleNamespace(
request_item=request_item,
mcp_tool=SimpleNamespace(name=name, on_approval_request=None),
)
approval_item = ToolApprovalItem(
agent=agent,
raw_item=request_item,
tool_name=name,
)
return request, approval_item
def test_wrapped_mcp_approval_uses_the_same_canonical_request_id() -> None:
agent = Agent(name="mcp-approval-agent")
approval_item = ToolApprovalItem(
agent=agent,
raw_item={
"type": "hosted_tool_call",
"id": "outer-item-id",
"call_id": "shared-request",
"name": "lookup",
"provider_data": {
"type": "mcp_approval_request",
"name": "lookup",
"server_label": "test_server",
"arguments": '{"query":"safe"}',
},
},
tool_name="lookup",
)
changed_request, _ = _build_mcp_approval_request(
agent,
call_id="shared-request",
arguments='{"query":"changed"}',
)
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
context.approve_tool(approval_item)
assert set(context._tool_invocations) == {"shared-request"}
with pytest.raises(ModelBehaviorError, match="unique call ID"):
collect_manual_mcp_approvals(
agent=agent,
requests=[changed_request],
context_wrapper=context,
existing_pending_by_call_id={"shared-request": approval_item},
)
def test_sticky_mcp_approval_rejects_same_id_on_another_server() -> None:
agent = Agent(name="mcp-approval-agent")
_, approval_item = _build_mcp_approval_request(agent, server_label="server_a")
changed_request, _ = _build_mcp_approval_request(agent, server_label="server_b")
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
context.approve_tool(approval_item, always_approve=True)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
collect_manual_mcp_approvals(
agent=agent,
requests=[changed_request],
context_wrapper=context,
existing_pending_by_call_id={"mcp_call_0": approval_item},
)
def test_sticky_mcp_approval_reprompts_new_id_on_another_server() -> None:
agent = Agent(name="mcp-approval-agent")
_, approval_item = _build_mcp_approval_request(agent, server_label="server_a")
changed_request, _ = _build_mcp_approval_request(
agent,
server_label="server_b",
call_id="mcp_call_1",
)
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
context.approve_tool(approval_item, always_approve=True)
responses, pending = collect_manual_mcp_approvals(
agent=agent,
requests=[changed_request],
context_wrapper=context,
existing_pending_by_call_id={},
)
assert responses == []
assert len(pending) == 1
assert pending[0].raw_item is changed_request.request_item
@pytest.mark.asyncio
async def test_serialized_sticky_mcp_scope_reprompts_another_server() -> None:
agent = Agent(name="mcp-approval-agent")
_, approval_item = _build_mcp_approval_request(agent, server_label="server_a")
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
state = RunState(context=context, original_input="", starting_agent=agent)
state.approve(approval_item, always_approve=True)
restored = await RunState.from_json(agent, state.to_json())
assert restored._context is not None
changed_request, _ = _build_mcp_approval_request(
agent,
server_label="server_b",
call_id="mcp_call_1",
)
responses, pending = collect_manual_mcp_approvals(
agent=agent,
requests=[changed_request],
context_wrapper=restored._context,
existing_pending_by_call_id={},
)
assert responses == []
assert len(pending) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("schema_version", ["1.13", "1.14"])
@pytest.mark.parametrize("missing_field", ["arguments", "server_label"])
async def test_unbindable_legacy_mcp_approval_requires_current_reapproval(
schema_version: str,
missing_field: str,
) -> None:
agent = Agent(name="mcp-approval-agent")
current_request, _ = _build_mcp_approval_request(agent)
approval_item = ToolApprovalItem(
agent=agent,
raw_item=current_request.request_item,
tool_name="lookup",
)
state = make_state_with_interruptions(agent, [approval_item])
assert state._context is not None
state._context._rebuild_approvals( # noqa: SLF001
{
"lookup": {
"approved": ["mcp_call_0"],
"rejected": [],
}
}
)
serialized = state.to_json()
serialized["$schemaVersion"] = schema_version
serialized["context"].pop("tool_invocations", None)
serialized["current_step"]["data"]["interruptions"][0]["raw_item"].pop(missing_field)
restored = await RunState.from_json(agent, serialized)
assert restored._context is not None
restored_item = restored.get_interruptions()[0]
responses, pending = collect_manual_mcp_approvals(
agent=agent,
requests=[current_request],
context_wrapper=restored._context,
existing_pending_by_call_id={"mcp_call_0": restored_item},
)
assert responses == []
assert len(pending) == 1
assert pending[0].raw_item is current_request.request_item
def _build_unbindable_current_mcp_request() -> Any:
request_item = McpApprovalRequest.model_construct(
id="mcp_call_0",
type="mcp_approval_request",
name="lookup",
server_label="test_server",
)
return SimpleNamespace(
request_item=request_item,
mcp_tool=SimpleNamespace(name="lookup", on_approval_request=None),
)
@pytest.mark.asyncio
async def test_unbindable_mcp_callback_request_requires_manual_reapproval() -> None:
callback_calls = 0
def approve_request(_request: MCPToolApprovalRequest) -> MCPToolApprovalFunctionResult:
nonlocal callback_calls
callback_calls += 1
return {"approve": True}
mcp_tool = HostedMCPTool(
tool_config={
"type": "mcp",
"server_label": "test_server",
"server_url": "https://example.com",
"require_approval": "always",
},
on_approval_request=approve_request,
)
request = McpApprovalRequest.model_construct(
id="mcp_call_0",
type="mcp_approval_request",
name="lookup",
server_label="test_server",
)
agent = Agent(
name="mcp-approval-agent",
model=ScriptedModel(steps=[[request]]),
tools=[mcp_tool],
)
result = await Runner.run(agent, "lookup")
assert callback_calls == 0
assert len(result.interruptions) == 1
assert result.interruptions[0].raw_item == request
assert result.interruptions[0].raw_item is not request
@pytest.mark.parametrize("always_approve", [False, True], ids=["per-call", "sticky"])
def test_unbindable_current_manual_mcp_request_requires_reapproval(
always_approve: bool,
) -> None:
agent = Agent(name="mcp-approval-agent")
_, approval_item = _build_mcp_approval_request(agent)
current_request = _build_unbindable_current_mcp_request()
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
context.approve_tool(approval_item, always_approve=always_approve)
responses, pending = collect_manual_mcp_approvals(
agent=agent,
requests=[current_request],
context_wrapper=context,
existing_pending_by_call_id={"mcp_call_0": approval_item},
)
assert responses == []
assert len(pending) == 1
assert pending[0].raw_item is current_request.request_item
def test_unbindable_current_hosted_mcp_request_requires_reapproval() -> None:
agent = Agent(name="mcp-approval-agent")
_, approval_item = _build_mcp_approval_request(agent)
current_request = _build_unbindable_current_mcp_request()
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
context.approve_tool(approval_item)
appended: list[Any] = []
pending, pending_ids = process_hosted_mcp_approvals(
original_pre_step_items=[approval_item],
mcp_approval_requests=[current_request],
context_wrapper=context,
agent=agent,
append_item=appended.append,
)
assert len(pending) == 1
assert pending[0].raw_item is current_request.request_item
assert pending_ids == {"mcp_call_0"}
assert appended == pending
@pytest.mark.parametrize("with_callback", [False, True], ids=["manual", "callback"])
@pytest.mark.asyncio
async def test_runner_omits_completed_mcp_approval_request_replay(
with_callback: bool,
) -> None:
callback_calls = 0
def approve_request(_request: MCPToolApprovalRequest) -> MCPToolApprovalFunctionResult:
nonlocal callback_calls
callback_calls += 1
return {"approve": True}
mcp_tool = HostedMCPTool(
tool_config={
"type": "mcp",
"server_label": "test_server",
"server_url": "https://example.com",
"require_approval": "always",
},
on_approval_request=approve_request if with_callback else None,
)
first_request = McpApprovalRequest(
id="mcp_call_0",
type="mcp_approval_request",
name="lookup",
server_label="test_server",
arguments='{"query": "safe", "limit": 1}',
)
replayed_request = McpApprovalRequest(
id="mcp_call_0",
type="mcp_approval_request",
name="lookup",
server_label="test_server",
arguments='{"limit":1,"query":"safe"}',
)
model = ScriptedModel()
model.extend([[first_request], [replayed_request], [get_text_message("done")]])
agent = Agent(name="mcp-approval-agent", model=model, tools=[mcp_tool])
first = await Runner.run(agent, "lookup")
if with_callback:
result = first
else:
assert len(first.interruptions) == 1
state = first.to_state()
state.approve(first.interruptions[0])
result = await Runner.run(agent, state)
assert result.final_output == "done"
assert callback_calls == int(with_callback)
model_input = model.calls[-1].input
assert isinstance(model_input, list)
replay_items = [
item
for item in model_input
if item.get("type") == "mcp_approval_request" and item.get("id") == "mcp_call_0"
]
approval_responses = [
item
for item in model_input
if item.get("type") == "mcp_approval_response"
and item.get("approval_request_id") == "mcp_call_0"
]
assert len(replay_items) == 1
assert len(approval_responses) == 1
@pytest.mark.asyncio
async def test_serialized_completed_manual_mcp_approval_skips_exact_replay() -> None:
agent = Agent(name="mcp-approval-agent")
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
request, approval_item = _build_mcp_approval_request(agent)
state = RunState(context=context, original_input="", starting_agent=agent)
state.approve(approval_item)
first_responses, first_pending = collect_manual_mcp_approvals(
agent=agent,
requests=[request],
context_wrapper=context,
existing_pending_by_call_id={"mcp_call_0": approval_item},
)
assert len(first_responses) == 1
assert first_pending == []
context._mark_tool_call_completed(first_responses[0].raw_item)
state._generated_items = [approval_item, first_responses[0]]
restored = await RunState.from_string(agent, state.to_string())
assert restored._context is not None
replayed_responses, replayed_pending = collect_manual_mcp_approvals(
agent=agent,
requests=[request],
context_wrapper=restored._context,
existing_pending_by_call_id={"mcp_call_0": approval_item},
)
assert replayed_responses == []
assert replayed_pending == []
def test_completed_hosted_mcp_approval_reconciliation_skips_exact_replay() -> None:
agent = Agent(name="mcp-approval-agent")
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
request, approval_item = _build_mcp_approval_request(agent)
context.approve_tool(approval_item)
appended: list[Any] = []
process_hosted_mcp_approvals(
original_pre_step_items=[approval_item],
mcp_approval_requests=[request],
context_wrapper=context,
agent=agent,
append_item=appended.append,
)
assert len(appended) == 1
context._mark_tool_call_completed(appended[0].raw_item)
appended.clear()
pending, pending_ids = process_hosted_mcp_approvals(
original_pre_step_items=[approval_item],
mcp_approval_requests=[request],
context_wrapper=context,
agent=agent,
append_item=appended.append,
)
assert appended == []
assert pending == []
assert pending_ids == set()
def test_sticky_manual_mcp_approval_rejects_same_id_for_changed_tool_name() -> None:
agent = Agent(name="mcp-approval-agent")
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
_, approval_item = _build_mcp_approval_request(agent)
context.approve_tool(approval_item, always_approve=True)
same_tool, _ = _build_mcp_approval_request(
agent,
arguments=json.dumps({"query": "changed"}),
)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
collect_manual_mcp_approvals(
agent=agent,
requests=[same_tool],
context_wrapper=context,
existing_pending_by_call_id={"mcp_call_0": approval_item},
)
changed_context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
changed_context.approve_tool(approval_item, always_approve=True)
changed_tool, _ = _build_mcp_approval_request(
agent,
name="delete_all",
arguments=json.dumps({"confirm": True}),
)
with pytest.raises(ModelBehaviorError, match="unique call ID"):
collect_manual_mcp_approvals(
agent=agent,
requests=[changed_tool],
context_wrapper=changed_context,
existing_pending_by_call_id={"mcp_call_0": approval_item},
)
def test_sticky_hosted_mcp_approval_rejects_same_id_for_changed_tool_name() -> None:
agent = Agent(name="mcp-approval-agent")
context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
_, approval_item = _build_mcp_approval_request(agent)
context.approve_tool(approval_item, always_approve=True)
same_tool, _ = _build_mcp_approval_request(
agent,
arguments=json.dumps({"query": "changed"}),
)
same_appended: list[Any] = []
with pytest.raises(ModelBehaviorError, match="unique call ID"):
process_hosted_mcp_approvals(
original_pre_step_items=[approval_item],
mcp_approval_requests=[same_tool],
context_wrapper=context,
agent=agent,
append_item=same_appended.append,
)
changed_context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
changed_context.approve_tool(approval_item, always_approve=True)
changed_tool, _ = _build_mcp_approval_request(
agent,
name="delete_all",
arguments=json.dumps({"confirm": True}),
)
changed_appended: list[Any] = []
with pytest.raises(ModelBehaviorError, match="unique call ID"):
process_hosted_mcp_approvals(
original_pre_step_items=[approval_item],
mcp_approval_requests=[changed_tool],
context_wrapper=changed_context,
agent=agent,
append_item=changed_appended.append,
)
assert same_appended == []
assert changed_appended == []
@pytest.mark.parametrize(
("second_arguments", "should_raise"),
[('{ "value" : "safe" }', False), ('{"value":"changed"}', True)],
ids=["same_invocation", "changed_invocation"],
)
def test_history_call_id_reuse_is_checked_without_current_response_call(
second_arguments: str,
should_raise: bool,
) -> None:
agent = Agent(name="agent")
output = {"type": "function_call_output", "call_id": "call_0", "output": "ok"}
existing_items: list[Any] = [
ToolCallItem(
agent=agent,
raw_item=cast(
Any, get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")
),
),
ToolCallOutputItem(agent=agent, raw_item=cast(Any, output), output="ok"),
ToolCallItem(
agent=agent,
raw_item=cast(
Any, get_function_tool_call("record_value", second_arguments, call_id="call_0")
),
),
ToolCallOutputItem(agent=agent, raw_item=cast(Any, dict(output)), output="ok"),
]
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
def dedupe() -> None:
_dedupe_processed_response_invocations(
processed_response,
context_wrapper=RunContextWrapper(context=None),
existing_items=existing_items,
)
if should_raise:
with pytest.raises(ModelBehaviorError, match="Run history reused a tool call ID"):
dedupe()
else:
dedupe()