openai-agents-python/tests/test_run_impl_resume_paths.py
Mark Stuart 71c2da4de4
fix: preserve tool outputs after callback failures (#5318)
* fix: preserve tool outputs after callback retries

* fix: keep callback retry outputs ordered and streamed

* fix: complete callback retry delivery and guardrail recovery

* fix: keep callback retry outputs private until delivery

* fix: retain private callback checkpoints and published history

* fix: retain published diagnostics and guard legacy retries
2026-10-07 04:19:46 +00:00

3664 lines
145 KiB
Python

import asyncio
import copy
import json
import sqlite3
import threading
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Literal, cast
import pytest
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputMessage
import agents.run as run_module
from agents import (
Agent,
AgentUpdatedStreamEvent,
Runner,
ShellTool,
ToolExecutionConfig,
function_tool,
handoff,
output_guardrail,
)
from agents.agent import StopAtTools, ToolsToFinalOutputResult
from agents.agent_output import AgentOutputSchema
from agents.decorators import tool, tool_input_guardrail, tool_output_guardrail
from agents.exceptions import (
InputGuardrailTripwireTriggered,
OutputGuardrailTripwireTriggered,
UserError,
_is_error_data_redacted,
)
from agents.guardrail import GuardrailFunctionOutput, input_guardrail
from agents.handoffs import HandoffInputData
from agents.items import (
MessageOutputItem,
ModelResponse,
ToolApprovalItem,
ToolCallItem,
ToolCallOutputItem,
TResponseInputItem,
)
from agents.lifecycle import RunHooks
from agents.memory import OpenAIResponsesCompactionSession, Session, SQLiteSession
from agents.result import RunResultStreaming, ToInputListMode
from agents.run import RunConfig
from agents.run_context import RunContextWrapper
from agents.run_internal import run_loop, turn_resolution
from agents.run_internal.agent_bindings import bind_public_agent
from agents.run_internal.run_loop import (
NextStepFinalOutput,
NextStepHandoff,
NextStepInterruption,
NextStepRunAgain,
ProcessedResponse,
SingleStepResult,
)
from agents.run_state import CURRENT_SCHEMA_VERSION, RunState
from agents.sandbox.runtime import SandboxRuntime
from agents.testing import ScriptedModel
from agents.tool import Tool
from agents.tool_guardrails import (
ToolGuardrailFunctionOutput,
ToolInputGuardrailData,
ToolOutputGuardrailData,
)
from agents.usage import Usage
from tests.test_responses import get_function_tool_call, get_text_message
from tests.utils.hitl import (
make_agent,
make_context_wrapper,
make_model_and_agent,
make_shell_call,
queue_function_call_and_text,
)
from tests.utils.simple_session import SimpleListSession
class _FailingResumeSession(SimpleListSession):
"""Control append acknowledgement at the public Session boundary."""
def __init__(self) -> None:
super().__init__()
self.failure: str | None = None
self.error = RuntimeError("session append failed")
self.block_next_add = False
self.add_started = asyncio.Event()
self.release_add = asyncio.Event()
self.fail_on_output: str | None = None
async def add_items(self, items: list[TResponseInputItem]) -> None:
if self.fail_on_output is not None and any(
self.fail_on_output in json.dumps(item, default=str) for item in items
):
self.fail_on_output = None
raise self.error
failure, self.failure = self.failure, None
if failure == "before":
raise self.error
if self.block_next_add:
self.block_next_add = False
self.add_started.set()
await self.release_add.wait()
if failure == "partial":
await super().add_items(items[:1])
raise self.error
await super().add_items(items)
if failure == "after":
raise self.error
class _FailSecondAddItemsSession(SimpleListSession):
"""Let the initial input-priming append succeed, then fail the next append.
Unlike ``_FailingResumeSession``, this targets a specific append by call order rather than
a resume-cycle phase, so it can isolate a fresh (non-resumed) run's first real turn save.
"""
def __init__(self) -> None:
super().__init__()
self.error = RuntimeError("session append failed")
self._call_count = 0
async def add_items(self, items: list[TResponseInputItem]) -> None:
self._call_count += 1
if self._call_count == 2:
raise self.error
await super().add_items(items)
class _FailSecondAddItemsSessionWithYield(_FailSecondAddItemsSession):
"""Same failure shape as ``_FailSecondAddItemsSession``, but the failing call performs a
genuine ``await`` (a scheduler yield) before raising, like a real I/O-backed Session
(SQLite, network, etc.) would. A purely synchronous raise never yields control back to the
``stream_events()`` consumer before the run-loop task finishes, so a test built on it cannot
observe whether an already-queued stream event was delivered before the error surfaced.
"""
async def add_items(self, items: list[TResponseInputItem]) -> None:
if self._call_count == 1:
await asyncio.sleep(0)
await super().add_items(items)
class _LostAckSQLiteSession(SQLiteSession):
fail_after_commit = False
error = RuntimeError("session append failed")
async def add_items(self, items: list[TResponseInputItem]) -> None:
await super().add_items(items)
if self.fail_after_commit:
self.fail_after_commit = False
raise self.error
async def _run_session_resume(
agent: Agent[Any],
value: str | RunState[Any],
session: Session | None,
streamed: bool,
hooks: RunHooks[Any] | None = None,
events: list[Any] | None = None,
):
config = RunConfig(tracing_disabled=True)
if not streamed:
return await Runner.run(agent, value, session=session, run_config=config, hooks=hooks)
result = Runner.run_streamed(agent, value, session=session, run_config=config, hooks=hooks)
async for event in result.stream_events():
if events is not None:
events.append(event)
return result
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"])
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
@pytest.mark.parametrize("failure", ["extractor", "end_hook"])
async def test_callback_retry_preserves_tool_output_in_result_history(
streamed: bool, round_trip: bool, failure: str
) -> None:
effects: list[str] = []
callback_calls: list[str] = []
def fail_callback() -> None:
callback_calls.append(failure)
raise RuntimeError("post-tool callback failed")
def extract_custom_data(_context: Any) -> dict[str, Any]:
fail_callback()
return {}
@tool_input_guardrail
def check_input(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info="input checked")
@tool_output_guardrail
def check_output(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info="output checked")
class FailingEndHook(RunHooks[Any]):
async def on_tool_end(
self, context: RunContextWrapper[Any], agent: Agent[Any], tool: Tool, result: Any
) -> None:
fail_callback()
@function_tool(
needs_approval=True,
custom_data_extractor=extract_custom_data if failure == "extractor" else None,
tool_input_guardrails=[check_input],
tool_output_guardrails=[check_output],
)
async def charge() -> str:
effects.append("charged")
return "receipt-7"
model = ScriptedModel(
[
[get_function_tool_call("charge", "{}", call_id="charge-1")],
[get_text_message("done")],
]
)
agent = Agent(
name="agent",
model=model,
tools=[charge],
tool_use_behavior=(
StopAtTools(stop_at_tool_names=["charge"])
if failure == "extractor"
else "run_llm_again"
),
)
hooks = FailingEndHook() if failure == "end_hook" else None
paused = await _run_session_resume(agent, "charge once", None, streamed, hooks)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
events: list[Any] = []
if streamed:
failed = Runner.run_streamed(
agent, state, run_config=RunConfig(tracing_disabled=True), hooks=hooks
)
with pytest.raises(UserError, match="post-tool callback failed"):
async for event in failed.stream_events():
events.append(event)
state = failed.to_state()
else:
with pytest.raises(UserError, match="post-tool callback failed"):
await _run_session_resume(agent, state, None, streamed, hooks)
assert [
item.output for item in state._generated_items if isinstance(item, ToolCallOutputItem)
] == ["receipt-7"]
assert [result.output.output_info for result in state._tool_input_guardrail_results] == [
"input checked"
]
assert [result.output.output_info for result in state._tool_output_guardrail_results] == [
"output checked"
]
if round_trip:
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
result = await _run_session_resume(agent, state, None, streamed, hooks, events)
assert effects == ["charged"]
assert callback_calls == [failure]
assert [result.output.output_info for result in result.tool_input_guardrail_results] == [
"input checked"
]
assert [result.output.output_info for result in result.tool_output_guardrail_results] == [
"output checked"
]
if streamed:
assert [
event.item.output
for event in events
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem)
] == ["receipt-7"]
assert result.final_output == ("receipt-7" if failure == "extractor" else "done")
assert [item.output for item in result.new_items if isinstance(item, ToolCallOutputItem)] == [
"receipt-7"
]
assert _call_pair(result.to_input_list(), "charge-1") == [
"function_call",
"function_call_output",
]
restored = await RunState.from_json(agent, result.to_state().to_json())
assert [
item.output for item in restored._session_items if isinstance(item, ToolCallOutputItem)
] == ["receipt-7"]
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"])
@pytest.mark.parametrize("previously_published", [False, True], ids=["pending", "published"])
async def test_callback_retry_keeps_current_response_tool_order(
streamed: bool, previously_published: bool
) -> None:
effects: list[str] = []
session = SimpleListSession()
events: list[Any] = []
@tool_input_guardrail
def check_input(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info=data.context.tool_call_id)
@tool_output_guardrail
def check_output(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info=data.output)
@function_tool(
needs_approval=True,
tool_input_guardrails=[check_input],
tool_output_guardrails=[check_output],
)
async def left() -> str:
effects.append("left")
return "left-result"
@function_tool(
needs_approval=True,
tool_input_guardrails=[check_input],
tool_output_guardrails=[check_output],
)
async def right() -> str:
effects.append("right")
return "right-result"
class FailingEndHook(RunHooks[Any]):
async def on_tool_end(
self, context: RunContextWrapper[Any], agent: Agent[Any], tool: Tool, result: Any
) -> None:
if tool.name == ("left" if previously_published else "right"):
raise RuntimeError("tool callback failed")
agent = Agent(
name="parallel",
model=ScriptedModel(
[
[
get_function_tool_call("left", "{}", call_id="left-1"),
get_function_tool_call("right", "{}", call_id="right-1"),
],
[get_text_message("done")],
]
),
tools=[left, right],
)
hooks = FailingEndHook()
paused = await _run_session_resume(agent, "run both", session, streamed, hooks)
state = paused.to_state()
left_approval, right_approval = state.get_interruptions()
state.approve(right_approval)
if previously_published:
partial = await _run_session_resume(agent, state, session, streamed, hooks)
assert [r.output.output_info for r in partial.tool_output_guardrail_results] == [
"right-result"
]
assert [
item.get("call_id")
for item in await session.get_items()
if item.get("type") == "function_call_output"
] == ["right-1"]
state = partial.to_state()
state.approve(state.get_interruptions()[0])
if streamed:
failed = Runner.run_streamed(
agent,
state,
session=session,
run_config=RunConfig(tracing_disabled=True),
hooks=hooks,
)
with pytest.raises(UserError, match="tool callback failed"):
async for _ in failed.stream_events():
pass
failed_output_results = failed.tool_output_guardrail_results
else:
with pytest.raises(UserError, match="tool callback failed") as caught:
await _run_session_resume(agent, state, session, streamed, hooks)
assert caught.value.run_data is not None
failed_output_results = caught.value.run_data.tool_output_guardrail_results
assert [r.output.output_info for r in failed_output_results] == (
["right-result"] if previously_published else []
)
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
state.approve(left_approval)
result = await _run_session_resume(agent, state, session, streamed, hooks, events)
assert effects == ["right", "left"]
assert result.final_output == "done"
expected_ids = ["right-1", "left-1"] if previously_published else ["left-1", "right-1"]
expected_outputs = (
["right-result", "left-result"] if previously_published else ["left-result", "right-result"]
)
assert [
item.output for item in result.new_items if isinstance(item, ToolCallOutputItem)
] == expected_outputs
assert [
item.get("call_id")
for item in result.to_input_list()
if item.get("type") == "function_call_output"
] == expected_ids
assert [
item.get("call_id")
for item in cast(ScriptedModel, agent.model).calls[-1].input
if item.get("type") == "function_call_output"
] == expected_ids
restored = await RunState.from_json(agent, result.to_state().to_json())
for history in (restored._session_items, restored._generated_items):
assert [
item.output for item in history if isinstance(item, ToolCallOutputItem)
] == expected_outputs
assert [
item.get("call_id")
for item in await session.get_items()
if item.get("type") == "function_call_output"
] == expected_ids
assert [r.output.output_info for r in result.tool_input_guardrail_results] == [
"right-1",
"left-1",
]
assert [r.output.output_info for r in result.tool_output_guardrail_results] == [
"right-result",
"left-result",
]
if streamed:
assert [
event.item.output
for event in events
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem)
] == (["left-result"] if previously_published else expected_outputs)
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"])
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
async def test_callback_retry_preserves_hosted_output_before_local_result(
streamed: bool, round_trip: bool
) -> None:
effects: list[str] = []
@function_tool
async def prelude() -> str:
return "prefix"
@function_tool(needs_approval=True)
async def charge() -> str:
effects.append("charged")
return "receipt-7"
class FailingChargeEndHook(RunHooks[Any]):
async def on_tool_end(
self, context: RunContextWrapper[Any], agent: Agent[Any], tool: Tool, result: Any
) -> None:
if tool.name == "charge":
raise RuntimeError("charge callback failed")
@output_guardrail
async def allowed(
context: RunContextWrapper[Any], agent: Agent[Any], output: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False)
agent = Agent(
name="hosted-shell",
model=ScriptedModel(
[
[get_function_tool_call("prelude", "{}", call_id="prelude")],
[
get_function_tool_call("charge", "{}", call_id="charge"),
make_shell_call("shell-1", id_value="sh-hosted", commands=["echo hi"]),
cast(
Any,
{
"type": "shell_call_output",
"id": "sh-out-hosted",
"call_id": "shell-1",
"status": "completed",
"output": [
{
"stdout": "hi",
"stderr": "",
"outcome": {"type": "exit", "exit_code": 0},
}
],
},
),
],
[get_text_message("done")],
]
),
tools=[prelude, charge, ShellTool(environment={"type": "container_auto"})],
output_guardrails=[allowed],
)
hooks = FailingChargeEndHook()
# ScriptedModel does not stream mapping-shaped hosted output items.
paused = await _run_session_resume(agent, "prepare and charge", None, False, hooks)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
with pytest.raises(UserError, match="charge callback failed"):
await _run_session_resume(agent, state, None, streamed, hooks)
if round_trip:
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
result = await _run_session_resume(agent, state, None, streamed, hooks)
assert result.final_output == "done"
assert effects == ["charged"]
assert [
item.get("call_id")
for item in result.to_input_list()
if item.get("type") in {"function_call_output", "shell_call_output"}
] == ["prelude", "shell-1", "charge"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"guardrail_outcome,prior_turn,with_session,retry_mode",
[
pytest.param("allowed", False, False, "streamed-live", id="first-turn-allowed"),
pytest.param("blocked", False, False, "streamed-live", id="first-turn-blocked"),
pytest.param("error", False, False, "streamed-json", id="first-turn-error"),
pytest.param("allowed", True, False, "streamed-json", id="prior-turn-allowed"),
pytest.param("blocked", True, False, "streamed-json", id="prior-turn-blocked"),
pytest.param("error", True, False, "streamed-json", id="prior-turn-error"),
pytest.param("error", True, True, "streamed-json", id="prior-turn-error-sqlite"),
pytest.param("error", True, True, "run-live", id="prior-turn-error-sqlite-run-live"),
pytest.param("error", True, True, "run-json", id="prior-turn-error-sqlite-run-json"),
],
)
async def test_failed_approval_callback_defers_stream_output_until_retry_guardrail(
guardrail_outcome: str,
prior_turn: bool,
with_session: bool,
retry_mode: Literal["streamed-live", "streamed-json", "run-live", "run-json"],
tmp_path: Path,
request: pytest.FixtureRequest,
) -> None:
effects: list[str] = []
callbacks: list[str] = []
guardrail_outputs: list[str] = []
guardrail_entered = asyncio.Event()
release_guardrail = asyncio.Event()
failed_outputs: list[str] = []
retry_outputs: list[str] = []
config = RunConfig(tracing_disabled=True)
session = SQLiteSession("approval-retry", tmp_path / "session.db") if with_session else None
if session is not None:
request.addfinalizer(session.close)
def input_outputs(result: RunResultStreaming) -> dict[str, list[Any]]:
modes: tuple[ToInputListMode, ...] = ("preserve_all", "normalized")
return {
mode: [
item.get("output")
for item in result.to_input_list(mode=mode)
if item.get("type") == "function_call_output"
]
for mode in modes
}
def fail_after_execution(_context: Any) -> dict[str, Any]:
callbacks.append("extractor")
raise RuntimeError("post-tool callback failed")
@tool_output_guardrail
def record_output(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info=data.output)
@function_tool(tool_output_guardrails=[record_output])
async def prelude() -> str:
return "earlier-result"
@function_tool(
needs_approval=True,
custom_data_extractor=fail_after_execution,
tool_output_guardrails=[record_output],
)
async def charge() -> str:
effects.append("charged")
return "receipt-7"
@output_guardrail
async def record_terminal_candidate(
_context: Any, _agent: Any, output: Any
) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(output_info=output, tripwire_triggered=False)
@output_guardrail
async def terminal_policy(_context: Any, _agent: Any, output: Any) -> GuardrailFunctionOutput:
guardrail_outputs.append(output)
guardrail_entered.set()
await release_guardrail.wait()
if guardrail_outcome == "error" and len(guardrail_outputs) == 1:
if retry_mode in {"run-live", "run-json"}:
raise UserError("policy temporarily unavailable")
raise RuntimeError("policy temporarily unavailable")
return GuardrailFunctionOutput(
output_info=None, tripwire_triggered=guardrail_outcome == "blocked"
)
model = ScriptedModel(
([[get_function_tool_call("prelude", "{}", call_id="prelude-1")]] if prior_turn else [])
+ [[get_function_tool_call("charge", "{}", call_id="charge-1")]]
)
agent = Agent(
name="agent",
model=model,
tools=[prelude, charge],
tool_use_behavior=StopAtTools(stop_at_tool_names=["charge"]),
output_guardrails=[record_terminal_candidate, terminal_policy],
)
paused = await Runner.run(agent, "charge once", session=session, run_config=config)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
prior_metadata = ["earlier-result"] if prior_turn else []
withheld_inputs = dict.fromkeys(("preserve_all", "normalized"), prior_metadata)
failed = Runner.run_streamed(agent, state, session=session, run_config=config)
with pytest.raises(UserError, match="post-tool callback failed"):
async for event in failed.stream_events():
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem):
# Copy the value at emission, so later owner replacements cannot hide a leak.
failed_outputs.append(event.item.output)
assert not guardrail_outputs
failed_metadata = [result.output.output_info for result in failed.tool_output_guardrail_results]
failed_input_outputs = input_outputs(failed)
if retry_mode != "streamed-live":
# Exercise the documented failed-result resume path and supported durable state format.
state = await RunState.from_json(agent, json.loads(json.dumps(failed.to_state().to_json())))
# For live retries, reuse the same approved state that was passed to the failed stream.
assert [result.output.output_info for result in state._tool_output_guardrail_results] == [
*prior_metadata,
"receipt-7",
]
retry = (
Runner.run_streamed(agent, state, session=session, run_config=config)
if retry_mode in {"streamed-live", "streamed-json"}
else None
)
initial_retry_metadata = (
[result.output.output_info for result in retry.tool_output_guardrail_results]
if retry is not None
else None
)
initial_retry_inputs = input_outputs(retry) if retry is not None else None
async def collect_retry() -> None:
if retry is None:
# Runner.run fails without returning a result; the caller resumes its same RunState.
await Runner.run(agent, state, session=session, run_config=config)
return
async for event in retry.stream_events():
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem):
retry_outputs.append(event.item.output)
consumer = asyncio.create_task(collect_retry())
try:
await asyncio.wait_for(guardrail_entered.wait(), timeout=5)
# Let the waiting stream consumer drain anything the producer queued before its await.
for _ in range(3):
await asyncio.sleep(0)
outputs_before_allow = list(retry_outputs)
metadata_before_allow = (
[result.output.output_info for result in retry.tool_output_guardrail_results]
if retry is not None
else None
)
pending_metadata = [
result.output.output_info for result in state._tool_output_guardrail_results
]
public_new_outputs = (
[item.output for item in retry.new_items if isinstance(item, ToolCallOutputItem)]
if retry is not None
else None
)
public_input_outputs = input_outputs(retry) if retry is not None else None
stored_before_allow = await session.get_items() if session is not None else None
finally:
release_guardrail.set()
if guardrail_outcome == "blocked":
with pytest.raises(OutputGuardrailTripwireTriggered):
await consumer
elif guardrail_outcome == "error":
if retry is None:
with pytest.raises(UserError, match="policy temporarily unavailable") as error:
await consumer
run_data = error.value.run_data
assert run_data is not None
assert [
result.output.output_info for result in run_data.tool_output_guardrail_results
] == prior_metadata
assert [
item.output for item in run_data.new_items if isinstance(item, ToolCallOutputItem)
] == prior_metadata
assert run_data.output_guardrail_results == []
else:
with pytest.raises(RuntimeError, match="policy temporarily unavailable"):
await consumer
else:
await consumer
# Assertions at the end ensure the failed snapshot really reaches the retry guardrail.
if guardrail_outcome == "error" and session is not None:
assert "function_call_output" not in _call_pair(await session.get_items(), "charge-1")
assert effects == ["charged"]
assert callbacks == ["extractor"]
assert guardrail_outputs == ["receipt-7"]
assert failed_outputs == [], "callback failure exposed unchecked terminal output"
assert failed_metadata == prior_metadata, (
"callback failure exposed unchecked guardrail metadata"
)
assert failed_input_outputs == withheld_inputs, "callback failure exposed unchecked model input"
assert outputs_before_allow == [], "retry exposed output while its guardrail was pending"
assert pending_metadata == [*prior_metadata, "receipt-7"]
if retry is not None:
assert initial_retry_metadata == prior_metadata, (
"retry exposed metadata before its guardrail"
)
assert initial_retry_inputs == withheld_inputs, "retry exposed model input before starting"
assert metadata_before_allow == prior_metadata, (
"retry exposed metadata during its guardrail"
)
assert public_new_outputs == prior_metadata, "retry exposed history during its guardrail"
assert public_input_outputs == withheld_inputs, (
"retry exposed model input during its guardrail"
)
assert len(model.calls) == (2 if prior_turn else 1)
if stored_before_allow is not None:
assert "function_call_output" not in _call_pair(stored_before_allow, "charge-1")
if guardrail_outcome == "error":
assert retry_outputs == []
if retry is not None:
assert retry.output_guardrail_results == []
assert [
result.output.output_info for result in retry.tool_output_guardrail_results
] == prior_metadata
assert input_outputs(retry) == withheld_inputs, (
"guardrail error exposed unchecked input"
)
state = retry.to_state()
restored = state
if retry_mode != "run-live":
restored = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
assert [
result.output.output_info for result in restored._tool_output_guardrail_results
] == [*prior_metadata, "receipt-7"]
guardrail_entered.clear()
release_guardrail.clear()
retry = Runner.run_streamed(agent, restored, session=session, run_config=config)
assert [
result.output.output_info for result in retry.tool_output_guardrail_results
] == prior_metadata
consumer = asyncio.create_task(collect_retry())
try:
await asyncio.wait_for(guardrail_entered.wait(), timeout=5)
for _ in range(3):
await asyncio.sleep(0)
assert retry_outputs == [], (
"second retry exposed output before its guardrail allowed it"
)
assert [
result.output.output_info for result in retry.tool_output_guardrail_results
] == prior_metadata
assert [
item.output for item in retry.new_items if isinstance(item, ToolCallOutputItem)
] == prior_metadata
assert input_outputs(retry) == withheld_inputs
if session is not None:
assert "function_call_output" not in _call_pair(
await session.get_items(), "charge-1"
)
finally:
release_guardrail.set()
await consumer
assert guardrail_outputs == ["receipt-7", "receipt-7"]
assert len(model.calls) == (2 if prior_turn else 1)
assert effects == ["charged"]
assert callbacks == ["extractor"]
if session is not None:
assert _call_pair(await session.get_items(), "charge-1") == [
"function_call",
"function_call_output",
]
elif guardrail_outcome == "blocked":
assert retry is not None
assert retry_outputs == []
assert all(
result.output.output_info != "receipt-7"
for result in retry.tool_output_guardrail_results
)
assert "receipt-7" not in state.to_string()
assert input_outputs(retry) == dict.fromkeys(
("preserve_all", "normalized"),
[*prior_metadata, run_loop._OUTPUT_GUARDRAIL_BLOCKED_TOOL_OUTPUT],
)
if guardrail_outcome != "blocked":
assert retry is not None
assert retry_outputs == ["receipt-7"]
assert retry.final_output == "receipt-7"
assert [result.output.output_info for result in retry.tool_output_guardrail_results] == [
*prior_metadata,
"receipt-7",
]
assert [
item.output for item in retry.new_items if isinstance(item, ToolCallOutputItem)
] == [*prior_metadata, "receipt-7"]
assert input_outputs(retry) == dict.fromkeys(
("preserve_all", "normalized"), [*prior_metadata, "receipt-7"]
)
restored = await RunState.from_json(agent, retry.to_state().to_json())
assert [
item.output for item in restored._session_items if isinstance(item, ToolCallOutputItem)
] == [*prior_metadata, "receipt-7"]
assert input_outputs(failed) == withheld_inputs, (
"later retries changed the failed stream's public output"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["extractor", "end_hook"])
@pytest.mark.parametrize("with_output_guardrail", [True, False])
async def test_redacted_callback_preserves_private_failed_stream_checkpoint(
failure: str, with_output_guardrail: bool
) -> None:
config = RunConfig(tracing_disabled=True)
modes: tuple[ToInputListMode, ...] = ("preserve_all", "normalized")
effects: list[str] = []
callback_calls: list[str] = []
checked: list[str] = []
emitted: list[Any] = []
@output_guardrail
def inner_policy(_context: Any, _agent: Any, output: Any) -> GuardrailFunctionOutput:
return GuardrailFunctionOutput(output_info=output, tripwire_triggered=True)
@function_tool
def inspect_callback() -> str:
return "inner-private-candidate"
inner_model = ScriptedModel(
[[get_function_tool_call("inspect_callback", "{}", call_id="inner-1")]]
)
inner = Agent(
name="inner",
model=inner_model,
tools=[inspect_callback],
tool_use_behavior=StopAtTools(stop_at_tool_names=["inspect_callback"]),
output_guardrails=[inner_policy],
)
async def nested_callback() -> None:
callback_calls.append(failure)
await Runner.run(inner, "check callback", run_config=config)
async def extract_custom_data(_context: Any) -> dict[str, Any]:
await nested_callback()
return {}
class EndHook(RunHooks[Any]):
async def on_tool_end(
self, context: RunContextWrapper[Any], agent: Agent[Any], tool: Tool, result: Any
) -> None:
await nested_callback()
@tool_output_guardrail
def record_output(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info=data.output)
@tool_input_guardrail
def record_input(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info="approval input checked")
@function_tool(
needs_approval=True,
custom_data_extractor=extract_custom_data if failure == "extractor" else None,
tool_input_guardrails=[record_input],
tool_output_guardrails=[record_output] if with_output_guardrail else [],
)
async def charge() -> str:
effects.append("charged")
return "receipt-7"
@output_guardrail
def outer_policy(_context: Any, _agent: Any, output: Any) -> GuardrailFunctionOutput:
checked.append(output)
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False)
model = ScriptedModel([[get_function_tool_call("charge", "{}", call_id="charge-1")]])
agent = Agent(
name="agent",
model=model,
tools=[charge],
tool_use_behavior=StopAtTools(stop_at_tool_names=["charge"]),
output_guardrails=[outer_policy],
)
hooks = EndHook() if failure == "end_hook" else None
paused = await Runner.run(agent, "charge once", run_config=config, hooks=hooks)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
failed = Runner.run_streamed(agent, state, run_config=config, hooks=hooks)
with pytest.raises(OutputGuardrailTripwireTriggered) as caught:
async for event in failed.stream_events():
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem):
emitted.append(event.item.output)
assert _is_error_data_redacted(caught.value)
assert caught.value.run_data is None
assert "inner-private-candidate" not in str(caught.value)
assert emitted == []
assert checked == []
assert failed.tool_output_guardrail_results == []
assert not any(isinstance(item, ToolCallOutputItem) for item in failed.new_items)
for mode in modes:
assert not any(
item.get("type") == "function_call_output" for item in failed.to_input_list(mode=mode)
)
restored = await RunState.from_json(agent, json.loads(json.dumps(failed.to_state().to_json())))
assert [result.output.output_info for result in restored._tool_input_guardrail_results] == [
"approval input checked"
]
retry = Runner.run_streamed(agent, restored, run_config=config, hooks=hooks)
async for event in retry.stream_events():
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem):
emitted.append(event.item.output)
assert checked == ["receipt-7"]
assert retry.final_output == "receipt-7"
assert effects == ["charged"]
assert callback_calls == [failure]
assert len(model.calls) == 1
assert len(inner_model.calls) == 1
assert emitted == ["receipt-7"]
assert [result.output.output_info for result in retry.tool_output_guardrail_results] == (
["receipt-7"] if with_output_guardrail else []
)
assert [result.output.output_info for result in retry.tool_input_guardrail_results] == [
"approval input checked"
]
assert not any(isinstance(item, ToolCallOutputItem) for item in failed.new_items)
for mode in modes:
assert not any(
item.get("type") == "function_call_output" for item in failed.to_input_list(mode=mode)
)
@pytest.mark.asyncio
async def test_streamed_preapproval_checkpoint_preserves_input_guardrail_results() -> None:
@tool_input_guardrail
def check_input(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info="input checked before approval")
@function_tool(needs_approval=True, tool_input_guardrails=[check_input])
def charge() -> str:
raise AssertionError("An unapproved tool must not run")
model = ScriptedModel([[get_function_tool_call("charge", "{}", call_id="charge-1")]])
agent = Agent(name="agent", model=model, tools=[charge])
result = Runner.run_streamed(
agent,
"charge once",
run_config=RunConfig(
tracing_disabled=True,
tool_execution=ToolExecutionConfig(pre_approval_tool_input_guardrails=True),
),
)
async for _ in result.stream_events():
pass
assert len(result.interruptions) == 1
restored = await RunState.from_json(agent, json.loads(json.dumps(result.to_state().to_json())))
assert [item.output.output_info for item in restored._tool_input_guardrail_results] == [
"input checked before approval"
]
@pytest.mark.asyncio
async def test_cancelled_approval_callback_stream_checkpoint_retries_accepted_output() -> None:
effects: list[str] = []
callbacks: list[str] = []
checked_outputs: list[str] = []
def cancel_after_execution(_context: Any) -> dict[str, Any]:
callbacks.append("extractor")
raise asyncio.CancelledError("post-tool callback cancelled")
@tool_output_guardrail
def record_output(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
return ToolGuardrailFunctionOutput.allow(output_info=data.output)
@function_tool(
needs_approval=True,
custom_data_extractor=cancel_after_execution,
tool_output_guardrails=[record_output],
)
async def charge() -> str:
effects.append("charged")
return "receipt-7"
@output_guardrail
async def allow_output(_context: Any, _agent: Any, output: Any) -> GuardrailFunctionOutput:
checked_outputs.append(output)
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False)
model = ScriptedModel([[get_function_tool_call("charge", "{}", call_id="charge-1")]])
agent = Agent(
name="agent",
model=model,
tools=[charge],
tool_use_behavior=StopAtTools(stop_at_tool_names=["charge"]),
output_guardrails=[allow_output],
)
config = RunConfig(tracing_disabled=True)
paused = await Runner.run(agent, "charge once", run_config=config)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
failed = Runner.run_streamed(agent, state, run_config=config)
failed_outputs = [
event.item.output
async for event in failed.stream_events()
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem)
]
# Background cancellation completes the stream silently; consumer cancellation still raises.
assert failed.is_complete
assert failed.run_loop_exception is None
assert failed.final_output is None
assert failed_outputs == []
assert checked_outputs == []
assert failed.tool_output_guardrail_results == []
restored = await RunState.from_json(agent, json.loads(json.dumps(failed.to_state().to_json())))
assert [
item.output for item in restored._generated_items if isinstance(item, ToolCallOutputItem)
] == ["receipt-7"]
assert [result.output.output_info for result in restored._tool_output_guardrail_results] == [
"receipt-7"
]
retry = Runner.run_streamed(agent, restored, run_config=config)
retry_outputs = [
event.item.output
async for event in retry.stream_events()
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem)
]
assert retry.final_output == "receipt-7"
assert retry_outputs == ["receipt-7"]
assert checked_outputs == ["receipt-7"]
assert effects == ["charged"]
assert callbacks == ["extractor"]
assert len(model.calls) == 1
assert _call_pair(retry.to_input_list(), "charge-1") == [
"function_call",
"function_call_output",
]
assert [result.output.output_info for result in retry.tool_output_guardrail_results] == [
"receipt-7"
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failing_streamed,retry_streamed,round_trip,retry_append",
[
(False, False, False, "success"),
(True, True, True, "success"),
(False, True, True, "lost-ack"),
(True, False, False, "lost-ack"),
],
ids=["run-live", "stream-json", "run-to-stream-lost-ack", "stream-to-run-lost-ack"],
)
async def test_callback_failure_retry_delivers_output_to_session(
failing_streamed: bool,
retry_streamed: bool,
round_trip: bool,
retry_append: str,
) -> None:
agent, model, session, state, effects = await _approved_session_state(failing_streamed)
callback_calls: list[str] = []
class FailingEndHook(RunHooks[Any]):
async def on_tool_end(
self, context: RunContextWrapper[Any], agent: Agent[Any], tool: Tool, result: Any
) -> None:
callback_calls.append(tool.name)
raise RuntimeError("post-tool callback failed")
hooks = FailingEndHook()
if failing_streamed:
failed = Runner.run_streamed(
agent,
state,
session=session,
run_config=RunConfig(tracing_disabled=True),
hooks=hooks,
)
with pytest.raises(UserError, match="post-tool callback failed"):
async for _ in failed.stream_events():
pass
state = failed.to_state()
else:
with pytest.raises(UserError, match="post-tool callback failed"):
await _run_session_resume(agent, state, session, False, hooks)
assert effects == [7]
assert callback_calls == ["charge"]
assert len(model.calls) == 1
assert _charge_pair(await session.get_items()) == ["function_call"]
if round_trip:
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
if retry_append == "lost-ack":
session.failure = "after"
with pytest.raises(RuntimeError) as error:
await _run_session_resume(agent, state, session, retry_streamed, hooks)
assert error.value is session.error
assert len(model.calls) == 1
assert effects == [7]
assert callback_calls == ["charge"]
assert _charge_pair(await session.get_items()) == [
"function_call",
"function_call_output",
]
# Also serialize while the Session acknowledgement is outstanding.
if round_trip:
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
result = await _run_session_resume(agent, state, session, retry_streamed, hooks)
expected_pair = ["function_call", "function_call_output"]
assert result.final_output == "done"
assert effects == [7]
assert callback_calls == ["charge"]
assert _charge_pair(result.to_input_list()) == expected_pair
assert _charge_pair(await session.get_items()) == expected_pair
assert _charge_pair(model.calls[-1].input) == expected_pair
await _run_session_resume(agent, "What was the receipt?", session, retry_streamed)
assert _charge_pair(model.calls[-1].input) == expected_pair
assert effects == [7]
async def _approved_session_state(streamed: bool, session: Session | None = None):
effects: list[int] = []
@tool(needs_approval=True)
async def charge(amount: int) -> str:
effects.append(amount)
return "receipt-7"
model = ScriptedModel(
[
[get_function_tool_call("charge", '{"amount":7}', call_id="charge-1")],
[get_text_message("done")],
[get_text_message("fresh")],
]
)
agent = Agent(name="payment", model=model, tools=[charge])
session = session if session is not None else _FailingResumeSession()
paused = await _run_session_resume(agent, "charge 7", session, streamed)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
return agent, model, session, state, effects
def _charge_pair(items: list[TResponseInputItem]) -> list[str]:
return [
str(item.get("type"))
for item in items
if isinstance(item, dict) and item.get("call_id") == "charge-1"
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failing_streamed,retry_streamed", [(False, False), (False, True), (True, False), (True, True)]
)
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
@pytest.mark.parametrize("failure", ["before", "after"], ids=["atomic-failure", "lost-ack"])
async def test_resumed_session_append_is_recovered_before_next_model(
failing_streamed: bool, retry_streamed: bool, round_trip: bool, failure: str
) -> None:
agent, model, session, state, effects = await _approved_session_state(failing_streamed)
session.failure = failure
with pytest.raises(RuntimeError) as error:
await _run_session_resume(agent, state, session, failing_streamed)
assert error.value is session.error
assert effects == [7]
assert len(model.calls) == 1
if round_trip:
state = await RunState.from_json(agent, state.to_json())
result = await _run_session_resume(agent, state, session, retry_streamed)
assert result.final_output == "done"
assert effects == [7]
expected_pair = ["function_call", "function_call_output"]
assert _charge_pair(await session.get_items()) == expected_pair
assert _charge_pair(result.to_input_list()) == expected_pair
await _run_session_resume(agent, "What was the receipt?", session, retry_streamed)
assert _charge_pair(model.calls[-1].input) == expected_pair
assert "pending_session_write" not in result.to_state().to_json()
@pytest.mark.asyncio
@pytest.mark.parametrize("retry_streamed", [False, True])
@pytest.mark.parametrize("mismatch", ["missing", "different-id", "changed-tail"])
async def test_resumed_session_append_rejects_ambiguous_recovery(
retry_streamed: bool, mismatch: str
) -> None:
agent, model, session, state, effects = await _approved_session_state(False)
session.failure = "before"
with pytest.raises(RuntimeError, match="session append failed"):
await _run_session_resume(agent, state, session, False)
state = await RunState.from_json(agent, state.to_json())
supplied_session: Session | None = session
if mismatch == "missing":
supplied_session = None
elif mismatch == "different-id":
supplied_session = SimpleListSession("other", await session.get_items())
else:
await session.add_items([{"role": "user", "content": "another writer"}])
before = await session.get_items()
with pytest.raises(UserError, match="pending Session write"):
await _run_session_resume(agent, state, supplied_session, retry_streamed)
assert len(model.calls) == 1
assert effects == [7]
assert await session.get_items() == before
@pytest.mark.asyncio
async def test_resumed_session_append_survives_repeated_failure_and_late_input() -> None:
agent, model, session, state, effects = await _approved_session_state(False)
for _ in range(2):
session.failure = "before"
with pytest.raises(RuntimeError, match="session append failed"):
await _run_session_resume(agent, state, session, False)
state = await RunState.from_json(agent, state.to_json())
assert len(model.calls) == 1
assert effects == [7]
state.add_input("What was the receipt?")
result = await _run_session_resume(agent, state, session, True)
assert result.final_output == "done"
stored = await session.get_items()
output_index = next(
i for i, item in enumerate(stored) if item.get("type") == "function_call_output"
)
late_index = next(
i for i, item in enumerate(stored) if item.get("content") == "What was the receipt?"
)
assert output_index < late_index
assert effects == [7]
async def _partially_approved_session_state(streamed: bool):
"""Pause on two approval-gated calls in one response and approve only the first."""
effects: list[int] = []
@tool(needs_approval=True)
async def charge(amount: int) -> str:
effects.append(amount)
return "receipt-7"
@tool(needs_approval=True)
async def notify() -> str:
raise AssertionError("the unresolved approval must not execute")
model = ScriptedModel(
[
[
get_function_tool_call("charge", '{"amount":7}', call_id="charge-1"),
get_function_tool_call("notify", "{}", call_id="notify-1"),
],
[get_text_message("done")],
]
)
agent = Agent(name="payment", model=model, tools=[charge, notify])
session = _FailingResumeSession()
paused = await _run_session_resume(agent, "charge 7 and notify", session, streamed)
state = paused.to_state()
charge_approval = next(
item for item in state.get_interruptions() if item.raw_item.call_id == "charge-1"
)
state.approve(charge_approval)
return agent, model, session, state, effects
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
async def test_renewed_interruption_recovers_failed_resumed_session_append(
streamed: bool, round_trip: bool
) -> None:
agent, model, session, state, effects = await _partially_approved_session_state(streamed)
session.failure = "before"
with pytest.raises(RuntimeError) as error:
await _run_session_resume(agent, state, session, streamed)
assert error.value is session.error
assert effects == [7]
assert _charge_pair(await session.get_items()) == ["function_call"]
if round_trip:
state = await RunState.from_json(agent, state.to_json())
pending = await _run_session_resume(agent, state, session, streamed)
pending_state = pending.to_state()
remaining = pending_state.get_interruptions()
assert [item.raw_item.call_id for item in remaining] == ["notify-1"]
assert len(model.calls) == 1
pending_state.reject(remaining[0], rejection_message="declined")
result = await _run_session_resume(agent, pending_state, session, streamed)
assert result.final_output == "done"
assert effects == [7]
expected_pair = ["function_call", "function_call_output"]
assert _charge_pair(await session.get_items()) == expected_pair
assert _charge_pair(result.to_input_list()) == expected_pair
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
async def test_resumed_committed_append_refreshes_compaction_input(
streamed: bool, round_trip: bool, tmp_path: Path
) -> None:
backend = _LostAckSQLiteSession("compaction-recovery", tmp_path / "history.db")
compaction_inputs: list[list[TResponseInputItem]] = []
compact_enabled = False
async def compact(**kwargs: Any) -> SimpleNamespace:
items = copy.deepcopy(kwargs["input"])
compaction_inputs.append(items)
return SimpleNamespace(output=items, usage=None)
session = OpenAIResponsesCompactionSession(
backend.session_id,
underlying_session=backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode="input",
should_trigger_compaction=lambda _: compact_enabled,
)
try:
agent, model, _, state, effects = await _approved_session_state(streamed, session)
# A normal declined compaction initializes the retained wrapper's history cache.
await session.run_compaction()
assert compaction_inputs == []
backend.fail_after_commit = True
with pytest.raises(RuntimeError) as error:
await _run_session_resume(agent, state, session, streamed)
assert error.value is backend.error
expected_pair = ["function_call", "function_call_output"]
assert _charge_pair(await backend.get_items(limit=100)) == expected_pair
if round_trip:
state = await RunState.from_json(agent, state.to_json())
compact_enabled = True
result = await _run_session_resume(agent, state, session, streamed)
assert result.final_output == "done"
assert effects == [7]
assert len(model.calls) == 2
assert len(compaction_inputs) == 1
assert _charge_pair(compaction_inputs[0]) == expected_pair
assert _charge_pair(await backend.get_items(limit=100)) == expected_pair
assert _charge_pair(result.to_input_list()) == expected_pair
assert "pending_session_write" not in result.to_state().to_json()
finally:
backend.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["input", "auto"])
async def test_compaction_reload_preserves_session_retrieval_window(
mode: Literal["input", "auto"], tmp_path: Path
) -> None:
backend = _LostAckSQLiteSession(
"bounded-compaction", tmp_path / "history.db", session_settings={"limit": 1}
)
compaction_inputs: list[list[TResponseInputItem]] = []
async def compact(**kwargs: Any) -> SimpleNamespace:
assert "previous_response_id" not in kwargs
items = copy.deepcopy(kwargs["input"])
compaction_inputs.append(items)
return SimpleNamespace(output=items, usage=None)
session = OpenAIResponsesCompactionSession(
backend.session_id,
underlying_session=backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode=mode,
)
old_items: list[TResponseInputItem] = [
{"role": "assistant", "content": f"old message {index}"} for index in range(12)
]
recovered_item: TResponseInputItem = {"role": "assistant", "content": "committed reply"}
try:
await backend.add_items(old_items)
# The configured window has one candidate, so the default threshold is not met.
await session.run_compaction({"response_id": "unstored-response", "store": False})
assert compaction_inputs == []
assert await backend.get_items(limit=100) == old_items
backend.fail_after_commit = True
with pytest.raises(RuntimeError) as error:
await session.add_items([recovered_item])
assert error.value is backend.error
assert await backend.get_items(limit=100) == [*old_items, recovered_item]
await session.run_compaction({"force": True, "store": False})
assert compaction_inputs == [[recovered_item]]
assert await backend.get_items(limit=100) == [recovered_item]
finally:
backend.close()
@pytest.mark.asyncio
async def test_cancelled_compaction_append_preserves_committed_and_surviving_writes() -> None:
appended = asyncio.Event()
wait_for_ack = asyncio.Event()
class DelayedAckSession(SimpleListSession):
delay_next_ack = True
async def add_items(self, items: list[TResponseInputItem]) -> None:
await super().add_items(items)
if self.delay_next_ack:
self.delay_next_ack = False
appended.set()
await wait_for_ack.wait()
backend = DelayedAckSession()
compaction_inputs: list[list[TResponseInputItem]] = []
async def compact(**kwargs: Any) -> SimpleNamespace:
items = copy.deepcopy(kwargs["input"])
compaction_inputs.append(items)
return SimpleNamespace(output=items, usage=None)
session = OpenAIResponsesCompactionSession(
backend.session_id,
underlying_session=backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode="input",
should_trigger_compaction=lambda _: False,
)
await session.run_compaction()
first_item: TResponseInputItem = {"role": "user", "content": "committed before cancellation"}
newer_item: TResponseInputItem = {"role": "user", "content": "surviving writer"}
first = asyncio.create_task(session.add_items([first_item]))
newer: asyncio.Task[None] | None = None
newer_started = asyncio.Event()
async def write_newer() -> None:
newer_started.set()
await session.add_items([newer_item])
try:
await asyncio.wait_for(appended.wait(), timeout=5)
newer = asyncio.create_task(write_newer())
await asyncio.wait_for(newer_started.wait(), timeout=5)
assert not newer.done()
first.cancel()
with pytest.raises(asyncio.CancelledError):
await first
await asyncio.wait_for(newer, timeout=5)
await session.run_compaction({"force": True})
assert compaction_inputs == [[first_item, newer_item]]
assert await backend.get_items() == [first_item, newer_item]
finally:
wait_for_ack.set()
tasks = [first, *([newer] if newer is not None else [])]
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_resumed_session_append_cancellation_retains_recoverable_state(
streamed: bool,
) -> None:
agent, model, session, state, effects = await _approved_session_state(streamed)
session.block_next_add = True
attempt = asyncio.create_task(_run_session_resume(agent, state, session, streamed))
try:
await asyncio.wait_for(session.add_started.wait(), timeout=5)
with pytest.raises(UserError, match="pending Session write is already in progress"):
await _run_session_resume(agent, state, session, not streamed)
assert len(model.calls) == 1
attempt.cancel()
with pytest.raises(asyncio.CancelledError):
await attempt
finally:
session.release_add.set()
if not attempt.done():
attempt.cancel()
await asyncio.gather(attempt, return_exceptions=True)
restored = await RunState.from_json(agent, state.to_json())
result = await _run_session_resume(agent, restored, session, not streamed)
assert result.final_output == "done"
assert effects == [7]
assert _charge_pair(await session.get_items()) == ["function_call", "function_call_output"]
@pytest.mark.asyncio
async def test_failed_streamed_result_checkpoint_retains_detached_pending_write() -> None:
agent, model, session, state, effects = await _approved_session_state(True)
session.failure = "before"
result = Runner.run_streamed(agent, state, session=session)
with pytest.raises(RuntimeError, match="session append failed"):
async for _ in result.stream_events():
pass
snapshot = result.to_state()
payload = snapshot.to_json()
payload["pending_session_write"]["items"][0]["output"] = "changed snapshot"
assert state.to_json()["pending_session_write"]["items"][0]["output"] == "receipt-7"
assert snapshot.to_json()["pending_session_write"]["items"][0]["output"] == "receipt-7"
await _run_session_resume(agent, snapshot, session, False)
assert effects == [7]
assert len(model.calls) == 2
assert _charge_pair(await session.get_items()) == ["function_call", "function_call_output"]
def _relabel_as_older_schema(payload: dict[str, Any], version: str) -> None:
"""Downgrade only what the destination label cannot represent, then relabel.
Agent-scoped approvals and MCP recipient bindings arrived in 1.18, which v0.23.0
released, so a 1.18 payload keeps them. Rewriting them unconditionally would hand
the reader a 1.17-shaped payload wearing a newer label, and a test built on that
would stay green even if the newer reader lost the fields.
"""
if tuple(int(part) for part in version.split(".", maxsplit=1)) < (1, 18):
for entry in payload["context"].pop("function_tool_approvals", []):
payload["context"]["approvals"][entry["tool_key"]] = entry["decision"]
response = payload.get("last_processed_response")
if isinstance(response, dict):
response.pop("mcp_tool_bindings", None)
payload["$schemaVersion"] = version
@pytest.mark.asyncio
@pytest.mark.parametrize(
"invalid",
[
"old-schema",
"batch-shape",
"compaction-under-1.17",
"held-shape",
"held-with-before",
"held-under-1-17",
"held-under-1-18",
"held-keys-without-held",
"policy-shape",
"response-boundary-shape",
"response-boundary-missing",
"response-boundary-outside-batch",
"acknowledgement-type",
"acknowledgement-without-before",
"compaction-exchange-type",
"compaction-exchange-policy",
"acknowledgement-under-1.17",
],
)
async def test_pending_session_write_rejects_invalid_serialized_checkpoint(invalid: str) -> None:
agent, _, session, state, _ = await _approved_session_state(False)
session.failure = "before"
with pytest.raises(RuntimeError):
await _run_session_resume(agent, state, session, False)
payload = state.to_json()
if invalid == "old-schema":
_relabel_as_older_schema(payload, "1.16")
elif invalid == "compaction-under-1.17":
_relabel_as_older_schema(payload, "1.17")
elif invalid == "acknowledgement-type":
payload["pending_session_write"]["append_acknowledged"] = "true"
elif invalid == "acknowledgement-without-before":
payload["pending_session_write"]["append_acknowledged"] = True
payload["pending_session_write"]["before"] = None
elif invalid == "compaction-exchange-type":
payload["pending_session_write"]["compaction_model_exchange"] = {
"item_digests": "not a digest list",
"reasoning_item_id_policy": None,
}
elif invalid == "compaction-exchange-policy":
payload["pending_session_write"]["compaction_model_exchange"] = {
"item_digests": [],
"reasoning_item_id_policy": "unknown",
}
elif invalid == "acknowledgement-under-1.17":
_relabel_as_older_schema(payload, "1.17")
for key in ("response_id", "store", "has_local_tool_outputs"):
payload["pending_session_write"].pop(key)
payload["pending_session_write"]["append_acknowledged"] = True
elif invalid == "batch-shape":
payload["pending_session_write"]["items"] = "not an item batch"
elif invalid == "held-shape":
payload["pending_session_write"]["held"] = "yes"
elif invalid == "held-keys-without-held":
# Conversion policy belongs only to a withheld response.
payload["pending_session_write"]["reasoning_item_id_policy"] = "omit"
elif invalid == "policy-shape":
# The conversion-policy key only speaks the two policy literals or None; any
# other value would silently change how a fold converts the batch's items.
payload["pending_session_write"]["held"] = True
payload["pending_session_write"]["before"] = None
payload["pending_session_write"]["reasoning_item_id_policy"] = "banana"
elif invalid.startswith("response-boundary-"):
# The boundary selects the response governed by the current filter.
payload["pending_session_write"]["held"] = True
payload["pending_session_write"]["before"] = None
if invalid == "response-boundary-shape":
boundary = {"turn": -1, "start": 0}
else:
boundary = {
"turn": state._current_turn,
"start": len(payload["pending_session_write"]["items"]) + 1,
}
if invalid != "response-boundary-missing":
payload["pending_session_write"]["current_response"] = boundary
elif invalid == "held-under-1-17":
# 1.17 defined the pending write as exactly four keys, so the held variant is
# only readable under the version that introduced it.
_relabel_as_older_schema(payload, "1.17")
payload["pending_session_write"]["held"] = True
payload["pending_session_write"]["before"] = None
elif invalid == "held-under-1-18":
# v0.23.0 released 1.18 with the compaction metadata only. Its reader rejects the
# held keys, so a well-formed held record under that label would misidentify the
# durable format; the held variant is readable only under the version that
# introduced it.
_relabel_as_older_schema(payload, "1.18")
payload["pending_session_write"]["held"] = True
payload["pending_session_write"]["before"] = None
payload["pending_session_write"]["current_response"] = {
"turn": state._current_turn,
"start": 0,
}
else:
# A held batch was never offered to the Session, so recorded digests and the
# held marker cannot coexist on one record.
payload["pending_session_write"]["held"] = True
with pytest.raises(UserError, match="pending Session write is invalid"):
await RunState.from_json(agent, payload)
@pytest.mark.asyncio
async def test_pending_session_write_without_the_held_key_keeps_its_meaning() -> None:
# The four-key form 1.17 defined still settles eagerly on resume entry under its
# own label, unchanged by the held variant that 1.18 introduced.
agent, model, session, state, effects = await _approved_session_state(False)
session.failure = "before"
with pytest.raises(RuntimeError):
await _run_session_resume(agent, state, session, False)
payload = state.to_json()
assert "held" not in payload["pending_session_write"]
_relabel_as_older_schema(payload, "1.17")
for key in ("response_id", "store", "has_local_tool_outputs"):
payload["pending_session_write"].pop(key, None)
restored = await RunState.from_json(agent, payload)
result = await _run_session_resume(agent, restored, session, False)
assert result.final_output == "done"
assert effects == [7]
assert _charge_pair(await session.get_items()) == ["function_call", "function_call_output"]
@pytest.mark.asyncio
@pytest.mark.parametrize("acknowledged", [False, True], ids=["in-flight", "acknowledged"])
async def test_compaction_metadata_stays_readable_under_1_18(acknowledged: bool) -> None:
# v0.23.0 shipped 1.18 with the compaction metadata on the pending write. That
# released format must keep loading under its own label, independently of the held
# keys that only a later version introduces.
agent, model, session, state, effects = await _approved_session_state(False)
session.failure = "before"
with pytest.raises(RuntimeError):
await _run_session_resume(agent, state, session, False)
payload = state.to_json()
pending = payload["pending_session_write"]
assert "held" not in pending
assert {"response_id", "store", "has_local_tool_outputs"} <= set(pending)
_relabel_as_older_schema(payload, "1.18")
if acknowledged:
pending["append_acknowledged"] = True
pending["compaction_model_exchange"] = {
"item_digests": list(pending["before"]),
"reasoning_item_id_policy": None,
}
restored = await RunState.from_json(agent, payload)
assert restored._pending_session_write is not None
assert restored._pending_session_write.get("append_acknowledged", False) is acknowledged
if acknowledged:
return
result = await _run_session_resume(agent, restored, session, False)
assert result.final_output == "done"
assert effects == [7]
assert _charge_pair(await session.get_items()) == ["function_call", "function_call_output"]
@pytest.mark.asyncio
async def test_resumed_session_append_partial_commit_fails_closed() -> None:
agent, model, session, state, effects = await _approved_session_state(False)
# Two approved calls produce one resumed batch, allowing an actual partial append.
second_call = get_function_tool_call("charge", '{"amount":7}', call_id="charge-2")
model = ScriptedModel(
[
[get_function_tool_call("charge", '{"amount":7}', call_id="charge-1"), second_call],
[get_text_message("done")],
]
)
agent.model = model
session = _FailingResumeSession()
paused = await _run_session_resume(agent, "charge twice", session, False)
state = paused.to_state()
for interruption in state.get_interruptions():
state.approve(interruption)
session.failure = "partial"
with pytest.raises(RuntimeError, match="session append failed"):
await _run_session_resume(agent, state, session, False)
before = await session.get_items()
restored = await RunState.from_json(agent, state.to_json())
with pytest.raises(UserError, match="history changed or is ambiguous"):
await _run_session_resume(agent, restored, session, True)
assert effects == [7, 7]
assert len(model.calls) == 1
assert await session.get_items() == before
@pytest.mark.asyncio
async def test_resolve_interrupted_turn_final_output_short_circuit(monkeypatch) -> None:
agent: Agent[dict[str, str]] = make_agent(model=ScriptedModel())
context_wrapper = make_context_wrapper()
async def fake_execute_tool_plan(*_: object, **__: object):
return [], [], [], [], [], [], [], []
async def fake_check_for_final_output_from_tools(*_: object, **__: object):
return ToolsToFinalOutputResult(is_final_output=True, final_output="done")
async def fake_execute_final_output(
*,
original_input,
new_response,
pre_step_items,
new_step_items,
final_output,
tool_input_guardrail_results,
tool_output_guardrail_results,
**__: object,
) -> SingleStepResult:
return SingleStepResult(
original_input=original_input,
model_response=new_response,
pre_step_items=pre_step_items,
new_step_items=new_step_items,
next_step=NextStepFinalOutput(final_output),
tool_input_guardrail_results=tool_input_guardrail_results,
tool_output_guardrail_results=tool_output_guardrail_results,
)
monkeypatch.setattr(
turn_resolution, "check_for_final_output_from_tools", fake_check_for_final_output_from_tools
)
monkeypatch.setattr(turn_resolution, "execute_final_output", fake_execute_final_output)
monkeypatch.setattr(turn_resolution, "_execute_tool_plan", fake_execute_tool_plan)
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await run_loop.resolve_interrupted_turn(
bindings=bind_public_agent(agent),
original_input="input",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=None,
)
assert isinstance(result, SingleStepResult)
assert isinstance(result.next_step, NextStepFinalOutput)
assert result.next_step.output == "done"
@pytest.mark.asyncio
async def test_resumed_session_persistence_uses_saved_count(monkeypatch) -> None:
agent = Agent(name="resume-agent")
context_wrapper: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
state = RunState(
context=context_wrapper,
original_input="input",
starting_agent=agent,
max_turns=1,
)
session = SimpleListSession()
raw_output = {"type": "function_call_output", "call_id": "call-1", "output": "ok"}
item_1 = ToolCallOutputItem(agent=agent, raw_item=raw_output, output="ok")
item_2 = ToolCallOutputItem(agent=agent, raw_item=dict(raw_output), output="ok")
step = SingleStepResult(
original_input="input",
model_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
pre_step_items=[],
new_step_items=[item_1, item_2],
next_step=NextStepFinalOutput("done"),
tool_input_guardrail_results=[],
tool_output_guardrail_results=[],
)
async def fake_run_single_turn(**_kwargs):
return step
monkeypatch.setattr(run_module, "run_single_turn", fake_run_single_turn)
runner = run_module.AgentRunner()
await runner.run(agent, state, session=session, run_config=RunConfig())
assert state._current_turn_persisted_item_count == 1
assert len(session.saved_items) == 1
@pytest.mark.asyncio
async def test_resumed_run_again_resets_persisted_count(monkeypatch) -> None:
agent = Agent(name="resume-agent")
context_wrapper: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
state = RunState(
context=context_wrapper,
original_input="input",
starting_agent=agent,
max_turns=2,
)
session = SimpleListSession()
state._current_step = NextStepInterruption(interruptions=[])
state._model_responses = [
ModelResponse(output=[], usage=Usage(), response_id="resp_1"),
]
state._last_processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
state._current_turn_persisted_item_count = 1
async def fake_resolve_interrupted_turn(**_kwargs):
return SingleStepResult(
original_input="input",
model_response=ModelResponse(output=[], usage=Usage(), response_id="resp_resume"),
pre_step_items=[],
new_step_items=[],
next_step=NextStepRunAgain(),
tool_input_guardrail_results=[],
tool_output_guardrail_results=[],
)
async def fake_run_single_turn(**_kwargs):
tool_call = cast(
ResponseFunctionToolCall,
get_function_tool_call("test_tool", "{}", call_id="call-1"),
)
tool_call_item = ToolCallItem(agent=agent, raw_item=tool_call)
tool_output_item = ToolCallOutputItem(
agent=agent,
raw_item={
"type": "function_call_output",
"call_id": "call-1",
"output": "ok",
},
output="ok",
)
message_item = MessageOutputItem(
agent=agent,
raw_item=cast(ResponseOutputMessage, get_text_message("final")),
)
return SingleStepResult(
original_input="input",
model_response=ModelResponse(
output=[get_text_message("final")],
usage=Usage(),
response_id="resp_final",
),
pre_step_items=[],
new_step_items=[tool_call_item, tool_output_item, message_item],
next_step=NextStepFinalOutput("done"),
tool_input_guardrail_results=[],
tool_output_guardrail_results=[],
)
monkeypatch.setattr(run_module, "resolve_interrupted_turn", fake_resolve_interrupted_turn)
monkeypatch.setattr(run_module, "run_single_turn", fake_run_single_turn)
runner = run_module.AgentRunner()
result = await runner.run(agent, state, session=session, run_config=RunConfig())
assert result.final_output == "done"
saved_types = [
item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
for item in session.saved_items
]
assert "function_call" in saved_types
@pytest.mark.asyncio
@pytest.mark.parametrize("continuation", ["run_again", "handoff"])
async def test_resumed_stream_waits_for_event_consumption_before_continuing(
monkeypatch: pytest.MonkeyPatch,
continuation: str,
) -> None:
agent = Agent(name="resume-agent")
delegate = Agent(name="delegate", output_type=int)
state: RunState[dict[str, str]] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=agent,
max_turns=2,
)
state._current_step = NextStepInterruption(interruptions=[])
state._model_responses = [
ModelResponse(output=[], usage=Usage(), response_id="resp_1"),
]
state._last_processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
tool_output_item = ToolCallOutputItem(
agent=agent,
raw_item={
"type": "function_call_output",
"call_id": "call-resume",
"output": "ok",
},
output="ok",
)
next_step = NextStepHandoff(delegate) if continuation == "handoff" else NextStepRunAgain()
allow_resume_resolution = asyncio.Event()
async def fake_resolve_interrupted_turn(**_kwargs: object) -> SingleStepResult:
await allow_resume_resolution.wait()
return SingleStepResult(
original_input="input",
model_response=ModelResponse(output=[], usage=Usage(), response_id="resp_resume"),
pre_step_items=[],
new_step_items=[tool_output_item],
next_step=next_step,
tool_input_guardrail_results=[],
tool_output_guardrail_results=[],
)
next_model_turn_started = asyncio.Event()
allow_model_turn_to_finish = asyncio.Event()
async def fake_run_single_turn_streamed(*_args: object, **_kwargs: object) -> SingleStepResult:
next_model_turn_started.set()
await allow_model_turn_to_finish.wait()
return SingleStepResult(
original_input="input",
model_response=ModelResponse(output=[], usage=Usage(), response_id="unexpected"),
pre_step_items=[],
new_step_items=[],
next_step=NextStepFinalOutput("unexpected"),
tool_input_guardrail_results=[],
tool_output_guardrail_results=[],
)
monkeypatch.setattr(run_loop, "resolve_interrupted_turn", fake_resolve_interrupted_turn)
monkeypatch.setattr(run_loop, "run_single_turn_streamed", fake_run_single_turn_streamed)
result = Runner.run_streamed(agent, state)
consumer_active = asyncio.Event()
consumer_suspended = asyncio.Event()
release_consumer = asyncio.Event()
cancel_called = asyncio.Event()
async def consume_events() -> None:
async for event in result.stream_events():
if event.type == "agent_updated_stream_event":
consumer_active.set()
if event.type == "run_item_stream_event" and event.name == "tool_output":
consumer_suspended.set()
await release_consumer.wait()
result.cancel(mode="after_turn")
cancel_called.set()
consumer_task = asyncio.create_task(consume_events())
await asyncio.wait_for(consumer_active.wait(), timeout=1)
allow_resume_resolution.set()
await asyncio.wait_for(consumer_suspended.wait(), timeout=1)
await asyncio.sleep(0)
await asyncio.sleep(0)
assert not next_model_turn_started.is_set()
release_consumer.set()
await asyncio.wait_for(cancel_called.wait(), timeout=1)
allow_model_turn_to_finish.set()
await asyncio.wait_for(consumer_task, timeout=1)
assert not next_model_turn_started.is_set()
assert result.final_output is None
expected_agent = delegate if continuation == "handoff" else agent
assert result.current_agent is expected_agent
assert result.last_agent is expected_agent
assert result.to_state()._current_agent is expected_agent
if continuation == "handoff":
assert result._current_agent_output_schema is not None
assert isinstance(result._current_agent_output_schema, AgentOutputSchema)
assert result._current_agent_output_schema.output_type is int
@pytest.mark.parametrize(
("conversation_id", "previous_response_id", "auto_previous_response_id"),
[
("conv_1", None, False),
(None, "resp_prev", False),
(None, None, True),
],
)
@pytest.mark.asyncio
async def test_resumed_interruption_passes_server_managed_conversation_flag(
monkeypatch: pytest.MonkeyPatch,
conversation_id: str | None,
previous_response_id: str | None,
auto_previous_response_id: bool,
) -> None:
agent = Agent(name="resume-agent")
context_wrapper: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={})
state = RunState(
context=context_wrapper,
original_input="input",
starting_agent=agent,
max_turns=1,
conversation_id=conversation_id,
previous_response_id=previous_response_id,
auto_previous_response_id=auto_previous_response_id,
)
state._current_step = NextStepInterruption(interruptions=[])
state._model_responses = [
ModelResponse(output=[], usage=Usage(), response_id="resp_1"),
]
state._last_processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
server_managed_values: list[bool] = []
async def fake_resolve_interrupted_turn(**kwargs: object) -> SingleStepResult:
server_managed_values.append(cast(bool, kwargs["server_manages_conversation"]))
return SingleStepResult(
original_input="input",
model_response=ModelResponse(output=[], usage=Usage(), response_id="resp_resume"),
pre_step_items=[],
new_step_items=[],
next_step=NextStepFinalOutput("done"),
tool_input_guardrail_results=[],
tool_output_guardrail_results=[],
)
monkeypatch.setattr(run_module, "resolve_interrupted_turn", fake_resolve_interrupted_turn)
runner = run_module.AgentRunner()
result = await runner.run(agent, state, run_config=RunConfig())
assert result.final_output == "done"
assert server_managed_values == [True]
def _sent_tool_outputs(model: ScriptedModel, *, first_call_index: int) -> list[tuple[str, str]]:
"""Collect the tool outputs the model received, from `first_call_index` onward."""
outputs: list[tuple[str, str]] = []
for call in model.calls[first_call_index:]:
for item in cast(list[dict[str, Any]], call.input):
if item.get("type") == "function_call_output":
outputs.append((str(item.get("call_id")), str(item.get("output"))))
return outputs
async def _run_server_managed(
agent: Agent[Any],
agent_input: Any,
*,
run_config: RunConfig,
use_conversation_id: bool,
streaming: bool,
) -> Any:
"""Run the agent under one of the server-managed continuation modes."""
kwargs: dict[str, Any] = (
{"conversation_id": "conv-resume"}
if use_conversation_id
else {"auto_previous_response_id": True}
)
if streaming:
streamed = Runner.run_streamed(agent, agent_input, run_config=run_config, **kwargs)
async for _ in streamed.stream_events():
pass
return streamed
return await Runner.run(agent, agent_input, run_config=run_config, **kwargs)
@pytest.mark.parametrize("streaming", [False, True], ids=["non_streamed", "streamed"])
@pytest.mark.parametrize("serialize_state", [False, True], ids=["live_state", "serialized_state"])
@pytest.mark.parametrize(
"use_conversation_id", [False, True], ids=["auto_previous_response_id", "conversation_id"]
)
@pytest.mark.asyncio
async def test_resumed_server_managed_run_sends_tool_not_found_output(
streaming: bool,
serialize_state: bool,
use_conversation_id: bool,
) -> None:
"""A resumed server-managed run must send the output built for a missing tool.
The interrupted turn answers the unknown tool locally while another call waits for
approval. The server already owns both calls, so resuming has to deliver both outputs.
"""
@function_tool(name_override="needs_ok", needs_approval=True)
async def needs_ok(text: str) -> str:
return text
model = ScriptedModel()
agent = Agent(name="test", model=model, tools=[needs_ok])
model.extend(
[
[
get_function_tool_call(
"needs_ok", json.dumps({"text": "one"}), call_id="call-approval"
),
get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"),
],
[get_text_message("done")],
]
)
run_config = RunConfig(tool_not_found_behavior="return_error_to_model")
async def run_once(agent_input: Any) -> Any:
return await _run_server_managed(
agent,
agent_input,
run_config=run_config,
use_conversation_id=use_conversation_id,
streaming=streaming,
)
first = await run_once("Use needs_ok and missing_tool")
state = first.to_state()
if serialize_state:
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
interruptions = state.get_interruptions()
assert [item.raw_item.call_id for item in interruptions] == ["call-approval"]
state.approve(interruptions[0])
resumed = await run_once(state)
assert resumed.final_output == "done"
delivered = _sent_tool_outputs(model, first_call_index=1)
assert sorted(call_id for call_id, _ in delivered) == ["call-approval", "call-missing"]
assert "missing_tool" in dict(delivered)["call-missing"]
@pytest.mark.asyncio
async def test_resumed_server_managed_run_sends_each_tool_output_once() -> None:
"""Staged approvals must deliver every output exactly once to a server-managed conversation.
Approving one of two gated calls resumes and interrupts again without a model request, so
the same model response stays current across both resumes.
"""
@function_tool(name_override="needs_ok", needs_approval=True)
async def needs_ok(text: str) -> str:
return f"ok:{text}"
model = ScriptedModel()
agent = Agent(name="test", model=model, tools=[needs_ok])
model.extend(
[
[
get_function_tool_call("needs_ok", json.dumps({"text": "a"}), call_id="call-a"),
get_function_tool_call("needs_ok", json.dumps({"text": "b"}), call_id="call-b"),
get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"),
],
[get_text_message("done")],
]
)
run_config = RunConfig(tool_not_found_behavior="return_error_to_model")
async def run_once(agent_input: Any) -> Any:
return await _run_server_managed(
agent,
agent_input,
run_config=run_config,
use_conversation_id=False,
streaming=False,
)
result = await run_once("Use needs_ok twice and missing_tool")
for expected_model_calls in (1, 2):
state = await RunState.from_json(agent, json.loads(json.dumps(result.to_state().to_json())))
interruptions = state.get_interruptions()
assert interruptions
state.approve(interruptions[0])
result = await run_once(state)
# Approving only the first gated call resumes without asking the model again.
assert len(model.calls) == expected_model_calls
assert result.final_output == "done"
delivered = _sent_tool_outputs(model, first_call_index=1)
assert sorted(call_id for call_id, _ in delivered) == ["call-a", "call-b", "call-missing"]
@pytest.mark.asyncio
async def test_resumed_approval_does_not_duplicate_session_items() -> None:
async def test_tool() -> str:
return "tool_result"
tool = function_tool(test_tool, name_override="test_tool", needs_approval=True)
model, agent = make_model_and_agent(name="test", tools=[tool])
session = SimpleListSession()
queue_function_call_and_text(
model,
get_function_tool_call("test_tool", json.dumps({}), call_id="call-resume"),
followup=[get_text_message("done")],
)
first = await Runner.run(agent, input="Use test_tool", session=session)
assert first.interruptions
state = first.to_state()
state.approve(first.interruptions[0])
resumed = await Runner.run(agent, state, session=session)
assert resumed.final_output == "done"
saved_items = await session.get_items()
call_count = sum(
1
for item in saved_items
if isinstance(item, dict)
and item.get("type") == "function_call"
and item.get("call_id") == "call-resume"
)
output_count = sum(
1
for item in saved_items
if isinstance(item, dict)
and item.get("type") == "function_call_output"
and item.get("call_id") == "call-resume"
)
assert call_count == 1
assert output_count == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"decision",
["approve", "reject", "always_approve", "always_reject", "legacy_approve", "legacy_reject"],
)
@pytest.mark.parametrize(
("schema_version", "expect_execution"),
[("1.6", True), ("1.7", False)],
)
async def test_resolve_interrupted_turn_only_uses_name_fallback_for_legacy_approval_agents(
schema_version: str,
expect_execution: bool,
decision: str,
) -> None:
calls: list[str] = []
@function_tool(name_override="needs_ok", needs_approval=True)
async def needs_ok(text: str) -> str:
calls.append(text)
return text
base_duplicate = Agent(name="duplicate", instructions="alpha", tools=[needs_ok])
resumed_duplicate = Agent(name="duplicate", instructions="zeta", tools=[needs_ok])
root = Agent(name="triage", handoffs=[base_duplicate, resumed_duplicate])
base_duplicate.handoffs = [root]
resumed_duplicate.handoffs = [root]
state: RunState[dict[str, str], Agent[Any]] = RunState(
context=RunContextWrapper(context={}),
original_input="input",
starting_agent=root,
max_turns=2,
)
state._current_agent = resumed_duplicate
state._current_step = NextStepInterruption(
interruptions=[
ToolApprovalItem(
agent=resumed_duplicate,
raw_item=cast(
ResponseFunctionToolCall,
get_function_tool_call(
"needs_ok",
json.dumps({"text": "one"}),
call_id="legacy-call",
),
),
)
]
)
state._last_processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
state._model_responses = [ModelResponse(output=[], usage=Usage(), response_id="resp")]
json_data = state.to_json()
current_agent_data = cast(dict[str, str], json_data["current_agent"])
assert current_agent_data["name"] == "duplicate"
assert "identity" in current_agent_data
interruption_data = cast(
dict[str, object],
json_data["current_step"]["data"]["interruptions"][0],
)
interruption_agent_data = cast(dict[str, str], interruption_data["agent"])
assert interruption_agent_data["identity"] == current_agent_data["identity"]
interruption_agent_data.pop("identity")
if tuple(int(part) for part in schema_version.split(".", maxsplit=1)) < (1, 18):
for entry in json_data["context"].pop("function_tool_approvals", []):
json_data["context"]["approvals"][entry["tool_key"]] = entry["decision"]
json_data["$schemaVersion"] = schema_version
if decision.startswith("legacy_"):
json_data["context"]["approvals"]["needs_ok"] = {
"approved": decision == "legacy_approve",
"rejected": decision == "legacy_reject",
"sticky_rejection_message": "Old unowned rejection",
}
restored = await RunState.from_json(root, json_data)
assert restored._schema_version == schema_version
assert restored._current_agent is resumed_duplicate
restored_approval = restored.get_interruptions()[0]
if decision in ("approve", "always_approve"):
restored.approve(restored_approval, always_approve=decision == "always_approve")
elif decision in ("reject", "always_reject"):
restored.reject(
restored_approval,
always_reject=decision == "always_reject",
rejection_message="Legacy exact rejection",
)
assert restored._context is not None
assert restored._last_processed_response is not None
result = await turn_resolution.resolve_interrupted_turn(
bindings=bind_public_agent(cast(Agent[dict[str, str]], restored._current_agent)),
original_input=restored._original_input,
original_pre_step_items=restored._generated_items,
new_response=restored._model_responses[-1],
processed_response=restored._last_processed_response,
hooks=RunHooks(),
context_wrapper=restored._context,
run_config=RunConfig(),
run_state=restored,
)
if expect_execution and decision in ("approve", "always_approve"):
assert isinstance(result.next_step, NextStepRunAgain)
assert calls == ["one"]
assert any(
isinstance(item, ToolCallOutputItem) and item.output == "one"
for item in result.new_step_items
)
elif expect_execution and decision in ("reject", "always_reject"):
assert isinstance(result.next_step, NextStepRunAgain)
assert calls == []
assert any(
isinstance(item, ToolCallOutputItem) and item.output == "Legacy exact rejection"
for item in result.new_step_items
)
else:
assert calls == []
assert not any(
isinstance(item, ToolCallOutputItem) and item.output == "one"
for item in result.new_step_items
)
if schema_version == "1.6" and decision.startswith("legacy_"):
assert isinstance(result.next_step, NextStepInterruption)
future = ToolApprovalItem(
agent=resumed_duplicate,
raw_item=get_function_tool_call("needs_ok", json.dumps({"text": "two"}), call_id="future"),
)
# Reconciliation honors the current decision without moving future scope.
assert (
restored._context.get_approval_status("needs_ok", "future", current_invocation=future)
is None
)
async def _approved_handoff_session_state(streamed: bool):
"""Pause on an approval-gated call that shares its response with a handoff."""
effects: list[int] = []
guardrail_calls: list[str] = []
hook_calls: list[str] = []
handoff_calls: list[str] = []
class CountingHooks(RunHooks[Any]):
async def on_tool_start(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
tool: Tool,
) -> None:
hook_calls.append("tool-start")
async def on_tool_end(
self,
context: RunContextWrapper[Any],
agent: Agent[Any],
tool: Tool,
result: object,
) -> None:
hook_calls.append("tool-end")
@tool_input_guardrail
def record_input(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
guardrail_calls.append("input")
return ToolGuardrailFunctionOutput.allow(output_info="input-checked")
@tool_output_guardrail
def record_output(_data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
guardrail_calls.append("output")
return ToolGuardrailFunctionOutput.allow(output_info="output-checked")
@tool(
needs_approval=True,
tool_input_guardrails=[record_input],
tool_output_guardrails=[record_output],
)
async def charge(amount: int) -> str:
effects.append(amount)
return "receipt-7"
model = ScriptedModel(
[
[
get_function_tool_call("charge", '{"amount":7}', call_id="charge-1"),
get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1"),
],
[get_text_message("done")],
[get_text_message("fresh")],
]
)
delegate = Agent(name="delegate", model=model)
route = handoff(delegate, on_handoff=lambda _context: handoff_calls.append("handoff"))
agent = Agent(name="triage", model=model, tools=[charge], handoffs=[route])
hooks = CountingHooks()
session = _FailingResumeSession()
paused = await _run_session_resume(
agent,
"charge 7 then hand off",
session,
streamed,
hooks,
)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
return agent, model, session, state, effects, guardrail_calls, hook_calls, handoff_calls, hooks
def _call_pair(items: list[TResponseInputItem], call_id: str) -> list[str]:
return [
str(item.get("type"))
for item in items
if isinstance(item, dict) and item.get("call_id") == call_id
]
def _guardrail_output_info(state: RunState[Any]) -> tuple[list[Any], list[Any]]:
return (
[item.output.output_info for item in state._tool_input_guardrail_results],
[item.output.output_info for item in state._tool_output_guardrail_results],
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failing_streamed,retry_streamed", [(False, False), (False, True), (True, False), (True, True)]
)
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
@pytest.mark.parametrize("failure", ["before", "after"], ids=["atomic-failure", "lost-ack"])
async def test_resumed_handoff_session_append_is_recovered_before_next_model(
failing_streamed: bool, retry_streamed: bool, round_trip: bool, failure: str
) -> None:
(
agent,
model,
session,
state,
effects,
guardrail_calls,
hook_calls,
handoff_calls,
hooks,
) = await _approved_handoff_session_state(failing_streamed)
session.failure = failure
if failing_streamed:
failed_result = Runner.run_streamed(
agent,
state,
session=session,
run_config=RunConfig(tracing_disabled=True),
hooks=hooks,
)
with pytest.raises(RuntimeError) as error:
async for _ in failed_result.stream_events():
pass
state = failed_result.to_state()
else:
with pytest.raises(RuntimeError) as error:
await _run_session_resume(agent, state, session, False, hooks)
assert error.value is session.error
assert effects == [7]
assert guardrail_calls == ["input", "output"]
assert hook_calls == ["tool-start", "tool-end"]
assert handoff_calls == ["handoff"]
assert len(model.calls) == 1
assert _guardrail_output_info(state) == (["input-checked"], ["output-checked"])
failed_payload = state.to_json()
pending_write = cast(dict[str, Any], failed_payload["pending_session_write"])
pending_items = cast(list[TResponseInputItem], pending_write["items"])
assert _call_pair(pending_items, "charge-1") == ["function_call_output"]
assert _call_pair(pending_items, "handoff-1") == ["function_call_output"]
if round_trip:
state = await RunState.from_json(agent, failed_payload)
assert _guardrail_output_info(state) == (["input-checked"], ["output-checked"])
assert state._current_agent is not None and state._current_agent.name == "delegate"
result = await _run_session_resume(agent, state, session, retry_streamed, hooks)
assert result.final_output == "done"
assert result.last_agent.name == "delegate"
assert effects == [7]
assert guardrail_calls == ["input", "output"]
assert hook_calls == ["tool-start", "tool-end"]
assert handoff_calls == ["handoff"]
assert [item.output.output_info for item in result.tool_input_guardrail_results] == [
"input-checked"
]
assert [item.output.output_info for item in result.tool_output_guardrail_results] == [
"output-checked"
]
assert len(model.calls) == 2
expected_pair = ["function_call", "function_call_output"]
stored = await session.get_items()
assert _call_pair(stored, "charge-1") == expected_pair
assert _call_pair(stored, "handoff-1") == expected_pair
assert _call_pair(result.to_input_list(), "charge-1") == expected_pair
assert _call_pair(result.to_input_list(), "handoff-1") == expected_pair
assert "pending_session_write" not in result.to_state().to_json()
@pytest.mark.asyncio
async def test_fresh_streamed_handoff_preserves_agent_after_session_append_failure() -> None:
"""A fresh (non-resumed) streamed run's generic-loop handoff branch must publish the new
agent and next-step state before the fallible session append, mirroring the fix already
applied to the is_resumed_state branch covered by
test_resumed_handoff_session_append_is_recovered_before_next_model. Every fresh streamed
run passes through this branch, not just resumed ones.
"""
model = ScriptedModel(
[
[get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1")],
[get_text_message("done")],
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(name="triage", model=model, handoffs=[delegate])
session = _FailSecondAddItemsSession()
failed_result = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
with pytest.raises(RuntimeError) as error:
async for _ in failed_result.stream_events():
pass
assert error.value is session.error
state = failed_result.to_state()
assert state._current_agent is not None
assert state._current_agent.name == "delegate"
assert failed_result.current_agent.name == "delegate"
result = await _run_session_resume(triage, state, session, False)
assert result.final_output == "done"
assert result.last_agent.name == "delegate"
assert len(model.calls) == 2
expected_pair = ["function_call", "function_call_output"]
stored = await session.get_items()
assert _call_pair(stored, "handoff-1") == expected_pair
assert "pending_session_write" not in result.to_state().to_json()
@pytest.mark.asyncio
async def test_fresh_streamed_handoff_publishes_agent_update_before_session_append_failure() -> (
None
):
"""A yielding Session failure still delivers the completed handoff's agent update."""
model = ScriptedModel(
[
[get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1")],
[get_text_message("done")],
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(name="triage", model=model, handoffs=[delegate])
session = _FailSecondAddItemsSessionWithYield()
failed_result = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
collected_events: list[Any] = []
caught: RuntimeError | None = None
try:
async for event in failed_result.stream_events():
collected_events.append(event)
except RuntimeError as error:
caught = error
assert caught is session.error
assert any(
isinstance(event, AgentUpdatedStreamEvent) and event.new_agent.name == "delegate"
for event in collected_events
)
@pytest.mark.asyncio
async def test_fresh_streamed_handoff_drains_agent_update_event_for_slow_consumer() -> None:
"""A session-append failure in the generic-loop handoff branch must mark itself for
stream-event draining, so a consumer that falls even slightly behind the producer (an
ordinary per-event delay, not a contrived zero-delay reader) still observes the
already-queued ``AgentUpdatedStreamEvent`` before the error surfaces.
test_fresh_streamed_handoff_publishes_agent_update_before_session_append_failure's
zero-delay consumer passes even without draining, since it never falls behind the
producer; this test exercises the actual drain guarantee stream_events() provides via
_mark_error_to_drain_stream_events()/_should_drain_stream_events_before_raising().
"""
model = ScriptedModel(
[
[get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1")],
[get_text_message("done")],
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(name="triage", model=model, handoffs=[delegate])
session = _FailSecondAddItemsSessionWithYield()
failed_result = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
collected_events: list[Any] = []
caught: RuntimeError | None = None
try:
async for event in failed_result.stream_events():
# An ordinary bit of per-event consumer work, enough to fall behind the producer.
await asyncio.sleep(0.001)
collected_events.append(event)
except RuntimeError as error:
caught = error
assert caught is session.error
assert any(
isinstance(event, AgentUpdatedStreamEvent) and event.new_agent.name == "delegate"
for event in collected_events
)
@pytest.mark.asyncio
@pytest.mark.parametrize("raises_error", [False, True], ids=["tripwire", "guardrail-error"])
@pytest.mark.parametrize("retry_streamed", [False, True])
async def test_fresh_streamed_handoff_failed_guardrail_state_cannot_resume(
raises_error: bool,
retry_streamed: bool,
) -> None:
handoff_observed = asyncio.Event()
guardrail_error = RuntimeError("guardrail backend failed")
@input_guardrail(run_in_parallel=True)
async def pending_guardrail(
ctx: RunContextWrapper[Any],
agent: Agent[Any],
input: str | list[TResponseInputItem],
) -> GuardrailFunctionOutput:
await asyncio.wait_for(handoff_observed.wait(), timeout=5)
if raises_error:
raise guardrail_error
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=True)
model = ScriptedModel(
[[get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1")]]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(
name="triage", model=model, handoffs=[delegate], input_guardrails=[pending_guardrail]
)
session = SimpleListSession()
streamed_result = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
events: list[Any] = []
expected_error = RuntimeError if raises_error else InputGuardrailTripwireTriggered
with pytest.raises(expected_error):
async for event in streamed_result.stream_events():
events.append(event)
if getattr(event, "name", None) == "handoff_occured":
handoff_observed.set()
await asyncio.sleep(0)
assert handoff_observed.is_set()
assert streamed_result.current_agent is triage
assert not any(
isinstance(event, AgentUpdatedStreamEvent) and event.new_agent is delegate
for event in events
)
state = streamed_result.to_state()
assert state.to_json()["terminal_unrecoverable"] is True
restored = await RunState.from_json(triage, state.to_json())
assert restored._current_agent is triage
for checkpoint in (state, restored):
with pytest.raises(UserError, match="cannot be resumed"):
await _run_session_resume(triage, checkpoint, session, retry_streamed)
assert len(model.calls) == 1
assert _call_pair(await session.get_items(), "handoff-1") == []
@pytest.mark.asyncio
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
async def test_fresh_streamed_handoff_replays_deferred_compaction_after_resume(
round_trip: bool,
) -> None:
"""A checkpointed handoff batch that fails to append and later settles via a separate,
standalone resume_pending_session_write() call (the generic resume-startup path in
run.py/run_loop.py, not the original save_result_to_session() call) must still apply the
same post-write Responses compaction decision save_result_to_session would have applied
inline, instead of silently and permanently losing it. See
.agents/references/session-persistence.md.
Uses a should_trigger_compaction hook keyed on response_id (as a caller doing per-turn
compaction routing would) to make the loss observable: without the fix, the handoff's own
response_id is never evaluated by the hook at all, and the deferral it would have set is
never recorded, so the later forced compaction on the delegate's turn never happens either.
"""
hook_calls: list[str | None] = []
def should_trigger_compaction(context: dict[str, Any]) -> bool:
hook_calls.append(context["response_id"])
return context["response_id"] == "resp-handoff"
compact_calls: list[list[TResponseInputItem]] = []
async def compact(**kwargs: Any) -> SimpleNamespace:
items = copy.deepcopy(kwargs["input"])
compact_calls.append(items)
return SimpleNamespace(output=items, usage=None)
backend = _FailSecondAddItemsSession()
session = OpenAIResponsesCompactionSession(
"compaction-handoff-test",
underlying_session=backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode="input",
should_trigger_compaction=should_trigger_compaction,
)
model = ScriptedModel(
[
{
"output": [
get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1")
],
"response_id": "resp-handoff",
},
{"output": [get_text_message("done")], "response_id": "resp-delegate"},
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(name="triage", model=model, handoffs=[delegate])
failed_result = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
with pytest.raises(RuntimeError) as error:
async for _ in failed_result.stream_events():
pass
assert error.value is backend.error
assert hook_calls == []
assert compact_calls == []
state = failed_result.to_state()
assert state._pending_session_write is not None
assert state._pending_session_write.get("response_id") == "resp-handoff"
assert state._pending_session_write.get("has_local_tool_outputs") is True
if round_trip:
payload = state.to_json()
assert payload["$schemaVersion"] == CURRENT_SCHEMA_VERSION
state = await RunState.from_json(triage, payload)
result = await _run_session_resume(triage, state, session, False)
assert result.final_output == "done"
# The handoff's own response_id must have been evaluated by the decision hook (and
# deferred), not skipped -- and, because force-compaction short-circuits the hook, it must
# be the only response_id the hook ever saw.
assert hook_calls == ["resp-handoff"]
# The deferred decision must actually have been forced through on the delegate's own save,
# i.e. the compact API was invoked at all -- not just checked and declined.
assert len(compact_calls) == 1
@pytest.mark.asyncio
async def test_fresh_streamed_handoff_retains_checkpoint_when_post_write_compaction_fails() -> None:
"""If the post-write compaction decision raises after a checkpointed handoff batch's append
has already settled, the checkpoint (``_pending_session_write``) must survive so a later
retry can redo just the compaction step -- clearing it before the fallible compaction call
would silently and permanently lose the requested deferred/forced compaction with no way to
recover it. See .agents/references/session-persistence.md.
"""
hook_calls: list[str | None] = []
compaction_error = RuntimeError("compaction decision hook exploded")
should_fail = True
def should_trigger_compaction(context: dict[str, Any]) -> bool:
hook_calls.append(context["response_id"])
if context["response_id"] == "resp-handoff" and should_fail:
raise compaction_error
return context["response_id"] == "resp-handoff"
compact_calls: list[list[TResponseInputItem]] = []
async def compact(**kwargs: Any) -> SimpleNamespace:
items = copy.deepcopy(kwargs["input"])
compact_calls.append(items)
return SimpleNamespace(output=items, usage=None)
backend = _FailSecondAddItemsSession()
session = OpenAIResponsesCompactionSession(
"compaction-handoff-failure-test",
underlying_session=backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode="input",
should_trigger_compaction=should_trigger_compaction,
)
model = ScriptedModel(
[
{
"output": [
get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1")
],
"response_id": "resp-handoff",
},
{"output": [get_text_message("done")], "response_id": "resp-delegate"},
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(name="triage", model=model, handoffs=[delegate])
failed_result = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
with pytest.raises(RuntimeError) as append_error:
async for _ in failed_result.stream_events():
pass
assert append_error.value is backend.error
state = failed_result.to_state()
assert state._pending_session_write is not None
# Resume: the append itself now succeeds (the backend's failure was one-shot), but the
# compaction decision hook raises for the handoff's own response_id.
with pytest.raises(RuntimeError) as compaction_error_info:
await _run_session_resume(triage, state, session, False)
assert compaction_error_info.value is compaction_error
# The checkpoint must still be present so a later retry can redo compaction alone, instead
# of the handoff's requested compaction being silently and permanently lost.
assert state._pending_session_write is not None
assert state._pending_session_write.get("response_id") == "resp-handoff"
# Retry: the hook no longer fails. The append must not be repeated (no duplicate items in
# session history), but compaction must actually run this time.
should_fail = False
hook_calls.clear()
result = await _run_session_resume(triage, state, session, False)
assert result.final_output == "done"
assert hook_calls == ["resp-handoff"]
assert len(compact_calls) == 1
stored = await session.get_items()
handoff_pair = [
str(item.get("type"))
for item in stored
if isinstance(item, dict) and item.get("call_id") == "handoff-1"
]
assert handoff_pair == ["function_call", "function_call_output"]
@pytest.mark.asyncio
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
@pytest.mark.parametrize("retry_streamed", [False, True], ids=["run", "streamed"])
async def test_handoff_resume_after_cancelled_compaction_commits(
round_trip: bool, retry_streamed: bool, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""Control SQLite's worker before commit while exercising public runner cancellation."""
backend = SQLiteSession("cancelled-compaction", tmp_path / "session.db")
replacement_started = asyncio.Event()
release_replacement = threading.Event()
loop = asyncio.get_running_loop()
compacted: list[TResponseInputItem] = [{"role": "assistant", "content": "compacted history"}]
batches: list[list[TResponseInputItem]] = []
original_insert = backend._insert_items
def insert_items(conn: sqlite3.Connection, items: list[TResponseInputItem]) -> None:
original_insert(conn, items)
batches.append(copy.deepcopy(items))
if items == compacted:
loop.call_soon_threadsafe(replacement_started.set)
assert release_replacement.wait(timeout=10)
monkeypatch.setattr(backend, "_insert_items", insert_items)
compact_calls = 0
async def compact(**kwargs: Any) -> SimpleNamespace:
nonlocal compact_calls
compact_calls += 1
return SimpleNamespace(output=compacted, usage=None)
session = OpenAIResponsesCompactionSession(
"cancelled-compaction",
backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode="input",
should_trigger_compaction=lambda context: context["response_id"] == "resp-handoff",
)
handoff_calls: list[str] = []
def retain_message(data: HandoffInputData) -> HandoffInputData:
return data.clone(
new_items=tuple(item for item in data.new_items if isinstance(item, MessageOutputItem))
)
model = ScriptedModel(
[
{
"output": [
get_text_message("delegating"),
get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1"),
],
"response_id": "resp-handoff",
},
{"output": [get_text_message("done")], "response_id": "resp-delegate"},
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(
name="triage",
model=model,
handoffs=[
handoff(
delegate,
input_filter=retain_message,
on_handoff=lambda _: handoff_calls.append("handoff"),
)
],
)
result = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
async def drain() -> None:
async for _ in result.stream_events():
pass
consumer = asyncio.create_task(drain())
try:
await asyncio.wait_for(replacement_started.wait(), timeout=10)
result.cancel()
release_replacement.set()
await asyncio.wait_for(consumer, timeout=10)
assert await session.get_items() == compacted
assert len(model.calls) == 1
state = result.to_state()
assert state._pending_session_write is None
if round_trip:
state = await RunState.from_json(triage, json.loads(state.to_string()))
resumed = await _run_session_resume(triage, state, session, retry_streamed)
assert resumed.final_output == "done"
assert len(model.calls) == 2
assert handoff_calls == ["handoff"]
assert compact_calls == 1
assert len(batches) == 4 # Initial input, handoff message, replacement, final output.
assert await session.get_items() == compacted + [resumed.new_items[-1].to_input_item()]
assert "pending_session_write" not in resumed.to_state().to_json()
finally:
release_replacement.set()
result.cancel()
await asyncio.gather(consumer, return_exceptions=True)
backend.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("retry_streamed", [False, True], ids=["run", "streamed"])
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
@pytest.mark.parametrize("restore_failure", [None, "empty", "partial"])
async def test_handoff_resume_retries_rolled_back_compaction(
retry_streamed: bool, round_trip: bool, restore_failure: str | None
) -> None:
"""A legacy Session rollback must retain compaction ownership without repeating the append."""
compacted: list[TResponseInputItem] = [{"role": "assistant", "content": "compacted history"}]
class FailReplacementSession(SimpleListSession):
fail = True
async def add_items(self, items: list[TResponseInputItem]) -> None:
if items == compacted and self.fail:
self.fail = False
raise RuntimeError("replacement failed before commit")
if not self.fail and restore_failure is not None:
if restore_failure == "partial":
await super().add_items(items[:1])
raise RuntimeError("restoration failed")
await super().add_items(items)
backend = FailReplacementSession()
compact_calls: list[list[TResponseInputItem]] = []
async def compact(**kwargs: Any) -> SimpleNamespace:
compact_calls.append(copy.deepcopy(kwargs["input"]))
return SimpleNamespace(output=compacted, usage=None)
session = OpenAIResponsesCompactionSession(
"rollback-compaction",
backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode="input",
should_trigger_compaction=lambda context: context["response_id"] == "resp-handoff",
)
handoff_calls: list[str] = []
def retain_message(data: HandoffInputData) -> HandoffInputData:
return data.clone(
new_items=tuple(item for item in data.new_items if isinstance(item, MessageOutputItem))
)
model = ScriptedModel(
[
{
"output": [
get_text_message("delegating"),
get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1"),
],
"response_id": "resp-handoff",
},
{"output": [get_text_message("done")], "response_id": "resp-delegate"},
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(
name="triage",
model=model,
handoffs=[
handoff(
delegate,
input_filter=retain_message,
on_handoff=lambda _: handoff_calls.append("handoff"),
)
],
)
failed = Runner.run_streamed(
triage, "hello", session=session, run_config=RunConfig(tracing_disabled=True)
)
with pytest.raises(RuntimeError, match="replacement failed before commit"):
async for _ in failed.stream_events():
pass
restored_history = await session.get_items()
assert len(model.calls) == 1
state = failed.to_state()
if round_trip:
state = await RunState.from_json(triage, json.loads(state.to_string()))
if restore_failure is not None:
assert len(restored_history) == (1 if restore_failure == "partial" else 0)
with pytest.raises(UserError, match="Cannot reconcile the pending Session write"):
await _run_session_resume(triage, state, session, retry_streamed)
assert state._pending_session_write is not None
assert state._pending_session_write["append_acknowledged"] is True
assert await session.get_items() == restored_history
assert len(model.calls) == 1
assert len(compact_calls) == 1
assert handoff_calls == ["handoff"]
return
assert len(restored_history) == 2
resumed = await _run_session_resume(triage, state, session, retry_streamed)
assert resumed.final_output == "done"
assert compact_calls == [restored_history, restored_history]
assert handoff_calls == ["handoff"]
assert len(model.calls) == 2
assert await session.get_items() == compacted + [resumed.new_items[-1].to_input_item()]
assert "pending_session_write" not in resumed.to_state().to_json()
@pytest.mark.asyncio
async def test_json_compaction_retry_does_not_promote_filtered_session_history() -> None:
backend = _FailingResumeSession()
hidden: TResponseInputItem = {"role": "user", "content": "synthetic omitted history"}
await backend.add_items([hidden])
backend.fail_on_output = "delegating"
compact_calls: list[list[TResponseInputItem]] = []
async def compact(**kwargs: Any) -> SimpleNamespace:
compact_calls.append(kwargs["input"])
return SimpleNamespace(output=[], usage=None)
session = OpenAIResponsesCompactionSession(
"filtered-retry",
backend,
client=cast(Any, SimpleNamespace(responses=SimpleNamespace(compact=compact))),
compaction_mode="input",
should_trigger_compaction=lambda context: context["response_id"] == "resp-handoff",
)
model = ScriptedModel(
[
{
"output": [
get_text_message("delegating"),
get_function_tool_call("transfer_to_delegate", "{}", call_id="handoff-1"),
],
"response_id": "resp-handoff",
},
{"output": [get_text_message("done")], "response_id": "resp-delegate"},
]
)
delegate = Agent(name="delegate", model=model)
triage = Agent(
name="triage",
model=model,
handoffs=[
handoff(
delegate,
input_filter=lambda data: data.clone(
new_items=tuple(
item for item in data.new_items if isinstance(item, MessageOutputItem)
)
),
)
],
)
failed = Runner.run_streamed(
triage,
"hello",
session=session,
run_config=RunConfig(
tracing_disabled=True, session_input_callback=lambda _history, new: new
),
)
with pytest.raises(RuntimeError, match="session append failed"):
async for _ in failed.stream_events():
pass
payload = json.loads(failed.to_state().to_string())
assert "compaction_model_exchange" in payload["pending_session_write"]
state = await RunState.from_json(triage, payload)
resumed = await _run_session_resume(triage, state, session, False)
assert resumed.final_output == "done"
assert len(model.calls) == 2
assert all(hidden not in call.input for call in model.calls)
assert compact_calls == []
assert (await session.get_items())[0] == hidden
class _TerminalLifecycleHooks(RunHooks[Any]):
"""Count the agent lifecycle hooks an application can attach its own effects to."""
def __init__(self) -> None:
self.starts = 0
self.ends: list[str] = []
async def on_agent_start(self, context: Any, agent: Agent[Any]) -> None:
self.starts += 1
async def on_agent_end(self, context: Any, agent: Agent[Any], output: Any) -> None:
self.ends.append(str(output))
async def _terminal_output_session_state(
streamed: bool,
session: Session | None = None,
hooks: RunHooks[Any] | None = None,
):
"""Pause on an approval whose tool output becomes the terminal agent output."""
effects: list[int] = []
@tool(needs_approval=True)
async def charge(amount: int) -> str:
effects.append(amount)
return "receipt-7"
model = ScriptedModel(
[
[get_function_tool_call("charge", '{"amount":7}', call_id="charge-1")],
[get_text_message("retry-final")],
]
)
agent = Agent(
name="payment",
model=model,
tools=[charge],
tool_use_behavior="stop_on_first_tool",
)
session = session if session is not None else _FailingResumeSession()
paused = await _run_session_resume(agent, "charge 7", session, streamed, hooks=hooks)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
return agent, model, session, state, effects
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failing_streamed,retry_streamed", [(False, False), (False, True), (True, False), (True, True)]
)
@pytest.mark.parametrize("round_trip", [False, True], ids=["live", "json"])
@pytest.mark.parametrize("failure", ["before", "after"], ids=["atomic-failure", "lost-ack"])
async def test_terminal_session_append_failure_rejects_every_later_resume(
failing_streamed: bool, retry_streamed: bool, round_trip: bool, failure: str
) -> None:
"""An accepted terminal output whose append failed is not resumable, and never replayed."""
hooks = _TerminalLifecycleHooks()
agent, model, session, state, effects = await _terminal_output_session_state(
failing_streamed, hooks=hooks
)
session.failure = failure
with pytest.raises(RuntimeError) as error:
await _run_session_resume(agent, state, session, failing_streamed, hooks=hooks)
assert error.value is session.error
# The output, its guardrails, and its terminal hooks all completed exactly once.
assert effects == [7]
assert len(model.calls) == 1
assert hooks.ends == ["receipt-7"]
starts_after_failure = hooks.starts
assert state.to_json()["terminal_unrecoverable"] is True
if round_trip:
state = await RunState.from_json(agent, state.to_json())
# Every later resume fails closed, including a second one.
for _ in range(2):
with pytest.raises(UserError, match="cannot be resumed"):
await _run_session_resume(agent, state, session, retry_streamed, hooks=hooks)
assert len(model.calls) == 1
assert effects == [7]
assert hooks.starts == starts_after_failure
assert hooks.ends == ["receipt-7"]
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_unrecoverable_terminal_state_rejects_before_any_resumed_work(
streamed: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
"""The rejection precedes Session reconciliation and sandbox preparation."""
agent, model, session, state, effects = await _terminal_output_session_state(streamed)
session.failure = "before"
with pytest.raises(RuntimeError, match="session append failed"):
await _run_session_resume(agent, state, session, streamed)
async def _fail_get_items(*args: Any, **kwargs: Any) -> list[TResponseInputItem]:
raise AssertionError("Session reconciliation must not run for a rejected terminal state")
async def _fail_prepare_agent(*args: Any, **kwargs: Any):
raise AssertionError("sandbox preparation must not run for a rejected terminal state")
monkeypatch.setattr(type(session), "get_items", _fail_get_items)
monkeypatch.setattr(SandboxRuntime, "prepare_agent", _fail_prepare_agent)
restored = await RunState.from_json(agent, state.to_json())
with pytest.raises(UserError, match="cannot be resumed"):
await _run_session_resume(agent, restored, session, not streamed)
assert len(model.calls) == 1
assert effects == [7]
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True])
async def test_terminal_marker_is_cleared_once_the_turn_is_persisted(streamed: bool) -> None:
"""A terminal turn that persists cleanly leaves a normal, unmarked result."""
agent, model, session, state, effects = await _terminal_output_session_state(streamed)
result = await _run_session_resume(agent, state, session, streamed)
assert result.final_output == "receipt-7"
assert effects == [7]
assert "terminal_unrecoverable" not in result.to_state().to_json()
assert _charge_pair(await session.get_items()) == ["function_call", "function_call_output"]
@pytest.mark.asyncio
async def test_terminal_marker_rejects_an_older_schema_label() -> None:
"""The marker is only honored on the schema boundary that introduced it."""
agent, _, session, state, _ = await _terminal_output_session_state(False)
session.failure = "before"
with pytest.raises(RuntimeError, match="session append failed"):
await _run_session_resume(agent, state, session, False)
payload = state.to_json()
for entry in payload["context"].pop("function_tool_approvals", []):
payload["context"]["approvals"][entry["tool_key"]] = entry["decision"]
payload["$schemaVersion"] = "1.16"
with pytest.raises(UserError, match="terminal marker is invalid"):
await RunState.from_json(agent, payload)
@pytest.mark.asyncio
async def test_failed_stream_result_checkpoint_keeps_the_terminal_marker() -> None:
"""A checkpoint taken from a failed streamed run stays closed to resumes.
A streamed result exists before its terminal append does, so `to_state()` is reachable on the
failed attempt. If that snapshot dropped the marker it would look like an ordinary resumable
state and bypass the rejection entirely.
"""
agent, model, session, state, effects = await _terminal_output_session_state(True)
session.failure = "before"
streamed = Runner.run_streamed(
agent, state, session=session, run_config=RunConfig(tracing_disabled=True)
)
with pytest.raises(RuntimeError, match="session append failed"):
async for _ in streamed.stream_events():
pass
checkpoint = streamed.to_state()
assert checkpoint.to_json()["terminal_unrecoverable"] is True
restored = await RunState.from_json(agent, checkpoint.to_json())
for candidate in (checkpoint, restored):
with pytest.raises(UserError, match="cannot be resumed"):
await _run_session_resume(agent, candidate, session, False)
assert len(model.calls) == 1
assert effects == [7]
@pytest.mark.asyncio
async def test_max_turns_handler_output_is_marked_before_it_is_persisted() -> None:
"""The max-turns fallback is a final output too, so its failed append closes the state."""
handler_calls: list[str] = []
hooks = _TerminalLifecycleHooks()
@tool(needs_approval=True)
async def charge(amount: int) -> str:
return "receipt-7"
model = ScriptedModel(
[
[get_function_tool_call("charge", '{"amount":7}', call_id="charge-1")],
[get_text_message("unused")],
]
)
agent = Agent(name="payment", model=model, tools=[charge])
session = _FailingResumeSession()
config = RunConfig(tracing_disabled=True)
paused = await Runner.run(
agent, "charge 7", session=session, run_config=config, max_turns=1, hooks=hooks
)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
def _handler(_data: Any) -> str:
handler_calls.append("handled")
return "max-turns-output"
session.fail_on_output = "max-turns-output"
with pytest.raises(RuntimeError, match="session append failed"):
await Runner.run(
agent,
state,
session=session,
run_config=config,
hooks=hooks,
error_handlers={"max_turns": _handler},
)
# The handler and its end hook each ran exactly once before the append failed.
assert handler_calls == ["handled"]
assert hooks.ends == ["max-turns-output"]
assert state.to_json()["terminal_unrecoverable"] is True
with pytest.raises(UserError, match="cannot be resumed"):
await Runner.run(
agent,
state,
session=session,
run_config=config,
hooks=hooks,
error_handlers={"max_turns": _handler},
)
assert handler_calls == ["handled"]
assert hooks.ends == ["max-turns-output"]
@pytest.mark.asyncio
async def test_max_turns_guardrail_failure_leaves_the_state_retryable() -> None:
"""A handler output that never passed its guardrails must not close the state.
`finalize_max_turns_handler_output()` drives the same save callback from its guardrail-error
path. Marking there would reject every later resume for an output the caller never received,
which is a worse outcome than the replay the marker exists to prevent.
"""
guardrail_calls: list[str] = []
@tool(needs_approval=True)
async def charge(amount: int) -> str:
return "receipt-7"
@output_guardrail
async def exploding(context: Any, agent: Agent[Any], output: Any) -> GuardrailFunctionOutput:
guardrail_calls.append(str(output))
raise RuntimeError("guardrail exploded")
model = ScriptedModel([[get_function_tool_call("charge", '{"amount":7}', call_id="charge-1")]])
agent = Agent(name="payment", model=model, tools=[charge], output_guardrails=[exploding])
session = _FailingResumeSession()
config = RunConfig(tracing_disabled=True)
paused = await Runner.run(agent, "charge 7", session=session, run_config=config, max_turns=1)
state = paused.to_state()
state.approve(state.get_interruptions()[0])
handlers: dict[str, Any] = {"max_turns": lambda _data: "max-turns-output"}
session.fail_on_output = "max-turns-output"
with pytest.raises(RuntimeError, match="session append failed"):
await Runner.run(agent, state, session=session, run_config=config, error_handlers=handlers)
assert guardrail_calls == ["max-turns-output"]
assert "terminal_unrecoverable" not in state.to_json()
# The retry reports the real guardrail failure rather than a fail-closed rejection.
with pytest.raises(RuntimeError, match="guardrail exploded"):
await Runner.run(agent, state, session=session, run_config=config, error_handlers=handlers)
@pytest.mark.asyncio
@pytest.mark.parametrize("guardrail_blocks", [False, True], ids=["allowed", "blocked"])
@pytest.mark.parametrize(
"published_sibling", [False, True], ids=["no-prior-output", "prior-output"]
)
async def test_legacy_first_turn_callback_failure_withholds_normalized_output(
guardrail_blocks: bool,
published_sibling: bool,
) -> None:
effects: list[str] = []
checked: list[str] = []
config = RunConfig(tracing_disabled=True)
def fail_after_execution(_context: Any) -> dict[str, Any]:
raise RuntimeError("post-tool callback failed")
@function_tool(needs_approval=True, custom_data_extractor=fail_after_execution)
async def charge() -> str:
effects.append("charged")
return "receipt-7"
@output_guardrail
def terminal(_context: Any, _agent: Any, output: Any) -> GuardrailFunctionOutput:
checked.append(output)
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=guardrail_blocks)
@function_tool(needs_approval=True)
async def prelude() -> str:
return "published-receipt"
calls = [get_function_tool_call("charge", "{}", call_id="charge-1")]
if published_sibling:
calls.insert(0, get_function_tool_call("prelude", "{}", call_id="prelude-1"))
model = ScriptedModel([calls])
agent = Agent(
name="legacy-retry",
model=model,
tools=[prelude, charge],
tool_use_behavior=StopAtTools(stop_at_tool_names=["charge"]),
output_guardrails=[terminal],
)
paused = await Runner.run(agent, "charge once", run_config=config)
if published_sibling:
partial_state = paused.to_state()
partial_state.approve(partial_state.get_interruptions()[0])
paused = await Runner.run(agent, partial_state, run_config=config)
payload = json.loads(paused.to_state().to_string())
# Schema 1.16 predates current-response ownership and held Session writes.
payload["$schemaVersion"] = "1.16"
payload["context"].pop("function_tool_approvals", None)
payload.pop("generated_session_item_indexes", None)
payload.pop("current_response_generated_item_ownership", None)
payload.pop("pending_session_write", None)
state = await RunState.from_json(agent, payload)
state.approve(state.get_interruptions()[0])
failed = Runner.run_streamed(agent, state, run_config=config)
with pytest.raises(UserError, match="post-tool callback failed"):
async for _ in failed.stream_events():
pass
assert effects == ["charged"]
assert checked == []
modes: tuple[ToInputListMode, ...] = ("preserve_all", "normalized")
for mode in modes:
assert [
item.get("output")
for item in failed.to_input_list(mode=mode)
if item.get("type") == "function_call_output"
] == (["published-receipt"] if published_sibling else [])
# Resume the same source so later public reads cannot borrow its mutated boundary.
retry = Runner.run_streamed(agent, state, run_config=config)
events = []
if guardrail_blocks:
with pytest.raises(OutputGuardrailTripwireTriggered):
async for event in retry.stream_events():
events.append(event)
else:
async for event in retry.stream_events():
events.append(event)
assert retry.final_output == "receipt-7"
assert checked == ["receipt-7"]
assert effects == ["charged"]
assert [
event.item.output
for event in events
if event.type == "run_item_stream_event" and isinstance(event.item, ToolCallOutputItem)
] == ([] if guardrail_blocks else ["receipt-7"])
for mode in modes:
assert [
item.get("output")
for item in failed.to_input_list(mode=mode)
if item.get("type") == "function_call_output"
] == (["published-receipt"] if published_sibling else [])
@pytest.mark.asyncio
@pytest.mark.parametrize("streamed", [False, True], ids=["run", "streamed"])
@pytest.mark.parametrize("failure", ["extractor", "tool_use_behavior"])
async def test_legacy_later_turn_callback_failure_rejects_incomplete_retry(
streamed: bool,
failure: str,
) -> None:
effects: list[str] = []
charged_decisions: list[str] = []
def fail_after_execution(_context: Any) -> dict[str, Any]:
raise RuntimeError("post-tool callback failed")
@function_tool
async def before() -> str:
return "before-ok"
@function_tool(
needs_approval=True,
custom_data_extractor=fail_after_execution if failure == "extractor" else None,
)
async def charge() -> str:
effects.append("charged")
return "receipt-7"
def decide_final_output(_context: Any, results: list[Any]) -> ToolsToFinalOutputResult:
if any(result.tool.name == "charge" for result in results):
charged_decisions.append("charge")
if len(charged_decisions) == 1:
raise RuntimeError("post-tool callback failed")
return ToolsToFinalOutputResult(is_final_output=False, final_output=None)
model = ScriptedModel(
[
[get_function_tool_call("before", "{}", call_id="before-1")],
[get_function_tool_call("charge", "{}", call_id="charge-1")],
[get_text_message("done")],
]
)
agent = Agent(
name="legacy-later-turn",
model=model,
tools=[before, charge],
tool_use_behavior=decide_final_output
if failure == "tool_use_behavior"
else "run_llm_again",
)
paused = await _run_session_resume(agent, "charge once", None, False)
assert [item.output for item in paused.new_items if isinstance(item, ToolCallOutputItem)] == [
"before-ok"
]
payload = json.loads(paused.to_state().to_string())
# Schema 1.16 cannot prove which saved items belong to a later model response.
payload["$schemaVersion"] = "1.16"
payload["context"].pop("function_tool_approvals", None)
payload.pop("generated_session_item_indexes", None)
payload.pop("current_response_generated_item_ownership", None)
payload.pop("pending_session_write", None)
state = await RunState.from_json(agent, payload)
state.approve(state.get_interruptions()[0])
callback_error = UserError if failure == "extractor" else RuntimeError
if streamed:
failed = Runner.run_streamed(agent, state, run_config=RunConfig(tracing_disabled=True))
with pytest.raises(callback_error, match="post-tool callback failed"):
async for _ in failed.stream_events():
pass
modes: tuple[ToInputListMode, ...] = ("preserve_all", "normalized")
for mode in modes:
assert [
item.get("output")
for item in failed.to_input_list(mode=mode)
if item.get("type") == "function_call_output"
] == ["before-ok"]
state = failed.to_state()
else:
with pytest.raises(callback_error, match="post-tool callback failed"):
await _run_session_resume(agent, state, None, streamed)
assert effects == ["charged"]
assert len(model.calls) == 2
restored = await RunState.from_json(agent, json.loads(state.to_string()))
with pytest.raises(UserError, match="cannot be resumed"):
await _run_session_resume(agent, restored, None, not streamed)
assert effects == ["charged"]
assert charged_decisions == (["charge"] if failure == "tool_use_behavior" else [])
assert len(model.calls) == 2