openai-agents-python/tests/test_repl.py
Kazuhiro Sera d4d835e719
fix: tool approval handling in the demo REPL (#6045)
Fix tool approval handling in the demo REPL
2026-11-16 22:46:19 +00:00

241 lines
8 KiB
Python

import re
import pytest
from agents import Agent, RunContextWrapper, ShellTool, ToolSearchTool, handoff, run_demo_loop
from agents.decorators import tool
from agents.testing import ScriptedModel, assistant_message, function_call
from agents.tool import DEFAULT_APPROVAL_REJECTION_MESSAGE
from .test_responses import (
get_function_tool,
get_function_tool_call,
get_handoff_tool_call,
get_text_input_item,
get_text_message,
)
from .utils.hitl import make_shell_call
@pytest.mark.asyncio
async def test_run_demo_loop_conversation(monkeypatch, capsys):
model = ScriptedModel()
model.extend([[get_text_message("hello")], [get_text_message("good")]])
agent = Agent(name="test", model=model)
inputs = iter(["Hi", "How are you?", "quit"])
monkeypatch.setattr("builtins.input", lambda _=" > ": next(inputs))
await run_demo_loop(agent, stream=False)
output = capsys.readouterr().out
assert "hello" in output
assert "good" in output
assert model.calls[-1].input == [
get_text_input_item("Hi"),
get_text_message("hello").model_dump(exclude_unset=True),
get_text_input_item("How are you?"),
]
@pytest.mark.asyncio
async def test_run_demo_loop_streaming(monkeypatch, capsys):
model = ScriptedModel()
target_agent = Agent(name="target", model=model)
agent = Agent(
name="test",
model=model,
tools=[get_function_tool("foo", "tool_result")],
handoffs=[target_agent],
)
# A single user turn that exercises every streamed event branch:
# a tool call, the tool output, a handoff (agent update), then a text answer.
model.extend(
[
[get_function_tool_call("foo", "{}")],
[get_handoff_tool_call(target_agent)],
[get_text_message("all done")],
]
)
inputs = iter(["Hello", "exit"])
monkeypatch.setattr("builtins.input", lambda _=" > ": next(inputs))
await run_demo_loop(agent, stream=True)
output = capsys.readouterr().out
assert "all done" in output
assert "[tool called]" in output
assert "[tool output: tool_result]" in output
assert "[Agent updated: target]" in output
@pytest.mark.parametrize("stream", [False, True], ids=["nonstreaming", "streaming"])
@pytest.mark.asyncio
async def test_run_demo_loop_resolves_tool_approvals(monkeypatch, capsys, stream):
recorded: list[str] = []
@tool(needs_approval=True)
def record(value: str) -> str:
recorded.append(f"visible {value}")
return f"visible {value}"
@tool(name_override="record", defer_loading=True, needs_approval=True)
def deferred_record(value: str) -> str:
recorded.append(f"deferred {value}")
return f"deferred {value}"
done = assistant_message("All done")
model = ScriptedModel(
[
[
function_call("record", {"value": "alpha"}, call_id="call_1"),
function_call("record", {"value": "alpha"}, call_id="call_2", namespace="record"),
],
[function_call("record", {"value": "gamma"}, call_id="call_3", namespace="record")],
[done],
[assistant_message("You're welcome")],
]
)
agent = Agent(name="test", model=model, tools=[record, deferred_record, ToolSearchTool()])
inputs = iter(["Record these", "y", "", "yes", "Thanks", "quit"])
prompts: list[str] = []
def read_input(prompt: str) -> str:
prompts.append(capsys.readouterr().out + prompt)
return next(inputs)
monkeypatch.setattr("builtins.input", read_input)
await run_demo_loop(agent, stream=stream)
assert recorded == ["visible alpha", "deferred gamma"]
for prompt, value in zip(prompts[1:4], ["alpha", "alpha", "gamma"], strict=True):
assert "record" in prompt and value in prompt
namespace_label = re.compile(r"namespace\W+record\b", re.IGNORECASE)
assert not namespace_label.search(prompts[1])
assert namespace_label.search(prompts[2])
assert len(model.calls) == 4
assert all(call.streamed == stream for call in model.calls)
_, first_resume, second_resume, follow_up = model.calls
expected_outputs = [
("call_1", "visible alpha"),
("call_2", DEFAULT_APPROVAL_REJECTION_MESSAGE),
("call_3", "deferred gamma"),
]
for call, expected in [(first_resume, expected_outputs[:2]), (second_resume, expected_outputs)]:
assert [item for item in call.input if item.get("role") == "user"] == [
get_text_input_item("Record these")
]
assert (
sorted(
(item["call_id"], item["output"])
for item in call.input
if item.get("type") == "function_call_output"
)
== expected
)
assert follow_up.input == [
*second_resume.input,
done.model_dump(exclude_unset=True),
get_text_input_item("Thanks"),
]
@pytest.mark.parametrize("stream", [False, True], ids=["nonstreaming", "streaming"])
@pytest.mark.asyncio
async def test_run_demo_loop_resumes_with_handoff_context(monkeypatch, stream):
def update_context(wrapper: RunContextWrapper[str]) -> None:
wrapper.context = "after handoff"
@tool(needs_approval=True)
def read_context(wrapper: RunContextWrapper[str]) -> str:
return wrapper.context
model = ScriptedModel()
target = Agent(name="target", model=model, tools=[read_context])
source = Agent(
name="source", model=model, handoffs=[handoff(target, on_handoff=update_context)]
)
model.extend(
[
[get_handoff_tool_call(target)],
[function_call("read_context", {}, call_id="read")],
[assistant_message("done")],
]
)
inputs = iter(["Start", "yes", "quit"])
monkeypatch.setattr("builtins.input", lambda _: next(inputs))
await run_demo_loop(source, stream=stream, context="before handoff")
assert [
item["output"]
for item in model.calls[-1].input
if item.get("type") == "function_call_output" and item["call_id"] == "read"
] == ["after handoff"]
@pytest.mark.parametrize("stop", [EOFError, KeyboardInterrupt])
@pytest.mark.asyncio
async def test_run_demo_loop_exits_during_tool_approval(monkeypatch, capsys, stop):
model = ScriptedModel(
[
[
make_shell_call("call_1", commands=["echo alpha"]),
make_shell_call("call_2", commands=["echo beta"]),
]
]
)
shell = ShellTool(
executor=lambda _: pytest.fail("Unexpected shell execution"), needs_approval=True
)
agent = Agent(name="test", model=model, tools=[shell])
inputs = iter(["Run these commands", "y"])
def read_input(prompt: str) -> str:
try:
return next(inputs)
except StopIteration:
raise stop from None
monkeypatch.setattr("builtins.input", read_input)
await run_demo_loop(agent, stream=False)
output = capsys.readouterr().out
assert "echo alpha" in output and "echo beta" in output
assert len(model.calls) == 1
@pytest.mark.asyncio
async def test_run_demo_loop_exits_on_eof(monkeypatch, capsys):
model = ScriptedModel()
agent = Agent(name="test", model=model)
def raise_eof(_=" > ") -> str:
raise EOFError
monkeypatch.setattr("builtins.input", raise_eof)
await run_demo_loop(agent, stream=False)
# The loop should terminate cleanly without ever invoking the model.
assert not model.calls
@pytest.mark.asyncio
async def test_run_demo_loop_skips_empty_input(monkeypatch, capsys):
model = ScriptedModel()
model.extend([[get_text_message("hello")]])
agent = Agent(name="test", model=model)
# Empty lines are ignored; only the non-empty input reaches the runner.
inputs = iter(["", "Hi", "quit"])
monkeypatch.setattr("builtins.input", lambda _=" > ": next(inputs))
await run_demo_loop(agent, stream=False)
output = capsys.readouterr().out
assert "hello" in output
assert model.calls[-1].input == [get_text_input_item("Hi")]